use alloc::{boxed::Box, format, string::String, string::ToString, vec::Vec};
use core::fmt::{Debug, Write as _};
use core::time::Duration;
use std::collections::{HashMap, HashSet};
use std::io::{BufReader, Read as _, Seek as _, SeekFrom, Write as _};
use std::path::{Component, Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use instant_xml::FromXml;
use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle};
use ureq::Agent;
use url::Url;
use walkdir::WalkDir;
pub use ureq::{Agent as HttpAgent, AgentBuilder as HttpAgentBuilder};
const WT_DATA_EXTENSION: &str = ".wt.data";
const WT_INDEX_EXTENSION: &str = ".wt.idx";
const MANIFEST_FILE: &str = "manifest.v1";
const MANIFEST_MAGIC_V1: &[u8; 8] = b"WTS3M001";
const MANIFEST_MAGIC_V2: &[u8; 8] = b"WTS3M002";
const SEGMENT_TARGET: usize = 4 * 1024 * 1024;
const CHANGE_BLOCK_SIZE: usize = data_bucket::PAGE_SIZE;
const MAX_MANIFEST_FILES: usize = 16_384;
const MAX_MANIFEST_EXTENTS: usize = 4_194_304;
const MAX_REQUIRED_MANIFEST_BYTES: usize = 64 * 1024 * 1024;
const MAX_CAPTURE_LISTING_BYTES: usize = 8 * 1024 * 1024;
const MAX_CAPTURE_LISTING_PAGES: usize = 4096;
const MAX_CAPTURE_KEY_BYTES: usize = 64 * 1024 * 1024;
fn safe_s3_request_error(operation: &'static str, error: ureq::Error) -> eyre::Report {
match error {
ureq::Error::Status(status, _) => {
eyre::eyre!("S3 {operation} failed with HTTP status {status}")
}
ureq::Error::Transport(_) => eyre::eyre!("S3 {operation} transport failed"),
}
}
fn safe_s3_read_error(_: std::io::Error) -> eyre::Report {
eyre::eyre!("S3 response body read failed")
}
fn optional_s3_response(
response: Result<ureq::Response, ureq::Error>,
) -> eyre::Result<Option<ureq::Response>> {
match response {
Ok(response) => Ok(Some(response)),
Err(ureq::Error::Status(404, _)) => Ok(None),
Err(error) => Err(safe_s3_request_error("read", error)),
}
}
#[derive(instant_xml::FromXml)]
#[xml(
rename = "ListBucketResult",
ns("http://s3.amazonaws.com/doc/2006-03-01/")
)]
struct CaptureListingState {
#[xml(rename = "IsTruncated")]
is_truncated: bool,
}
fn parse_capture_listing(bytes: &[u8]) -> eyre::Result<rusty_s3::actions::ListObjectsV2Response> {
if bytes.len() > MAX_CAPTURE_LISTING_BYTES {
eyre::bail!("migration listing response byte limit exceeded");
}
let xml = core::str::from_utf8(bytes)?;
let state: CaptureListingState = instant_xml::from_str(xml)?;
let parsed = rusty_s3::actions::ListObjectsV2::parse_response(xml)?;
if !parsed.common_prefixes.is_empty() {
eyre::bail!("migration listing unexpectedly grouped keys");
}
match (
state.is_truncated,
parsed.next_continuation_token.as_deref(),
) {
(true, Some(token)) if !token.is_empty() => {}
(false, None) => {}
_ => eyre::bail!("migration listing has inconsistent truncation and continuation state"),
}
Ok(parsed)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct LegacyInventoryLimits {
pub max_roots: usize,
pub max_manifest_bytes: usize,
pub max_references: usize,
pub max_objects: usize,
pub max_key_bytes: usize,
pub max_metadata_bytes: usize,
}
impl Default for LegacyInventoryLimits {
fn default() -> Self {
Self {
max_roots: 128,
max_manifest_bytes: 64 * 1024 * 1024,
max_references: MAX_MANIFEST_EXTENTS,
max_objects: 1_000_000,
max_key_bytes: 4_096,
max_metadata_bytes: 256 * 1024 * 1024,
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LegacyInventoryObject {
pub key: String,
pub size: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LegacyObjectListing {
pub objects: Vec<LegacyInventoryObject>,
pub complete: bool,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LegacyRootAccounting {
pub logical_file_bytes: u64,
pub unique_chunk_bytes: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LegacyInventory {
pub listing_complete: bool,
pub current_root: LegacyRootAccounting,
pub retained_roots: Vec<LegacyRootAccounting>,
pub current_only_chunk_bytes: u64,
pub shared_chunk_bytes: u64,
pub retained_only_chunk_bytes: u64,
pub unreferenced_chunk_bytes: Option<u64>,
pub unlisted_references: Vec<[u8; 32]>,
pub preserved_objects: Vec<LegacyInventoryObject>,
}
struct DecodedInventoryRoot {
accounting: LegacyRootAccounting,
references: std::collections::BTreeMap<[u8; 32], u32>,
}
fn checked_inventory_add(total: u64, amount: u64) -> eyre::Result<u64> {
total
.checked_add(amount)
.ok_or_else(|| eyre::eyre!("legacy inventory byte count overflow"))
}
fn decode_inventory_root(
bytes: &[u8],
max_references: usize,
) -> eyre::Result<DecodedInventoryRoot> {
let manifest = TableManifest::decode(bytes)?;
let mut references = std::collections::BTreeMap::new();
let mut logical_file_bytes = 0_u64;
for file in &manifest.files {
logical_file_bytes = checked_inventory_add(logical_file_bytes, file.length)?;
for extent in &file.extents {
match references.insert(extent.hash, extent.segment_length) {
Some(previous) if previous != extent.segment_length => {
eyre::bail!("legacy roots disagree on a segment's whole length");
}
_ => {}
}
if references.len() > max_references {
eyre::bail!("legacy inventory root reference limit exceeded");
}
}
}
Ok(DecodedInventoryRoot {
accounting: LegacyRootAccounting {
logical_file_bytes,
unique_chunk_bytes: 0,
},
references,
})
}
fn validate_inventory_namespace(namespace: &str) -> eyre::Result<()> {
if namespace.is_empty()
|| !namespace.ends_with('/')
|| namespace.starts_with('/')
|| namespace.contains('\\')
|| namespace.bytes().any(|byte| byte.is_ascii_control())
|| namespace[..namespace.len() - 1]
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
{
eyre::bail!("legacy inventory namespace is not canonical");
}
Ok(())
}
fn hash_from_canonical_chunk_key(key: &str, namespace: &str) -> Option<[u8; 32]> {
let name = key.strip_prefix(namespace)?.strip_prefix("chunks/")?;
if name.len() != 64
|| !name
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return None;
}
let mut hash = [0_u8; 32];
for (index, pair) in name.as_bytes().as_chunks::<2>().0.iter().enumerate() {
let digit = |byte: u8| {
if byte.is_ascii_digit() {
byte - b'0'
} else {
byte - b'a' + 10
}
};
hash[index] = (digit(pair[0]) << 4) | digit(pair[1]);
}
Some(hash)
}
pub fn inventory_legacy_roots(
namespace: &str,
current_manifest: &[u8],
retained_manifests: &[Vec<u8>],
listing: LegacyObjectListing,
limits: LegacyInventoryLimits,
) -> eyre::Result<LegacyInventory> {
validate_inventory_namespace(namespace)?;
let root_count = retained_manifests
.len()
.checked_add(1)
.ok_or_else(|| eyre::eyre!("legacy inventory root count overflow"))?;
if root_count > limits.max_roots {
eyre::bail!("legacy inventory root limit exceeded");
}
let manifest_bytes =
retained_manifests
.iter()
.try_fold(current_manifest.len(), |total, bytes| {
total
.checked_add(bytes.len())
.ok_or_else(|| eyre::eyre!("legacy inventory manifest byte count overflow"))
})?;
if manifest_bytes > limits.max_manifest_bytes {
eyre::bail!("legacy inventory aggregate manifest byte limit exceeded");
}
if listing.objects.len() > limits.max_objects {
eyre::bail!("legacy inventory object limit exceeded");
}
let mut current = decode_inventory_root(current_manifest, limits.max_references)?;
let mut retained = Vec::with_capacity(retained_manifests.len());
let mut union_references = std::collections::BTreeMap::new();
let mut retained_union = HashSet::new();
for (hash, length) in ¤t.references {
union_references.insert(*hash, *length);
if union_references.len() > limits.max_references {
eyre::bail!("legacy inventory union reference limit exceeded");
}
}
for bytes in retained_manifests {
let root = decode_inventory_root(bytes, limits.max_references)?;
for (hash, length) in &root.references {
if let Some(previous) = union_references.insert(*hash, *length) {
if previous != *length {
eyre::bail!("legacy roots disagree on a segment's whole length");
}
}
if union_references.len() > limits.max_references {
eyre::bail!("legacy inventory union reference limit exceeded");
}
retained_union.insert(*hash);
}
retained.push(root);
}
let mut listed = std::collections::BTreeMap::new();
let mut metadata_bytes = 0_usize;
for object in listing.objects {
if object.key.len() > limits.max_key_bytes {
eyre::bail!("legacy inventory key byte limit exceeded");
}
metadata_bytes = metadata_bytes
.checked_add(object.key.len())
.and_then(|total| total.checked_add(core::mem::size_of::<LegacyInventoryObject>()))
.ok_or_else(|| eyre::eyre!("legacy inventory metadata byte count overflow"))?;
if metadata_bytes > limits.max_metadata_bytes {
eyre::bail!("legacy inventory metadata byte limit exceeded");
}
let relative_key = object.key.strip_prefix(namespace);
if object.key.bytes().any(|byte| byte.is_ascii_control())
|| object.key.contains('\\')
|| relative_key.is_none()
|| relative_key.is_some_and(|relative| {
relative.is_empty()
|| relative
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
})
{
eyre::bail!("legacy inventory listing contains a malformed or out-of-namespace key");
}
if listed.insert(object.key, object.size).is_some() {
eyre::bail!("legacy inventory listing contains a duplicate key");
}
}
let mut canonical_chunks = std::collections::BTreeMap::new();
let mut preserved_objects = Vec::new();
for (key, size) in listed {
if let Some(hash) = hash_from_canonical_chunk_key(&key, namespace) {
canonical_chunks.insert(hash, (key, size));
} else {
preserved_objects.push(LegacyInventoryObject { key, size });
}
}
for root in core::iter::once(&mut current).chain(retained.iter_mut()) {
root.accounting.unique_chunk_bytes =
root.references.values().try_fold(0_u64, |total, length| {
checked_inventory_add(total, u64::from(*length))
})?;
}
let mut unlisted_references = Vec::new();
for (hash, expected_length) in &union_references {
match canonical_chunks.get(hash) {
Some((_, observed_size)) if *observed_size == u64::from(*expected_length) => {}
Some(_) => eyre::bail!(
"listed legacy chunk size differs from its expected whole-segment length"
),
None if listing.complete => {
eyre::bail!("complete legacy listing omits a referenced segment")
}
None => unlisted_references.push(*hash),
}
}
let mut current_only_chunk_bytes = 0_u64;
let mut shared_chunk_bytes = 0_u64;
let mut retained_only_chunk_bytes = 0_u64;
for (hash, expected_length) in &union_references {
let bytes = u64::from(*expected_length);
if current.references.contains_key(hash) {
if retained_union.contains(hash) {
shared_chunk_bytes = checked_inventory_add(shared_chunk_bytes, bytes)?;
} else {
current_only_chunk_bytes = checked_inventory_add(current_only_chunk_bytes, bytes)?;
}
} else {
retained_only_chunk_bytes = checked_inventory_add(retained_only_chunk_bytes, bytes)?;
}
}
let unreferenced_chunk_bytes = if listing.complete {
Some(
canonical_chunks
.iter()
.filter(|(hash, _)| !union_references.contains_key(*hash))
.try_fold(0_u64, |total, (_, (_, size))| {
checked_inventory_add(total, *size)
})?,
)
} else {
None
};
Ok(LegacyInventory {
listing_complete: listing.complete,
current_root: current.accounting,
retained_roots: retained.into_iter().map(|root| root.accounting).collect(),
current_only_chunk_bytes,
shared_chunk_bytes,
retained_only_chunk_bytes,
unreferenced_chunk_bytes,
unlisted_references,
preserved_objects,
})
}
#[derive(Clone)]
pub struct LegacyS3Config {
pub bucket_name: String,
pub endpoint: String,
pub access_key: String,
pub secret_key: String,
pub region: Option<String>,
pub prefix: Option<String>,
}
impl core::fmt::Debug for LegacyS3Config {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("LegacyS3Config").finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct SegmentExtent {
file_offset: u64,
length: u32,
segment_offset: u32,
segment_length: u32,
hash: [u8; 32],
}
impl SegmentExtent {
fn file_end(&self) -> eyre::Result<u64> {
self.file_offset
.checked_add(u64::from(self.length))
.ok_or_else(|| eyre::eyre!("S3 manifest extent offset overflow"))
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct ManifestFile {
path: String,
length: u64,
extents: Vec<SegmentExtent>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
struct TableManifest {
files: Vec<ManifestFile>,
}
impl TableManifest {
fn encode(&self) -> eyre::Result<Vec<u8>> {
let file_count = u32::try_from(self.files.len())
.map_err(|_| eyre::eyre!("too many S3 manifest files"))?;
if self.files.len() > MAX_MANIFEST_FILES {
return Err(eyre::eyre!("too many S3 manifest files"));
}
let total_extents = self.files.iter().try_fold(0_usize, |total, file| {
total
.checked_add(file.extents.len())
.ok_or_else(|| eyre::eyre!("S3 manifest extent count overflow"))
})?;
if total_extents > MAX_MANIFEST_EXTENTS {
return Err(eyre::eyre!("too many extents in S3 manifest"));
}
let mut bytes = Vec::new();
bytes.extend_from_slice(MANIFEST_MAGIC_V2);
bytes.extend_from_slice(&file_count.to_le_bytes());
for file in &self.files {
validate_manifest_file(file)?;
validate_relative_path(&file.path)?;
let path = file.path.as_bytes();
let path_len = u16::try_from(path.len())
.map_err(|_| eyre::eyre!("S3 manifest path is too long"))?;
let extent_count = u32::try_from(file.extents.len())
.map_err(|_| eyre::eyre!("too many extents in S3 manifest"))?;
bytes.extend_from_slice(&path_len.to_le_bytes());
bytes.extend_from_slice(path);
bytes.extend_from_slice(&file.length.to_le_bytes());
bytes.extend_from_slice(&extent_count.to_le_bytes());
for extent in &file.extents {
bytes.extend_from_slice(&extent.file_offset.to_le_bytes());
bytes.extend_from_slice(&extent.length.to_le_bytes());
bytes.extend_from_slice(&extent.segment_offset.to_le_bytes());
bytes.extend_from_slice(&extent.segment_length.to_le_bytes());
bytes.extend_from_slice(&extent.hash);
}
}
let checksum = blake3::hash(&bytes);
bytes.extend_from_slice(checksum.as_bytes());
Ok(bytes)
}
fn decode(bytes: &[u8]) -> eyre::Result<Self> {
if bytes.len() < MANIFEST_MAGIC_V2.len() + 4 + 32 {
return Err(eyre::eyre!("S3 manifest is truncated"));
}
let (payload, checksum) = bytes.split_at(bytes.len() - 32);
if blake3::hash(payload).as_bytes() != checksum {
return Err(eyre::eyre!("S3 manifest checksum mismatch"));
}
let magic = payload
.get(..MANIFEST_MAGIC_V2.len())
.ok_or_else(|| eyre::eyre!("S3 manifest is truncated"))?;
if magic == MANIFEST_MAGIC_V1 {
return Self::decode_v1(payload);
}
if magic != MANIFEST_MAGIC_V2 {
return Err(eyre::eyre!("unsupported S3 manifest format"));
}
let mut reader = ManifestReader::new(payload);
reader.take(MANIFEST_MAGIC_V2.len())?;
let file_count = reader.u32()? as usize;
if file_count > MAX_MANIFEST_FILES {
return Err(eyre::eyre!("S3 manifest contains too many files"));
}
let mut files = Vec::with_capacity(file_count);
let mut previous_path: Option<String> = None;
let mut total_extents = 0_usize;
for _ in 0..file_count {
let path_len = reader.u16()? as usize;
if path_len == 0 {
return Err(eyre::eyre!("S3 manifest contains an empty path"));
}
let path = core::str::from_utf8(reader.take(path_len)?)?.to_string();
validate_relative_path(&path)?;
if previous_path
.as_ref()
.is_some_and(|previous| previous >= &path)
{
return Err(eyre::eyre!(
"S3 manifest file paths are not strictly sorted"
));
}
previous_path = Some(path.clone());
let length = reader.u64()?;
let extent_count = reader.u32()? as usize;
total_extents = total_extents
.checked_add(extent_count)
.ok_or_else(|| eyre::eyre!("S3 manifest extent count overflow"))?;
if total_extents > MAX_MANIFEST_EXTENTS {
return Err(eyre::eyre!("S3 manifest contains too many extents"));
}
let mut extents = Vec::with_capacity(extent_count);
for _ in 0..extent_count {
let file_offset = reader.u64()?;
let extent_length = reader.u32()?;
let segment_offset = reader.u32()?;
let segment_length = reader.u32()?;
let mut hash = [0_u8; 32];
let hash_length = hash.len();
hash.copy_from_slice(reader.take(hash_length)?);
extents.push(SegmentExtent {
file_offset,
length: extent_length,
segment_offset,
segment_length,
hash,
});
}
let file = ManifestFile {
path,
length,
extents,
};
validate_manifest_file(&file)?;
files.push(file);
}
if !reader.is_empty() {
return Err(eyre::eyre!("S3 manifest has trailing data"));
}
Ok(Self { files })
}
fn decode_v1(payload: &[u8]) -> eyre::Result<Self> {
let mut reader = ManifestReader::new(payload);
if reader.take(MANIFEST_MAGIC_V1.len())? != MANIFEST_MAGIC_V1 {
return Err(eyre::eyre!("unsupported S3 manifest format"));
}
let file_count = reader.u32()? as usize;
if file_count > MAX_MANIFEST_FILES {
return Err(eyre::eyre!("S3 manifest contains too many files"));
}
let mut files = Vec::with_capacity(file_count);
let mut previous_path: Option<String> = None;
let mut total_chunks = 0_usize;
for _ in 0..file_count {
let path_len = reader.u16()? as usize;
if path_len == 0 {
return Err(eyre::eyre!("S3 manifest contains an empty path"));
}
let path = core::str::from_utf8(reader.take(path_len)?)?.to_string();
validate_relative_path(&path)?;
if previous_path
.as_ref()
.is_some_and(|previous| previous >= &path)
{
return Err(eyre::eyre!(
"S3 manifest file paths are not strictly sorted"
));
}
previous_path = Some(path.clone());
let length = reader.u64()?;
let chunk_count = reader.u32()? as usize;
total_chunks = total_chunks
.checked_add(chunk_count)
.ok_or_else(|| eyre::eyre!("S3 manifest chunk count overflow"))?;
if total_chunks > MAX_MANIFEST_EXTENTS {
return Err(eyre::eyre!("S3 manifest contains too many chunks"));
}
let expected_chunks = if length == 0 {
0
} else {
usize::try_from(length.div_ceil(SEGMENT_TARGET as u64))?
};
if chunk_count != expected_chunks {
return Err(eyre::eyre!(
"S3 manifest chunk count does not match file length"
));
}
let mut extents = Vec::with_capacity(chunk_count);
let mut file_offset = 0_u64;
for index in 0..chunk_count {
let chunk_length = reader.u32()?;
if chunk_length == 0 || chunk_length as usize > SEGMENT_TARGET {
return Err(eyre::eyre!("S3 manifest contains an invalid chunk length"));
}
if index + 1 != chunk_count && chunk_length as usize != SEGMENT_TARGET {
return Err(eyre::eyre!("S3 manifest contains a short interior chunk"));
}
let mut hash = [0_u8; 32];
let hash_length = hash.len();
hash.copy_from_slice(reader.take(hash_length)?);
extents.push(SegmentExtent {
file_offset,
length: chunk_length,
segment_offset: 0,
segment_length: chunk_length,
hash,
});
file_offset = file_offset
.checked_add(u64::from(chunk_length))
.ok_or_else(|| eyre::eyre!("S3 manifest file length overflow"))?;
}
let file = ManifestFile {
path,
length,
extents,
};
validate_manifest_file(&file)?;
files.push(file);
}
if !reader.is_empty() {
return Err(eyre::eyre!("S3 manifest has trailing data"));
}
Ok(Self { files })
}
fn committed_segments(&self) -> HashSet<[u8; 32]> {
self.files
.iter()
.flat_map(|file| file.extents.iter().map(|extent| extent.hash))
.collect()
}
fn file(&self, path: &str) -> Option<&ManifestFile> {
self.files
.binary_search_by(|file| file.path.as_str().cmp(path))
.ok()
.map(|index| &self.files[index])
}
}
fn validate_manifest_file(file: &ManifestFile) -> eyre::Result<()> {
let mut described_length = 0_u64;
for extent in &file.extents {
if extent.file_offset != described_length {
return Err(eyre::eyre!(
"S3 manifest extents do not cover the file contiguously"
));
}
if extent.length == 0
|| extent.segment_length == 0
|| extent.segment_length as usize > SEGMENT_TARGET
{
return Err(eyre::eyre!("S3 manifest contains an invalid extent length"));
}
let segment_end = extent
.segment_offset
.checked_add(extent.length)
.ok_or_else(|| eyre::eyre!("S3 manifest segment offset overflow"))?;
if segment_end > extent.segment_length {
return Err(eyre::eyre!("S3 manifest extent exceeds its segment"));
}
described_length = extent.file_end()?;
}
if described_length != file.length {
return Err(eyre::eyre!(
"S3 manifest extents do not cover the file length"
));
}
Ok(())
}
struct ManifestReader<'a> {
bytes: &'a [u8],
position: usize,
}
impl<'a> ManifestReader<'a> {
fn new(bytes: &'a [u8]) -> Self {
Self { bytes, position: 0 }
}
fn take(&mut self, length: usize) -> eyre::Result<&'a [u8]> {
let end = self
.position
.checked_add(length)
.ok_or_else(|| eyre::eyre!("S3 manifest offset overflow"))?;
let value = self
.bytes
.get(self.position..end)
.ok_or_else(|| eyre::eyre!("S3 manifest is truncated"))?;
self.position = end;
Ok(value)
}
fn u16(&mut self) -> eyre::Result<u16> {
let mut bytes = [0_u8; 2];
let length = bytes.len();
bytes.copy_from_slice(self.take(length)?);
Ok(u16::from_le_bytes(bytes))
}
fn u32(&mut self) -> eyre::Result<u32> {
let mut bytes = [0_u8; 4];
let length = bytes.len();
bytes.copy_from_slice(self.take(length)?);
Ok(u32::from_le_bytes(bytes))
}
fn u64(&mut self) -> eyre::Result<u64> {
let mut bytes = [0_u8; 8];
let length = bytes.len();
bytes.copy_from_slice(self.take(length)?);
Ok(u64::from_le_bytes(bytes))
}
fn is_empty(&self) -> bool {
self.position == self.bytes.len()
}
}
#[derive(Clone, Debug)]
pub struct LegacySnapshotConfig {
pub table_path: String,
pub s3: LegacyS3Config,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MigrationCaptureLimits {
pub max_objects: usize,
pub max_files: usize,
pub max_total_bytes: u64,
pub max_key_bytes: usize,
}
impl Default for MigrationCaptureLimits {
fn default() -> Self {
Self {
max_objects: 100_000,
max_files: 16_384,
max_total_bytes: 64 * 1024 * 1024 * 1024,
max_key_bytes: 4_096,
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MigrationCaptureProvenance {
pub namespace: String,
pub files: Vec<(String, u64)>,
pub total_bytes: u64,
}
pub enum MigrationCaptureOutcome {
Manifest {
snapshot: Box<RestoredLegacySnapshot>,
provenance: MigrationCaptureProvenance,
},
LegacyFiles {
provenance: MigrationCaptureProvenance,
},
Absent {
provenance: MigrationCaptureProvenance,
},
}
type RestoredContent = HashMap<String, (u64, Vec<[u8; 32]>)>;
#[derive(Clone, Copy, Debug)]
pub struct ExclusiveFencedLegacyDestination(());
impl ExclusiveFencedLegacyDestination {
pub fn acknowledge_external_fence() -> Self {
Self(())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct LegacyPublicationSummary {
pub file_count: usize,
pub unique_chunk_count: usize,
pub manifest_blake3: [u8; 32],
}
type OfflineChunks = std::collections::BTreeMap<[u8; 32], (PathBuf, u64, usize)>;
fn build_offline_snapshot(table_path: &Path) -> eyre::Result<(TableManifest, OfflineChunks)> {
if !std::fs::symlink_metadata(table_path)?.is_dir() {
eyre::bail!("offline table path must be a real directory");
}
let mut files = Vec::new();
let mut chunks = OfflineChunks::new();
let mut entries = 0_usize;
let mut manifest_bytes = 44_usize;
let mut extent_count = 0_usize;
for entry in WalkDir::new(table_path).max_open(16) {
let entry = entry?;
entries += 1;
if entries > 1_000_000 || entry.depth() > 128 {
eyre::bail!("offline inventory traversal limit exceeded");
}
if entry.file_type().is_symlink() {
eyre::bail!("offline inventory contains a symbolic link");
}
if !entry.file_type().is_dir() && !entry.file_type().is_file() {
eyre::bail!("offline inventory contains a nonregular entry");
}
if !entry.file_type().is_file() || !is_table_file(entry.path()) {
continue;
}
if files.len() >= MAX_MANIFEST_FILES {
eyre::bail!("offline inventory file limit exceeded");
}
let path = canonical_relative_path(table_path, entry.path())?;
let length = entry.metadata()?.len();
let count = usize::try_from(length.div_ceil(SEGMENT_TARGET as u64))?;
extent_count = extent_count
.checked_add(count)
.ok_or_else(|| eyre::eyre!("offline extent count overflow"))?;
manifest_bytes = manifest_bytes
.checked_add(path.len())
.and_then(|n| n.checked_add(14))
.and_then(|n| count.checked_mul(52).and_then(|size| n.checked_add(size)))
.ok_or_else(|| eyre::eyre!("offline manifest size overflow"))?;
if extent_count > MAX_MANIFEST_EXTENTS || manifest_bytes > MAX_REQUIRED_MANIFEST_BYTES {
eyre::bail!("offline manifest inventory limit exceeded");
}
let mut extents = Vec::with_capacity(count);
let mut offset = 0;
while offset < length {
let size = usize::try_from((length - offset).min(SEGMENT_TARGET as u64))?;
let bytes = read_file_range(entry.path(), offset, size)?;
let hash = *blake3::hash(&bytes).as_bytes();
if let Some((previous_path, previous_offset, previous_size)) = chunks.get(&hash) {
if *previous_size != size
|| read_file_range(previous_path, *previous_offset, *previous_size)? != bytes
{
eyre::bail!("offline content hash collision or changing source");
}
} else {
chunks.insert(hash, (entry.path().to_path_buf(), offset, size));
}
extents.push(SegmentExtent {
file_offset: offset,
length: u32::try_from(size)?,
segment_offset: 0,
segment_length: u32::try_from(size)?,
hash,
});
offset += size as u64;
}
files.push(ManifestFile {
path,
length,
extents,
});
}
files.sort_by(|a, b| a.path.cmp(&b.path));
if !files.iter().any(|file| file.path == WT_DATA_EXTENSION) {
eyre::bail!("offline inventory must contain root .wt.data");
}
let manifest = TableManifest { files };
manifest.encode()?;
Ok((manifest, chunks))
}
pub struct LegacySnapshotStore {
config: LegacySnapshotConfig,
bucket: Bucket,
credentials: Credentials,
client: Agent,
committed_manifest: Option<TableManifest>,
committed_blocks: HashMap<String, Vec<[u8; 32]>>,
expected_restored_content: Option<RestoredContent>,
}
pub struct RestoredLegacySnapshot {
config: LegacySnapshotConfig,
manifest: TableManifest,
restored_files: Vec<PathBuf>,
restored_content: RestoredContent,
client: HttpAgent,
}
impl RestoredLegacySnapshot {
pub fn table_path(&self) -> &Path {
Path::new(&self.config.table_path)
}
pub fn restored_files(&self) -> &[PathBuf] {
&self.restored_files
}
pub fn into_store_for(
self,
config: &LegacySnapshotConfig,
) -> eyre::Result<LegacySnapshotStore> {
if self.config.table_path != config.table_path
|| self.config.s3.endpoint != config.s3.endpoint
|| self.config.s3.bucket_name != config.s3.bucket_name
|| self.config.s3.prefix != config.s3.prefix
|| self.config.s3.region != config.s3.region
{
eyre::bail!("restored snapshot configuration identity mismatch");
}
if describe_table_content(Path::new(&config.table_path))? != self.restored_content {
eyre::bail!("restored snapshot directory no longer matches its token");
}
let (bucket, credentials, client) =
LegacySnapshotStore::create_bucket(&config.s3, self.client)?;
Ok(LegacySnapshotStore {
config: config.clone(),
bucket,
credentials,
client,
committed_manifest: Some(self.manifest),
committed_blocks: HashMap::new(),
expected_restored_content: Some(self.restored_content),
})
}
}
impl core::fmt::Debug for LegacySnapshotStore {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("LegacySnapshotStore")
.finish_non_exhaustive()
}
}
impl LegacySnapshotStore {
pub fn prepare_publication(
table_path: impl AsRef<Path>,
) -> eyre::Result<LegacyPublicationSummary> {
let (manifest, chunks) = build_offline_snapshot(table_path.as_ref())?;
let manifest_bytes = manifest.encode()?;
Ok(LegacyPublicationSummary {
file_count: manifest.files.len(),
unique_chunk_count: chunks.len(),
manifest_blake3: *blake3::hash(&manifest_bytes).as_bytes(),
})
}
pub async fn capture_for_migration(
config: LegacySnapshotConfig,
limits: MigrationCaptureLimits,
) -> eyre::Result<MigrationCaptureOutcome> {
Self::capture_for_migration_with_agent(
config,
limits,
HttpAgentBuilder::new()
.timeout(Duration::from_secs(30))
.build(),
)
.await
}
pub async fn capture_for_migration_with_agent(
config: LegacySnapshotConfig,
limits: MigrationCaptureLimits,
client: HttpAgent,
) -> eyre::Result<MigrationCaptureOutcome> {
validate_capture_target(Path::new(&config.table_path))?;
if limits.max_objects == 0 || limits.max_files == 0 || limits.max_key_bytes == 0 {
eyre::bail!("migration capture limits must be nonzero");
}
let (bucket, credentials, client) = Self::create_bucket(&config.s3, client)?;
let table_name = Self::table_name(&config)?;
let prefix = config.s3.prefix.as_deref().unwrap_or("");
let namespace = Self::full_s3_path(prefix, "", table_name);
let manifest_key = Self::full_s3_path(prefix, MANIFEST_FILE, table_name);
if let Some(bytes) = Self::get_object_optional_bounded(
&bucket,
&credentials,
&client,
&manifest_key,
MAX_REQUIRED_MANIFEST_BYTES,
)? {
let manifest = TableManifest::decode(&bytes)?;
if !manifest
.files
.iter()
.any(|file| file.path == WT_DATA_EXTENSION)
{
eyre::bail!("migration manifest has no primary .wt.data file");
}
if manifest.files.len() > limits.max_files {
eyre::bail!("migration manifest file limit exceeded");
}
let total_bytes = manifest.files.iter().try_fold(0_u64, |total, file| {
total
.checked_add(file.length)
.ok_or_else(|| eyre::eyre!("migration capture byte count overflow"))
})?;
if total_bytes > limits.max_total_bytes {
eyre::bail!("migration capture byte limit exceeded");
}
let mut file_inventory = manifest
.files
.iter()
.map(|file| (file.path.clone(), file.length))
.collect::<Vec<_>>();
file_inventory.sort_by(|left, right| left.0.cmp(&right.0));
Self::restore_manifest(
&bucket,
&credentials,
&client,
&config,
&manifest,
Some(SEGMENT_TARGET),
)
.await?;
let restored_content = describe_table_content(Path::new(&config.table_path))?;
if restored_content.len() != manifest.files.len()
|| manifest.files.iter().any(|file| {
restored_content
.get(&file.path)
.is_none_or(|(length, _)| *length != file.length)
})
{
eyre::bail!("captured manifest files differ from the committed inventory");
}
let restored_files = manifest
.files
.iter()
.map(|file| Path::new(&config.table_path).join(&file.path))
.collect();
return Ok(MigrationCaptureOutcome::Manifest {
snapshot: Box::new(RestoredLegacySnapshot {
config,
manifest,
restored_files,
restored_content,
client,
}),
provenance: MigrationCaptureProvenance {
namespace,
files: file_inventory,
total_bytes,
},
});
}
let (objects, complete) =
Self::list_capture_namespace(&bucket, &credentials, &client, &namespace, &limits)?;
if !complete {
eyre::bail!("migration namespace listing was incomplete");
}
if objects.is_empty() {
return Ok(MigrationCaptureOutcome::Absent {
provenance: MigrationCaptureProvenance {
namespace,
files: Vec::new(),
total_bytes: 0,
},
});
}
let mut raw = Vec::new();
let mut total_bytes = 0_u64;
for object in objects {
let relative = object
.key
.strip_prefix(&namespace)
.ok_or_else(|| eyre::eyre!("migration listing returned an out-of-namespace key"))?;
validate_relative_path(relative)?;
if !is_table_name(relative) {
eyre::bail!(
"legacy namespace contains a non-table object; capture is partial or unknown"
);
}
if relative.len() > limits.max_key_bytes {
eyre::bail!("migration capture key byte limit exceeded");
}
if raw.len() >= limits.max_files {
eyre::bail!("migration capture file limit exceeded");
}
total_bytes = total_bytes
.checked_add(object.size)
.ok_or_else(|| eyre::eyre!("migration capture byte count overflow"))?;
if total_bytes > limits.max_total_bytes {
eyre::bail!("migration capture byte limit exceeded");
}
raw.push((relative.to_string(), object.size, object.key));
}
if !raw.iter().any(|(path, _, _)| path == WT_DATA_EXTENSION) {
eyre::bail!("legacy namespace has no primary .wt.data file");
}
raw.sort_by(|left, right| left.0.cmp(&right.0));
let stage = staging_path(Path::new(&config.table_path), "migration-legacy")?;
if let Some(parent) = stage.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::create_dir(&stage)?;
let captured = (|| -> eyre::Result<()> {
for (relative, size, key) in &raw {
let destination = stage.join(relative);
if let Some(parent) = destination.parent() {
std::fs::create_dir_all(parent)?;
}
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(destination)?;
let action = bucket.get_object(Some(&credentials), key);
let response = client
.get(action.sign(Duration::from_secs(3600)).as_str())
.call()
.map_err(|error| safe_s3_request_error("read", error))?;
let mut reader = response.into_reader();
let copied = std::io::copy(&mut reader.by_ref().take(*size), &mut file)
.map_err(safe_s3_read_error)?;
let mut extra = [0_u8; 1];
if copied != *size || reader.read(&mut extra).map_err(safe_s3_read_error)? != 0 {
eyre::bail!("legacy object size changed after namespace listing");
}
file.sync_all()?;
}
validate_capture_target(Path::new(&config.table_path))?;
publish_staged_table(Path::new(&config.table_path), &stage)
})();
if let Err(error) = captured {
if let Err(cleanup) = std::fs::remove_dir_all(&stage) {
return Err(eyre::eyre!(
"{error}; failed to clean migration stage: {cleanup}"
));
}
return Err(error);
}
let files = raw
.into_iter()
.map(|(path, size, _)| (path, size))
.collect();
Ok(MigrationCaptureOutcome::LegacyFiles {
provenance: MigrationCaptureProvenance {
namespace,
files,
total_bytes,
},
})
}
fn list_capture_namespace(
bucket: &Bucket,
credentials: &Credentials,
client: &Agent,
namespace: &str,
limits: &MigrationCaptureLimits,
) -> eyre::Result<(Vec<LegacyInventoryObject>, bool)> {
let mut continuation: Option<String> = None;
let mut objects = Vec::new();
let mut seen_tokens = HashSet::new();
let mut seen_keys = HashSet::new();
let mut key_bytes = 0_usize;
let mut token_bytes = 0_usize;
for _ in 0..MAX_CAPTURE_LISTING_PAGES {
let mut action = bucket.list_objects_v2(Some(credentials));
action.with_prefix(namespace);
if let Some(token) = continuation.as_deref() {
action.with_continuation_token(token);
}
let url = action.sign(Duration::from_secs(3600));
let response = client
.get(url.as_str())
.call()
.map_err(|error| safe_s3_request_error("listing", error))?;
let mut bytes = Vec::new();
response
.into_reader()
.take((MAX_CAPTURE_LISTING_BYTES + 1) as u64)
.read_to_end(&mut bytes)
.map_err(safe_s3_read_error)?;
let parsed = parse_capture_listing(&bytes)?;
for object in parsed.contents {
if objects.len() >= limits.max_objects {
eyre::bail!("migration capture object limit exceeded");
}
let max_full_key = limits
.max_key_bytes
.checked_add(namespace.len())
.ok_or_else(|| eyre::eyre!("migration capture key limit overflow"))?;
if object.key.len() > max_full_key {
eyre::bail!("migration capture key byte limit exceeded");
}
key_bytes = key_bytes
.checked_add(object.key.len())
.ok_or_else(|| eyre::eyre!("migration listing key byte count overflow"))?;
if key_bytes > MAX_CAPTURE_KEY_BYTES {
eyre::bail!("migration listing aggregate key byte limit exceeded");
}
if !seen_keys.insert(object.key.clone()) {
eyre::bail!("migration listing contains a duplicate key");
}
objects.push(LegacyInventoryObject {
key: object.key,
size: object.size,
});
}
match parsed.next_continuation_token {
Some(token) if !seen_tokens.insert(token.clone()) => {
eyre::bail!("repeated S3 continuation token")
}
Some(token) => {
token_bytes = token_bytes.checked_add(token.len()).ok_or_else(|| {
eyre::eyre!("migration listing token byte count overflow")
})?;
if token_bytes > MAX_CAPTURE_KEY_BYTES {
eyre::bail!("migration listing aggregate token byte limit exceeded");
}
continuation = Some(token);
}
None => return Ok((objects, true)),
}
}
eyre::bail!("migration listing page limit exceeded")
}
pub async fn open(config: LegacySnapshotConfig) -> eyre::Result<Self> {
Self::open_with_agent(
config,
ureq::AgentBuilder::new()
.timeout(Duration::from_secs(30))
.build(),
)
.await
}
pub async fn open_with_agent(
config: LegacySnapshotConfig,
client: HttpAgent,
) -> eyre::Result<Self> {
let (bucket, credentials, client) = Self::create_bucket(&config.s3, client)?;
let committed_manifest =
Self::sync_from_s3(&bucket, &credentials, &client, &config).await?;
Ok(Self {
config,
bucket,
credentials,
client,
committed_manifest,
committed_blocks: HashMap::new(),
expected_restored_content: None,
})
}
pub async fn restore_required(
config: LegacySnapshotConfig,
) -> eyre::Result<RestoredLegacySnapshot> {
Self::restore_required_with_agent(
config,
HttpAgentBuilder::new()
.timeout(Duration::from_secs(30))
.build(),
)
.await
}
pub async fn restore_required_with_agent(
config: LegacySnapshotConfig,
client: HttpAgent,
) -> eyre::Result<RestoredLegacySnapshot> {
let (bucket, credentials, client) = Self::create_bucket(&config.s3, client)?;
let table_name = Self::table_name(&config)?;
let prefix = config.s3.prefix.as_deref().unwrap_or("");
let manifest_key = Self::full_s3_path(prefix, MANIFEST_FILE, table_name);
let bytes = Self::get_object_optional_bounded(
&bucket,
&credentials,
&client,
&manifest_key,
MAX_REQUIRED_MANIFEST_BYTES,
)?
.ok_or_else(|| eyre::eyre!("required S3 snapshot manifest is missing"))?;
let manifest = TableManifest::decode(&bytes)?;
if !manifest
.files
.iter()
.any(|file| file.path == WT_DATA_EXTENSION)
{
eyre::bail!("required S3 snapshot manifest contains no .wt.data file");
}
Self::restore_manifest(
&bucket,
&credentials,
&client,
&config,
&manifest,
Some(SEGMENT_TARGET),
)
.await?;
let restored_files = manifest
.files
.iter()
.map(|file| Path::new(&config.table_path).join(&file.path))
.collect();
let restored_content = describe_table_content(Path::new(&config.table_path))?;
if restored_content.len() != manifest.files.len()
|| manifest.files.iter().any(|file| {
restored_content
.get(&file.path)
.is_none_or(|(length, _)| *length != file.length)
})
{
eyre::bail!("restored snapshot files do not match its manifest");
}
Ok(RestoredLegacySnapshot {
config,
manifest,
restored_files,
restored_content,
client,
})
}
pub async fn publish_new_fenced(
config: LegacySnapshotConfig,
destination: ExclusiveFencedLegacyDestination,
) -> eyre::Result<LegacyPublicationSummary> {
Self::publish_new_fenced_with_agent(
config,
destination,
HttpAgentBuilder::new()
.timeout(Duration::from_secs(30))
.build(),
)
.await
}
pub async fn publish_new_fenced_with_agent(
config: LegacySnapshotConfig,
_destination: ExclusiveFencedLegacyDestination,
client: HttpAgent,
) -> eyre::Result<LegacyPublicationSummary> {
let table_path = Path::new(&config.table_path);
let (manifest, chunks) = build_offline_snapshot(table_path)?;
let manifest_bytes = manifest.encode()?;
let (bucket, credentials, client) = Self::create_bucket(&config.s3, client)?;
let table_name = Self::table_name(&config)?;
let prefix = config.s3.prefix.as_deref().unwrap_or("");
let key_for = |path: &str| Self::full_s3_path(prefix, path, table_name);
let get = |key: &str, limit| {
Self::get_object_optional_bounded(&bucket, &credentials, &client, key, limit)
};
let put = |key: &str, bytes: &[u8]| -> eyre::Result<()> {
let action = bucket.put_object(Some(&credentials), key);
let url = action.sign(Duration::from_secs(3600));
client
.put(url.as_str())
.send_bytes(bytes)
.map_err(|error| safe_s3_request_error("write", error))?;
Ok(())
};
let manifest_key = key_for(MANIFEST_FILE);
let existing_manifest = get(&manifest_key, MAX_REQUIRED_MANIFEST_BYTES)?;
if existing_manifest
.as_deref()
.is_some_and(|bytes| bytes != manifest_bytes.as_slice())
{
eyre::bail!("destination has a divergent manifest");
}
for (hash, (path, offset, size)) in &chunks {
let bytes = read_file_range(path, *offset, *size)?;
if blake3::hash(&bytes).as_bytes() != hash {
eyre::bail!("offline source changed during publication");
}
let key = key_for(&Self::chunk_path(hash));
match get(&key, *size)? {
Some(stored) if stored != bytes => eyre::bail!("destination has a divergent chunk"),
Some(_) => {}
None => put(&key, &bytes)?,
}
if get(&key, *size)?.as_deref() != Some(bytes.as_slice()) {
eyre::bail!("chunk exact readback failed before manifest publication");
}
}
if build_offline_snapshot(table_path)?.0 != manifest {
eyre::bail!("offline inventory changed before manifest publication");
}
if existing_manifest.is_none() {
put(&manifest_key, &manifest_bytes)?;
}
for (hash, (path, offset, size)) in &chunks {
let expected = read_file_range(path, *offset, *size)?;
if blake3::hash(&expected).as_bytes() != hash
|| get(&key_for(&Self::chunk_path(hash)), *size)?.as_deref()
!= Some(expected.as_slice())
{
eyre::bail!("published chunk exact readback failed");
}
}
if get(&manifest_key, MAX_REQUIRED_MANIFEST_BYTES)?.as_deref()
!= Some(manifest_bytes.as_slice())
{
eyre::bail!("published manifest exact readback failed");
}
Ok(LegacyPublicationSummary {
file_count: manifest.files.len(),
unique_chunk_count: chunks.len(),
manifest_blake3: *blake3::hash(&manifest_bytes).as_bytes(),
})
}
pub fn capture_baseline(&mut self) -> eyre::Result<()> {
let table_path = Path::new(&self.config.table_path);
let actual = describe_table_content(table_path)?;
if let Some(expected) = &self.expected_restored_content {
if &actual != expected {
eyre::bail!("restored snapshot directory no longer matches its token");
}
}
self.committed_blocks = actual
.into_iter()
.map(|(path, (_, hashes))| (path, hashes))
.collect();
self.expected_restored_content = None;
Ok(())
}
fn create_bucket(
config: &LegacyS3Config,
client: Agent,
) -> eyre::Result<(Bucket, Credentials, Agent)> {
let credentials = Credentials::new(&config.access_key, &config.secret_key);
let endpoint: Url = config.endpoint.parse()?;
let region = config.region.clone().unwrap_or_else(|| "auto".to_string());
let bucket = Bucket::new(endpoint, UrlStyle::Path, config.bucket_name.clone(), region)?;
Ok((bucket, credentials, client))
}
fn table_name(config: &LegacySnapshotConfig) -> eyre::Result<&str> {
Path::new(&config.table_path)
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| eyre::eyre!("invalid table path"))
}
fn full_s3_path(prefix: &str, s3_path: &str, table_name: &str) -> String {
let prefix = prefix.trim_end_matches('/');
let path = s3_path.trim_start_matches('/');
if prefix.is_empty() {
format!("{table_name}/{path}")
} else {
format!("{prefix}/{table_name}/{path}")
}
}
fn object_key(&self, path: &str) -> eyre::Result<String> {
Ok(Self::full_s3_path(
self.config.s3.prefix.as_deref().unwrap_or(""),
path,
Self::table_name(&self.config)?,
))
}
fn chunk_path(hash: &[u8; 32]) -> String {
let mut hex = String::with_capacity(64);
for byte in hash {
write!(&mut hex, "{byte:02x}").expect("writing to String cannot fail");
}
format!("chunks/{hex}")
}
fn get_object_optional(
bucket: &Bucket,
credentials: &Credentials,
client: &Agent,
key: &str,
) -> eyre::Result<Option<Vec<u8>>> {
let action = bucket.get_object(Some(credentials), key);
let url = action.sign(Duration::from_secs(3600));
let Some(response) = optional_s3_response(client.get(url.as_str()).call())? else {
return Ok(None);
};
let mut bytes = Vec::new();
response
.into_reader()
.read_to_end(&mut bytes)
.map_err(safe_s3_read_error)?;
Ok(Some(bytes))
}
fn get_object_optional_bounded(
bucket: &Bucket,
credentials: &Credentials,
client: &Agent,
key: &str,
max_bytes: usize,
) -> eyre::Result<Option<Vec<u8>>> {
let action = bucket.get_object(Some(credentials), key);
let url = action.sign(Duration::from_secs(3600));
let Some(response) = optional_s3_response(client.get(url.as_str()).call())? else {
return Ok(None);
};
let read_limit = u64::try_from(max_bytes)?
.checked_add(1)
.ok_or_else(|| eyre::eyre!("S3 object byte limit overflow"))?;
let mut bytes = Vec::new();
response
.into_reader()
.take(read_limit)
.read_to_end(&mut bytes)
.map_err(safe_s3_read_error)?;
if bytes.len() > max_bytes {
eyre::bail!("S3 object exceeds its configured byte limit");
}
Ok(Some(bytes))
}
fn put_object_verified(&self, key: &str, bytes: &[u8]) -> eyre::Result<()> {
let action = self.bucket.put_object(Some(&self.credentials), key);
let url = action.sign(Duration::from_secs(3600));
match self.client.put(url.as_str()).send_bytes(bytes) {
Ok(_) => Ok(()),
Err(put_error) => {
let stored =
Self::get_object_optional(&self.bucket, &self.credentials, &self.client, key)?;
if stored.as_deref() == Some(bytes) {
Ok(())
} else {
Err(safe_s3_request_error("write", put_error))
}
}
}
}
pub async fn sync(&mut self) -> eyre::Result<()> {
let table_path = Path::new(&self.config.table_path);
if !table_path.exists() {
return Ok(());
}
let mut local_files = WalkDir::new(table_path)
.into_iter()
.collect::<Result<Vec<_>, _>>()?
.into_iter()
.filter(|entry| entry.file_type().is_file())
.filter(|entry| is_table_file(entry.path()))
.map(|entry| {
let path = entry.path().to_path_buf();
let relative = canonical_relative_path(table_path, &path)?;
Ok((relative, path))
})
.collect::<eyre::Result<Vec<_>>>()?;
local_files.sort_by(|left, right| left.0.cmp(&right.0));
let committed_segments = self
.committed_manifest
.as_ref()
.map_or_else(HashSet::new, TableManifest::committed_segments);
let mut uploaded_segments = HashSet::new();
let mut files = Vec::with_capacity(local_files.len());
let mut next_blocks = HashMap::with_capacity(local_files.len());
for (relative, local_path) in local_files {
let blocks = describe_file_blocks(&local_path)?;
let file_length = blocks.last().map_or(0, LocalBlock::end);
let previous_file = self
.committed_manifest
.as_ref()
.and_then(|manifest| manifest.file(&relative));
let previous_hashes = self.committed_blocks.get(&relative);
let mut extents = match previous_file {
Some(file) => trim_extents(&file.extents, file_length)?,
None => Vec::new(),
};
let dirty = blocks
.iter()
.enumerate()
.map(|(index, block)| {
previous_file.is_none()
|| previous_hashes
.and_then(|hashes| hashes.get(index))
.is_none_or(|hash| hash != &block.hash)
})
.collect::<Vec<_>>();
let mut index = 0;
while index < blocks.len() {
if !dirty[index] {
index += 1;
continue;
}
let first = index;
let mut segment_length = blocks[index].length as usize;
index += 1;
while index < blocks.len()
&& dirty[index]
&& segment_length + blocks[index].length as usize <= SEGMENT_TARGET
{
segment_length += blocks[index].length as usize;
index += 1;
}
let file_offset = blocks[first].offset;
let bytes = read_file_range(&local_path, file_offset, segment_length)?;
let hash = *blake3::hash(&bytes).as_bytes();
if !committed_segments.contains(&hash) && uploaded_segments.insert(hash) {
let key = self.object_key(&Self::chunk_path(&hash))?;
self.put_object_verified(&key, &bytes)?;
}
let extent = SegmentExtent {
file_offset,
length: u32::try_from(segment_length)?,
segment_offset: 0,
segment_length: u32::try_from(segment_length)?,
hash,
};
extents = overlay_extent(&extents, extent)?;
}
let file = ManifestFile {
path: relative,
length: file_length,
extents,
};
validate_manifest_file(&file)?;
next_blocks.insert(
file.path.clone(),
blocks.into_iter().map(|block| block.hash).collect(),
);
files.push(file);
}
let manifest = TableManifest { files };
let manifest_bytes = manifest.encode()?;
let manifest_key = self.object_key(MANIFEST_FILE)?;
self.put_object_verified(&manifest_key, &manifest_bytes)?;
self.committed_manifest = Some(manifest);
self.committed_blocks = next_blocks;
tracing::debug!(
new_segments = uploaded_segments.len(),
"S3 table manifest committed"
);
Ok(())
}
async fn sync_from_s3(
bucket: &Bucket,
credentials: &Credentials,
client: &Agent,
config: &LegacySnapshotConfig,
) -> eyre::Result<Option<TableManifest>> {
let table_name = Self::table_name(config)?;
let prefix = config.s3.prefix.as_deref().unwrap_or("");
let manifest_key = Self::full_s3_path(prefix, MANIFEST_FILE, table_name);
if let Some(bytes) = Self::get_object_optional(bucket, credentials, client, &manifest_key)?
{
let manifest = TableManifest::decode(&bytes)?;
Self::restore_manifest(bucket, credentials, client, config, &manifest, None).await?;
tracing::info!(table_name, "S3 table manifest restored");
return Ok(Some(manifest));
}
if Self::restore_legacy_objects(bucket, credentials, client, config).await? {
tracing::info!(
table_name,
"legacy S3 table objects restored; next write will publish a manifest"
);
} else {
tracing::debug!(table_name, "no committed table objects found in S3");
}
Ok(None)
}
async fn restore_manifest(
bucket: &Bucket,
credentials: &Credentials,
client: &Agent,
config: &LegacySnapshotConfig,
manifest: &TableManifest,
max_chunk_bytes: Option<usize>,
) -> eyre::Result<()> {
let table_path = Path::new(&config.table_path);
let stage = staging_path(table_path, "stage")?;
remove_path_if_exists(&stage)?;
std::fs::create_dir_all(&stage)?;
let prefix = config.s3.prefix.as_deref().unwrap_or("");
let table_name = Self::table_name(config)?;
let restore_result = async {
for file in &manifest.files {
let local_path = stage.join(&file.path);
if let Some(parent) = local_path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut restored_file = std::fs::File::create(&local_path)?;
restored_file.set_len(file.length)?;
let mut by_segment: HashMap<[u8; 32], (u32, Vec<&SegmentExtent>)> = HashMap::new();
for extent in &file.extents {
let entry = by_segment
.entry(extent.hash)
.or_insert_with(|| (extent.segment_length, Vec::new()));
if entry.0 != extent.segment_length {
return Err(eyre::eyre!(
"S3 manifest gives one segment conflicting lengths"
));
}
entry.1.push(extent);
}
for (hash, (segment_length, extents)) in by_segment {
let key = Self::full_s3_path(prefix, &Self::chunk_path(&hash), table_name);
if max_chunk_bytes.is_some_and(|limit| segment_length as usize > limit) {
return Err(eyre::eyre!("S3 manifest segment exceeds its byte limit"));
}
let segment = match max_chunk_bytes {
Some(limit) => Self::get_object_optional_bounded(
bucket,
credentials,
client,
&key,
limit,
)?,
None => Self::get_object_optional(bucket, credentials, client, &key)?,
};
let bytes = segment
.ok_or_else(|| eyre::eyre!("S3 manifest references a missing segment"))?;
if bytes.len() != segment_length as usize
|| blake3::hash(&bytes).as_bytes() != &hash
{
return Err(eyre::eyre!(
"S3 segment failed length or hash validation: {key}"
));
}
for extent in extents {
let from = extent.segment_offset as usize;
let to = from
.checked_add(extent.length as usize)
.ok_or_else(|| eyre::eyre!("S3 manifest segment slice overflow"))?;
restored_file.seek(SeekFrom::Start(extent.file_offset))?;
restored_file.write_all(&bytes[from..to])?;
}
}
restored_file.flush()?;
}
Ok::<(), eyre::Report>(())
}
.await;
if let Err(error) = restore_result {
let _ = std::fs::remove_dir_all(&stage);
return Err(error);
}
publish_staged_table(table_path, &stage)
}
async fn restore_legacy_objects(
bucket: &Bucket,
credentials: &Credentials,
client: &Agent,
config: &LegacySnapshotConfig,
) -> eyre::Result<bool> {
use rusty_s3::actions::ListObjectsV2;
let table_path = Path::new(&config.table_path);
let table_name = Self::table_name(config)?;
let prefix = config.s3.prefix.as_deref().unwrap_or("");
let table_root = Self::full_s3_path(prefix, "", table_name);
let mut continuation = None;
let mut objects = Vec::new();
loop {
let mut action = bucket.list_objects_v2(Some(credentials));
action.with_prefix(&table_root);
if let Some(token) = continuation.as_deref() {
action.with_continuation_token(token);
}
let url = action.sign(Duration::from_secs(3600));
let response = client
.get(url.as_str())
.call()
.map_err(|error| safe_s3_request_error("listing", error))?;
let parsed = ListObjectsV2::parse_response(
&response.into_string().map_err(safe_s3_read_error)?,
)?;
for object in parsed.contents {
let Some(relative) = object.key.strip_prefix(&table_root) else {
continue;
};
if !is_table_name(relative) || validate_relative_path(relative).is_err() {
continue;
}
let bytes = Self::get_object_optional(bucket, credentials, client, &object.key)?
.ok_or_else(|| eyre::eyre!("listed S3 object disappeared: {}", object.key))?;
objects.push((relative.to_string(), bytes));
}
continuation = parsed.next_continuation_token;
if continuation.is_none() {
break;
}
}
if objects.is_empty() {
return Ok(false);
}
objects.sort_by(|left, right| left.0.cmp(&right.0));
let stage = staging_path(table_path, "legacy-stage")?;
remove_path_if_exists(&stage)?;
std::fs::create_dir_all(&stage)?;
for (relative, bytes) in objects {
std::fs::write(stage.join(relative), bytes)?;
}
publish_staged_table(table_path, &stage)?;
Ok(true)
}
}
fn is_table_name(name: &str) -> bool {
name.ends_with(WT_DATA_EXTENSION) || name.ends_with(WT_INDEX_EXTENSION)
}
#[derive(Clone, Debug)]
struct LocalBlock {
offset: u64,
length: u32,
hash: [u8; 32],
}
impl LocalBlock {
fn end(&self) -> u64 {
self.offset + u64::from(self.length)
}
}
fn describe_file_blocks(path: &Path) -> eyre::Result<Vec<LocalBlock>> {
let mut file = BufReader::with_capacity(SEGMENT_TARGET, std::fs::File::open(path)?);
let mut buffer = vec![0_u8; CHANGE_BLOCK_SIZE];
let mut blocks = Vec::new();
let mut offset = 0_u64;
loop {
let length = read_buffer(&mut file, &mut buffer)?;
if length == 0 {
break;
}
blocks.push(LocalBlock {
offset,
length: u32::try_from(length)?,
hash: *blake3::hash(&buffer[..length]).as_bytes(),
});
offset = offset
.checked_add(u64::try_from(length)?)
.ok_or_else(|| eyre::eyre!("local table file length overflow"))?;
}
Ok(blocks)
}
fn describe_table_content(table_path: &Path) -> eyre::Result<RestoredContent> {
if !table_path.exists() {
return Ok(HashMap::new());
}
let mut result = HashMap::new();
for entry in WalkDir::new(table_path) {
let entry = entry?;
if !entry.file_type().is_file() || !is_table_file(entry.path()) {
continue;
}
let relative = canonical_relative_path(table_path, entry.path())?;
let length = entry.metadata()?.len();
let hashes = describe_file_blocks(entry.path())?
.into_iter()
.map(|block| block.hash)
.collect();
result.insert(relative, (length, hashes));
}
Ok(result)
}
fn read_file_range(path: &Path, offset: u64, length: usize) -> eyre::Result<Vec<u8>> {
let mut file = std::fs::File::open(path)?;
file.seek(SeekFrom::Start(offset))?;
let mut bytes = vec![0_u8; length];
file.read_exact(&mut bytes)?;
Ok(bytes)
}
fn read_buffer(file: &mut impl std::io::Read, buffer: &mut [u8]) -> std::io::Result<usize> {
let mut length = 0;
while length < buffer.len() {
let read = file.read(&mut buffer[length..])?;
if read == 0 {
break;
}
length += read;
}
Ok(length)
}
fn trim_extents(extents: &[SegmentExtent], length: u64) -> eyre::Result<Vec<SegmentExtent>> {
let mut trimmed = Vec::new();
for extent in extents {
if extent.file_offset >= length {
break;
}
let keep = extent.file_end()?.min(length) - extent.file_offset;
let mut extent = extent.clone();
extent.length = u32::try_from(keep)?;
trimmed.push(extent);
}
Ok(trimmed)
}
fn overlay_extent(
extents: &[SegmentExtent],
replacement: SegmentExtent,
) -> eyre::Result<Vec<SegmentExtent>> {
let start = replacement.file_offset;
let end = replacement.file_end()?;
let mut result = Vec::with_capacity(extents.len() + 2);
let mut inserted = false;
for extent in extents {
let extent_end = extent.file_end()?;
if extent_end <= start {
result.push(extent.clone());
continue;
}
if extent.file_offset >= end {
if !inserted {
result.push(replacement.clone());
inserted = true;
}
result.push(extent.clone());
continue;
}
if extent.file_offset < start {
let mut left = extent.clone();
left.length = u32::try_from(start - extent.file_offset)?;
result.push(left);
}
if !inserted {
result.push(replacement.clone());
inserted = true;
}
if extent_end > end {
let skipped = u32::try_from(end - extent.file_offset)?;
let mut right = extent.clone();
right.file_offset = end;
right.length = u32::try_from(extent_end - end)?;
right.segment_offset = right
.segment_offset
.checked_add(skipped)
.ok_or_else(|| eyre::eyre!("S3 manifest segment offset overflow"))?;
result.push(right);
}
}
if !inserted {
result.push(replacement);
}
merge_adjacent_extents(result)
}
fn merge_adjacent_extents(extents: Vec<SegmentExtent>) -> eyre::Result<Vec<SegmentExtent>> {
let mut merged: Vec<SegmentExtent> = Vec::with_capacity(extents.len());
for extent in extents {
if let Some(previous) = merged.last_mut() {
let contiguous_file = previous.file_end()? == extent.file_offset;
let contiguous_segment = previous
.segment_offset
.checked_add(previous.length)
.is_some_and(|offset| offset == extent.segment_offset);
if contiguous_file
&& contiguous_segment
&& previous.segment_length == extent.segment_length
&& previous.hash == extent.hash
{
previous.length = previous
.length
.checked_add(extent.length)
.ok_or_else(|| eyre::eyre!("S3 manifest extent length overflow"))?;
continue;
}
}
merged.push(extent);
}
Ok(merged)
}
fn is_table_file(path: &Path) -> bool {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(is_table_name)
}
fn validate_relative_path(path: &str) -> eyre::Result<()> {
if path.is_empty()
|| path
.chars()
.any(|character| matches!(character, '\\' | ':' | '\0'))
{
return Err(eyre::eyre!("invalid S3 manifest path"));
}
let parsed = Path::new(path);
if parsed
.components()
.all(|component| matches!(component, Component::Normal(_)))
{
Ok(())
} else {
Err(eyre::eyre!("unsafe S3 manifest path: {path}"))
}
}
fn canonical_relative_path(root: &Path, path: &Path) -> eyre::Result<String> {
let relative = path.strip_prefix(root)?;
let mut value = String::new();
for component in relative.components() {
let Component::Normal(component) = component else {
return Err(eyre::eyre!("unsafe local table path: {}", path.display()));
};
let component = component
.to_str()
.ok_or_else(|| eyre::eyre!("table path is not valid UTF-8: {}", path.display()))?;
if !value.is_empty() {
value.push('/');
}
value.push_str(component);
}
validate_relative_path(&value)?;
Ok(value)
}
fn staging_path(table_path: &Path, label: &str) -> eyre::Result<PathBuf> {
let parent = table_path
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
let name = table_path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| eyre::eyre!("invalid table path"))?;
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
Ok(parent.join(format!(
".{name}.s3-{label}-{}-{timestamp}",
std::process::id()
)))
}
fn validate_capture_target(table_path: &Path) -> eyre::Result<()> {
let mut current = PathBuf::new();
for component in table_path.components() {
match component {
Component::RootDir => current.push(component.as_os_str()),
Component::CurDir => {}
Component::Normal(part) => current.push(part),
Component::ParentDir => eyre::bail!("migration capture path contains traversal"),
Component::Prefix(_) => {
eyre::bail!("migration capture path contains a platform prefix")
}
}
match std::fs::symlink_metadata(¤t) {
Ok(metadata) if metadata.file_type().is_symlink() => {
eyre::bail!("migration capture path contains a symbolic link")
}
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
}
match std::fs::symlink_metadata(table_path) {
Ok(_) => eyre::bail!("migration capture destination must not already exist"),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
}
fn remove_path_if_exists(path: &Path) -> eyre::Result<()> {
match std::fs::symlink_metadata(path) {
Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path)?,
Ok(_) => std::fs::remove_file(path)?,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(error.into()),
}
Ok(())
}
fn publish_staged_table(table_path: &Path, stage: &Path) -> eyre::Result<()> {
if let Some(parent) = table_path
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
{
std::fs::create_dir_all(parent)?;
}
if !table_path.exists() {
std::fs::rename(stage, table_path)?;
return Ok(());
}
let backup = staging_path(table_path, "backup")?;
std::fs::rename(table_path, &backup)?;
if let Err(error) = std::fs::rename(stage, table_path) {
if let Err(rollback_error) = std::fs::rename(&backup, table_path) {
return Err(eyre::eyre!(
"failed to install restored S3 table ({error}) and failed to restore local table ({rollback_error})"
));
}
return Err(error.into());
}
if let Err(error) = std::fs::remove_dir_all(&backup) {
tracing::warn!(path = %backup.display(), error = %error, "restored table but could not remove backup directory");
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use alloc::borrow::ToOwned;
const SIGNED_ERROR_URL: &str = "https://synthetic-user:synthetic-password@example.invalid/object?X-Amz-Credential=synthetic-credential&X-Amz-Signature=synthetic-signature&X-Amz-Security-Token=synthetic-token";
fn signed_status_error(status: u16) -> ureq::Error {
let response: ureq::Response = format!(
"HTTP/1.1 {status} {SIGNED_ERROR_URL}\r\nLocation: {SIGNED_ERROR_URL}\r\n\r\n{SIGNED_ERROR_URL}"
)
.parse()
.unwrap();
ureq::Error::Status(status, response)
}
fn signed_io_error() -> std::io::Error {
std::io::Error::other(std::io::Error::other(SIGNED_ERROR_URL))
}
fn assert_safe_s3_error(error: eyre::Report, expected: &str) {
assert_eq!(error.to_string(), expected);
assert_eq!(error.chain().count(), 1, "unsafe source must be discarded");
assert!(error.downcast_ref::<ureq::Error>().is_none());
assert!(error.downcast_ref::<std::io::Error>().is_none());
let error = error.wrap_err("snapshot operation failed");
let formatted = format!("{error}\n{error:#}\n{error:?}\n{error:#?}");
let sources = error
.chain()
.map(|source| format!("{source}\n{source:?}"))
.collect::<Vec<_>>()
.join("\n");
for text in [&formatted, &sources] {
assert!(text.contains(expected));
for secret in [
SIGNED_ERROR_URL,
"example.invalid",
"synthetic-user",
"synthetic-password",
"synthetic-credential",
"synthetic-signature",
"synthetic-token",
"X-Amz-",
] {
assert!(!text.contains(secret), "signed URL escaped sanitization");
}
}
}
#[test]
fn signed_request_errors_discard_status_and_transport_payloads() {
for operation in ["read", "listing", "write"] {
for status in [403, 404, 503] {
assert_safe_s3_error(
safe_s3_request_error(operation, signed_status_error(status)),
&format!("S3 {operation} failed with HTTP status {status}"),
);
}
let transport = ureq::Error::from(signed_io_error());
assert!(matches!(transport, ureq::Error::Transport(_)));
assert_safe_s3_error(
safe_s3_request_error(operation, transport),
&format!("S3 {operation} transport failed"),
);
}
}
#[test]
fn signed_response_read_errors_discard_nested_sources() {
assert_safe_s3_error(
safe_s3_read_error(signed_io_error()),
"S3 response body read failed",
);
}
#[test]
fn optional_s3_response_only_treats_404_as_absence() {
assert!(optional_s3_response(Err(signed_status_error(404)))
.unwrap()
.is_none());
assert!(
optional_s3_response(Ok(ureq::Response::new(200, "OK", "").unwrap()))
.unwrap()
.is_some()
);
for status in [403, 503] {
assert_safe_s3_error(
optional_s3_response(Err(signed_status_error(status))).unwrap_err(),
&format!("S3 read failed with HTTP status {status}"),
);
}
assert_safe_s3_error(
optional_s3_response(Err(ureq::Error::from(signed_io_error()))).unwrap_err(),
"S3 read transport failed",
);
}
fn inventory_manifest(contents: &[&[u8]]) -> Vec<u8> {
TableManifest {
files: contents
.iter()
.enumerate()
.map(|(i, bytes)| ManifestFile {
path: format!("{i}.wt.data"),
length: bytes.len() as u64,
extents: vec![extent(0, bytes)],
})
.collect(),
}
.encode()
.unwrap()
}
fn inventory_object(bytes: &[u8]) -> LegacyInventoryObject {
LegacyInventoryObject {
key: format!("db/table/chunks/{}", blake3::hash(bytes).to_hex()),
size: bytes.len() as u64,
}
}
fn snapshot_http(responses: Vec<(u16, Vec<u8>)>) -> (String, std::thread::JoinHandle<()>) {
snapshot_http_requests(
responses
.into_iter()
.map(|(status, body)| ("GET", "", None, status, body))
.collect(),
)
}
type HttpSequenceResponse = (&'static str, &'static str, Option<Vec<u8>>, u16, Vec<u8>);
fn snapshot_http_requests(
responses: Vec<HttpSequenceResponse>,
) -> (String, std::thread::JoinHandle<()>) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let task = std::thread::spawn(move || {
let deadline = std::time::Instant::now() + Duration::from_secs(5);
for (method, object, expected_body, status, body) in responses {
let mut socket = loop {
match listener.accept() {
Ok((socket, _)) => break socket,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
assert!(
std::time::Instant::now() < deadline,
"snapshot request did not arrive"
);
std::thread::sleep(Duration::from_millis(1));
}
Err(error) => panic!("accept: {error}"),
}
};
socket.set_nonblocking(false).unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(2)))
.unwrap();
let mut request = Vec::new();
let mut byte = [0];
while !request.ends_with(b"\r\n\r\n") {
socket.read_exact(&mut byte).unwrap();
request.push(byte[0]);
assert!(request.len() < 8192);
}
let headers = std::str::from_utf8(&request).unwrap();
let mut line = headers.lines().next().unwrap().split_whitespace();
assert_eq!(line.next(), Some(method));
let target = line.next().unwrap().split('?').next().unwrap();
if !object.is_empty() {
assert!(target.contains("/test/isolated/"));
if object == "chunks/" {
assert!(target.contains("/chunks/"));
} else {
assert!(target.ends_with(object));
}
}
let length: usize = headers
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse().unwrap())
})
.unwrap_or(0);
let mut uploaded = vec![0; length];
socket.read_exact(&mut uploaded).unwrap();
if let Some(expected) = expected_body {
assert_eq!(uploaded, expected);
if object == "chunks/" {
assert!(target.ends_with(blake3::hash(&uploaded).to_hex().as_str()));
}
}
write!(
socket,
"HTTP/1.1 {status} Result\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.unwrap();
socket.write_all(&body).unwrap();
}
});
(endpoint, task)
}
fn snapshot_config(endpoint: String, path: &Path) -> LegacySnapshotConfig {
LegacySnapshotConfig {
table_path: path.to_string_lossy().into_owned(),
s3: LegacyS3Config {
endpoint,
bucket_name: "test".into(),
access_key: "test-access".into(),
secret_key: "test-secret".into(),
region: None,
prefix: Some("isolated".into()),
},
}
}
fn list_xml(contents: &str, truncated: bool, token: &str) -> Vec<u8> {
let count = contents.matches("<Contents>").count();
format!("<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>test</Name><Prefix>isolated/table</Prefix><KeyCount>{count}</KeyCount><MaxKeys>1000</MaxKeys>{contents}<IsTruncated>{truncated}</IsTruncated>{}</ListBucketResult>",
if token.is_empty() { String::new() } else { format!("<NextContinuationToken>{token}</NextContinuationToken>") })
.into_bytes()
}
#[test]
fn migration_listing_requires_explicit_consistent_completion() {
assert!(parse_capture_listing(&list_xml("", false, "")).is_ok());
assert!(parse_capture_listing(&list_xml("", true, "next")).is_ok());
assert!(parse_capture_listing(&list_xml("", true, "")).is_err());
assert!(parse_capture_listing(&list_xml("", false, "next")).is_err());
for state in [
"",
"<IsTruncated/>",
"<IsTruncated>maybe</IsTruncated>",
"<IsTruncated>false</IsTruncated><IsTruncated>false</IsTruncated>",
] {
let xml = format!(
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">{state}</ListBucketResult>"
);
assert!(parse_capture_listing(xml.as_bytes()).is_err());
}
}
#[test]
fn migration_listing_rejects_oversized_response() {
let mut xml = list_xml("", false, "");
xml.resize(MAX_CAPTURE_LISTING_BYTES + 1, b' ');
let error = parse_capture_listing(&xml).unwrap_err();
assert!(error.to_string().contains("response byte limit"));
}
#[test]
fn migration_truncated_listing_without_token_cannot_prove_absence() {
let path = std::env::temp_dir().canonicalize().unwrap().join(format!(
"databucket-capture-truncated-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let (endpoint, server) = snapshot_http_requests(vec![
("GET", "manifest.v1", None, 404, Vec::new()),
("GET", "", None, 200, list_xml("", true, "")),
]);
let result = nagoya::block_on(LegacySnapshotStore::capture_for_migration(
snapshot_config(endpoint, &path),
MigrationCaptureLimits::default(),
));
server.join().unwrap();
assert!(result.is_err());
assert!(!path.exists());
}
#[test]
fn migration_capture_distinguishes_absence_and_complete_raw_files() {
let empty = std::env::temp_dir().canonicalize().unwrap().join(format!(
"databucket-capture-empty-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let (endpoint, server) = snapshot_http_requests(vec![
("GET", "manifest.v1", None, 404, Vec::new()),
("GET", "", None, 200, list_xml("", true, "next-page")),
("GET", "", None, 200, list_xml("", false, "")),
]);
let result = nagoya::block_on(LegacySnapshotStore::capture_for_migration(
snapshot_config(endpoint, &empty),
MigrationCaptureLimits::default(),
));
server.join().unwrap();
match result.unwrap() {
MigrationCaptureOutcome::Absent { provenance } => {
assert!(provenance.files.is_empty());
assert_eq!(provenance.total_bytes, 0);
}
_ => panic!("empty complete listing must prove absence"),
}
assert!(!empty.exists());
let raw_path = std::env::temp_dir().canonicalize().unwrap().join(format!(
"databucket-capture-raw-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let contents = "<Contents><LastModified>2026-01-01T00:00:00Z</LastModified><ETag>\"a\"</ETag><Key>isolated/table/.wt.data</Key><Size>3</Size></Contents><Contents><LastModified>2026-01-01T00:00:00Z</LastModified><ETag>\"b\"</ETag><Key>isolated/table/index.wt.idx</Key><Size>2</Size></Contents>";
let (endpoint, server) = snapshot_http_requests(vec![
("GET", "manifest.v1", None, 404, Vec::new()),
(
"GET",
"",
None,
200,
list_xml(
&contents.replace(
"isolated/table/",
&format!(
"isolated/{}/",
raw_path.file_name().unwrap().to_str().unwrap()
),
),
false,
"",
),
),
("GET", ".wt.data", None, 200, b"abc".to_vec()),
("GET", "index.wt.idx", None, 200, b"xy".to_vec()),
]);
let result = nagoya::block_on(LegacySnapshotStore::capture_for_migration(
snapshot_config(endpoint, &raw_path),
MigrationCaptureLimits::default(),
));
server.join().unwrap();
match result.unwrap() {
MigrationCaptureOutcome::LegacyFiles { provenance } => {
assert_eq!(provenance.files.len(), 2);
assert_eq!(provenance.total_bytes, 5);
}
_ => panic!("raw listing should produce a legacy capture"),
}
assert_eq!(std::fs::read(raw_path.join(".wt.data")).unwrap(), b"abc");
assert_eq!(std::fs::read(raw_path.join("index.wt.idx")).unwrap(), b"xy");
std::fs::remove_dir_all(raw_path).unwrap();
}
#[test]
fn migration_capture_returns_verified_manifest_provenance() {
let path = std::env::temp_dir().canonicalize().unwrap().join(format!(
"databucket-capture-manifest-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let manifest = TableManifest {
files: vec![ManifestFile {
path: ".wt.data".into(),
length: 3,
extents: vec![extent(0, b"abc")],
}],
}
.encode()
.unwrap();
let (endpoint, server) = snapshot_http(vec![(200, manifest), (200, b"abc".to_vec())]);
let result = nagoya::block_on(LegacySnapshotStore::capture_for_migration(
snapshot_config(endpoint, &path),
MigrationCaptureLimits::default(),
))
.unwrap();
server.join().unwrap();
match result {
MigrationCaptureOutcome::Manifest {
snapshot,
provenance,
} => {
assert_eq!(provenance.files, vec![(".wt.data".to_owned(), 3)]);
assert_eq!(provenance.total_bytes, 3);
assert_eq!(
std::fs::read(snapshot.table_path().join(".wt.data")).unwrap(),
b"abc"
);
}
_ => panic!("committed manifest should return manifest provenance"),
}
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn migration_capture_rejects_partial_or_unknown_namespaces() {
for (contents, status) in [
(
"<Contents><LastModified>2026-01-01T00:00:00Z</LastModified><ETag>\"a\"</ETag><Key>isolated/table/chunks/abcd</Key><Size>3</Size></Contents>",
200,
),
(
"<Contents><LastModified>2026-01-01T00:00:00Z</LastModified><ETag>\"a\"</ETag><Key>isolated/table/unknown.json</Key><Size>3</Size></Contents>",
200,
),
("", 403),
] {
let path = std::env::temp_dir().canonicalize().unwrap().join(format!(
"databucket-capture-reject-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let body = if status == 200 {
list_xml(
&contents.replace(
"isolated/table/",
&format!("isolated/{}/", path.file_name().unwrap().to_str().unwrap()),
),
false,
"",
)
} else {
Vec::new()
};
let (endpoint, server) = snapshot_http_requests(vec![
("GET", "manifest.v1", None, 404, Vec::new()),
("GET", "", None, status, body),
]);
let result = nagoya::block_on(LegacySnapshotStore::capture_for_migration(
snapshot_config(endpoint, &path),
MigrationCaptureLimits::default(),
));
server.join().unwrap();
assert!(
result.is_err(),
"partial or inaccessible namespaces are errors"
);
assert!(!path.exists());
}
}
#[test]
fn offline_publication_http_order_failures_and_exact_retry() {
let path = std::env::temp_dir().join(format!(
"databucket-publish-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&path).unwrap();
std::fs::write(path.join(".wt.data"), b"abc").unwrap();
let manifest = build_offline_snapshot(&path).unwrap().0.encode().unwrap();
let missing_head = ("GET", "manifest.v1", None, 404, Vec::new());
let missing_chunk = ("GET", "chunks/", None, 404, Vec::new());
let put_chunk = ("PUT", "chunks/", Some(b"abc".to_vec()), 200, Vec::new());
let read_chunk = ("GET", "chunks/", None, 200, b"abc".to_vec());
let put_head = (
"PUT",
"manifest.v1",
Some(manifest.clone()),
200,
Vec::new(),
);
let read_head = ("GET", "manifest.v1", None, 200, manifest.clone());
let prefix = vec![
missing_head.clone(),
missing_chunk.clone(),
put_chunk.clone(),
];
let mut success = prefix.clone();
success.extend([
read_chunk.clone(),
put_head.clone(),
read_chunk.clone(),
read_head.clone(),
]);
let mut cases = vec![
(success, true),
(
vec![
read_head.clone(),
read_chunk.clone(),
read_chunk.clone(),
read_chunk.clone(),
read_head,
],
true,
),
(vec![("GET", "manifest.v1", None, 503, Vec::new())], false),
(
vec![("GET", "manifest.v1", None, 200, b"divergent".to_vec())],
false,
),
(
vec![
missing_head.clone(),
("GET", "chunks/", None, 403, Vec::new()),
],
false,
),
(
vec![
missing_head.clone(),
("GET", "chunks/", None, 200, b"bad".to_vec()),
],
false,
),
(
vec![
missing_head,
missing_chunk,
("PUT", "chunks/", Some(b"abc".to_vec()), 500, Vec::new()),
],
false,
),
];
for (status, bytes) in [(404, Vec::new()), (200, b"bad".to_vec())] {
let mut responses = prefix.clone();
responses.push(("GET", "chunks/", None, status, bytes));
cases.push((responses, false));
}
let mut failed_head = prefix.clone();
failed_head.extend([
read_chunk.clone(),
(
"PUT",
"manifest.v1",
Some(manifest.clone()),
500,
Vec::new(),
),
]);
cases.push((failed_head, false));
let mut failed_final_chunk = prefix.clone();
failed_final_chunk.extend([
read_chunk.clone(),
put_head.clone(),
("GET", "chunks/", None, 404, Vec::new()),
]);
cases.push((failed_final_chunk, false));
let mut failed_final_head = prefix;
failed_final_head.extend([
read_chunk.clone(),
put_head,
read_chunk,
("GET", "manifest.v1", None, 200, b"bad".to_vec()),
]);
cases.push((failed_final_head, false));
for (responses, succeeds) in cases {
let (endpoint, server) = snapshot_http_requests(responses);
let result = nagoya::block_on(LegacySnapshotStore::publish_new_fenced(
snapshot_config(endpoint, &path),
ExclusiveFencedLegacyDestination::acknowledge_external_fence(),
));
server.join().unwrap();
assert_eq!(result.is_ok(), succeeds, "{result:?}");
if let Ok(summary) = result {
assert_eq!(summary.file_count, 1);
assert_eq!(summary.unique_chunk_count, 1);
assert_eq!(summary.manifest_blake3, *blake3::hash(&manifest).as_bytes());
}
}
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn offline_inventory_primary_dedup_and_symlinks() {
let path = std::env::temp_dir().join(format!(
"databucket-inventory-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&path).unwrap();
std::fs::write(path.join("index.wt.idx"), b"abc").unwrap();
assert!(build_offline_snapshot(&path).is_err());
std::fs::write(path.join(".wt.data"), b"abc").unwrap();
let (manifest, chunks) = build_offline_snapshot(&path).unwrap();
assert_eq!(manifest.files.len(), 2);
assert_eq!(chunks.len(), 1);
assert!(manifest
.files
.iter()
.all(|file| file.extents[0].hash == *blake3::hash(b"abc").as_bytes()));
#[cfg(unix)]
{
std::os::unix::fs::symlink(path.join(".wt.data"), path.join("alias.wt.idx")).unwrap();
assert!(build_offline_snapshot(&path).is_err());
}
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn publication_preflight_summarizes_deduplicated_snapshot_without_writes() {
let path = std::env::temp_dir().join(format!(
"databucket-preflight-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&path).unwrap();
std::fs::write(path.join(".wt.data"), b"same bytes").unwrap();
std::fs::write(path.join("index.wt.idx"), b"same bytes").unwrap();
let (manifest, chunks) = build_offline_snapshot(&path).unwrap();
let manifest_bytes = manifest.encode().unwrap();
let before_data = std::fs::read(path.join(".wt.data")).unwrap();
let before_index = std::fs::read(path.join("index.wt.idx")).unwrap();
let summary = LegacySnapshotStore::prepare_publication(&path).unwrap();
assert_eq!(summary.file_count, 2);
assert_eq!(summary.unique_chunk_count, 1);
assert_eq!(
summary.manifest_blake3,
*blake3::hash(&manifest_bytes).as_bytes()
);
assert_eq!(summary.file_count, manifest.files.len());
assert_eq!(summary.unique_chunk_count, chunks.len());
assert_eq!(std::fs::read(path.join(".wt.data")).unwrap(), before_data);
assert_eq!(
std::fs::read(path.join("index.wt.idx")).unwrap(),
before_index
);
std::fs::remove_file(path.join(".wt.data")).unwrap();
assert!(LegacySnapshotStore::prepare_publication(&path).is_err());
assert!(!path.join(".wt.data").exists());
assert_eq!(
std::fs::read(path.join("index.wt.idx")).unwrap(),
before_index
);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn strict_restore_requires_manifest_and_adopts_without_refetch() {
let path = std::env::temp_dir().join(format!(
"databucket-strict-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let manifest = TableManifest {
files: vec![ManifestFile {
path: ".wt.data".into(),
length: 3,
extents: vec![extent(0, b"abc")],
}],
}
.encode()
.unwrap();
let (endpoint, server) = snapshot_http(vec![(200, manifest), (200, b"abc".to_vec())]);
let config = snapshot_config(endpoint, &path);
let restored =
nagoya::block_on(LegacySnapshotStore::restore_required(config.clone())).unwrap();
server.join().unwrap();
assert_eq!(std::fs::read(path.join(".wt.data")).unwrap(), b"abc");
assert_eq!(restored.restored_files(), &[path.join(".wt.data")]);
let mut store = restored.into_store_for(&config).unwrap();
store.capture_baseline().unwrap();
std::fs::remove_dir_all(&path).unwrap();
let (endpoint, server) = snapshot_http(vec![(404, Vec::new())]);
let config = snapshot_config(endpoint, &path);
assert!(nagoya::block_on(LegacySnapshotStore::restore_required(config)).is_err());
server.join().unwrap();
assert!(
!path.exists(),
"missing remote state must not initialize a local table"
);
}
#[test]
fn strict_restore_rejects_superseded_same_length_snapshot() {
let path = std::env::temp_dir().join(format!(
"databucket-stale-{}-{}",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let manifest = |bytes: &[u8]| {
TableManifest {
files: vec![ManifestFile {
path: ".wt.data".into(),
length: bytes.len() as u64,
extents: vec![extent(0, bytes)],
}],
}
.encode()
.unwrap()
};
let (endpoint, server) = snapshot_http(vec![
(200, manifest(b"abc")),
(200, b"abc".to_vec()),
(200, manifest(b"xyz")),
(200, b"xyz".to_vec()),
]);
let config = snapshot_config(endpoint, &path);
let first =
nagoya::block_on(LegacySnapshotStore::restore_required(config.clone())).unwrap();
let second =
nagoya::block_on(LegacySnapshotStore::restore_required(config.clone())).unwrap();
server.join().unwrap();
assert!(
first.into_store_for(&config).is_err(),
"superseded token must not adopt a different snapshot"
);
let mut store = second.into_store_for(&config).unwrap();
std::fs::write(path.join(".wt.data"), b"def").unwrap();
assert!(
store.capture_baseline().is_err(),
"mutation between adoption and baseline must be rejected"
);
std::fs::remove_dir_all(path).unwrap();
}
#[test]
fn inventory_deduplicates_shared_roots_and_preserves_unknown_objects() {
let current = inventory_manifest(&[b"a", b"bb"]);
let retained = inventory_manifest(&[b"bb", b"ccc"]);
let unknown = LegacyInventoryObject {
key: "db/table/raw.wt.data".into(),
size: 99,
};
let listing = LegacyObjectListing {
objects: vec![
inventory_object(b"a"),
inventory_object(b"bb"),
inventory_object(b"ccc"),
inventory_object(b"dddd"),
unknown.clone(),
],
complete: true,
};
let result = inventory_legacy_roots(
"db/table/",
¤t,
&[retained],
listing,
LegacyInventoryLimits::default(),
)
.unwrap();
assert_eq!(result.current_root.unique_chunk_bytes, 3);
assert_eq!(result.retained_roots[0].unique_chunk_bytes, 5);
assert_eq!(
(
result.current_only_chunk_bytes,
result.shared_chunk_bytes,
result.retained_only_chunk_bytes
),
(1, 2, 3)
);
assert_eq!(result.unreferenced_chunk_bytes, Some(4));
assert_eq!(result.preserved_objects, vec![unknown]);
}
#[test]
fn inventory_incomplete_listing_keeps_expected_bytes_but_no_orphan_total() {
let manifest = inventory_manifest(&[b"abc"]);
let listing = LegacyObjectListing {
objects: Vec::new(),
complete: false,
};
let result = inventory_legacy_roots(
"db/table/",
&manifest,
&[],
listing.clone(),
LegacyInventoryLimits::default(),
)
.unwrap();
assert_eq!(result.current_root.unique_chunk_bytes, 3);
assert_eq!(result.current_only_chunk_bytes, 3);
assert_eq!(result.unreferenced_chunk_bytes, None);
assert_eq!(
result.unlisted_references,
vec![*blake3::hash(b"abc").as_bytes()]
);
assert!(inventory_legacy_roots(
"db/table/",
&manifest,
&[],
LegacyObjectListing {
complete: true,
..listing
},
LegacyInventoryLimits::default()
)
.is_err());
}
#[test]
fn inventory_refuses_conflicting_roots_corruption_and_scope_escape() {
let manifest = inventory_manifest(&[b"abc"]);
let mut conflicting = TableManifest::decode(&manifest).unwrap();
conflicting.files[0].extents[0].segment_length = 4;
let conflicting = conflicting.encode().unwrap();
let listing = LegacyObjectListing {
objects: vec![inventory_object(b"abc")],
complete: true,
};
assert!(inventory_legacy_roots(
"db/table/",
&manifest,
&[conflicting],
listing.clone(),
LegacyInventoryLimits::default()
)
.is_err());
let mut corrupt = manifest.clone();
corrupt[12] ^= 1;
assert!(inventory_legacy_roots(
"db/table/",
&corrupt,
&[],
listing.clone(),
LegacyInventoryLimits::default()
)
.is_err());
let outside = LegacyObjectListing {
objects: vec![LegacyInventoryObject {
key: "other/table/chunks/abc".into(),
size: 3,
}],
complete: true,
};
assert!(inventory_legacy_roots(
"db/table/",
&manifest,
&[],
outside,
LegacyInventoryLimits::default()
)
.is_err());
assert!(inventory_legacy_roots(
"db/../table/",
&manifest,
&[],
listing,
LegacyInventoryLimits::default()
)
.is_err());
}
#[test]
fn inventory_enforces_aggregate_manifest_reference_and_metadata_limits() {
let manifest = inventory_manifest(&[b"abc"]);
let listing = LegacyObjectListing {
objects: vec![inventory_object(b"abc")],
complete: true,
};
let limits = LegacyInventoryLimits {
max_manifest_bytes: manifest.len(),
..Default::default()
};
assert!(inventory_legacy_roots(
"db/table/",
&manifest,
core::slice::from_ref(&manifest),
listing.clone(),
limits
)
.is_err());
let limits = LegacyInventoryLimits {
max_references: 0,
..Default::default()
};
assert!(
inventory_legacy_roots("db/table/", &manifest, &[], listing.clone(), limits).is_err()
);
let limits = LegacyInventoryLimits {
max_metadata_bytes: 1,
..Default::default()
};
assert!(
inventory_legacy_roots("db/table/", &manifest, &[], listing.clone(), limits).is_err()
);
let limits = LegacyInventoryLimits {
max_key_bytes: 1,
..Default::default()
};
assert!(inventory_legacy_roots("db/table/", &manifest, &[], listing, limits).is_err());
}
#[test]
fn s3_config_debug_redacts_credentials() {
let config = LegacyS3Config {
bucket_name: "bucket".to_owned(),
endpoint: "https://example.invalid".to_owned(),
access_key: "test-access-marker".to_owned(),
secret_key: "test-secret-marker".to_owned(),
region: Some("region".to_owned()),
prefix: None,
};
let debug = format!("{config:?}");
assert!(!debug.contains("test-access-marker"));
assert!(!debug.contains("test-secret-marker"));
assert!(debug.contains("LegacyS3Config"));
}
fn extent(file_offset: u64, bytes: &[u8]) -> SegmentExtent {
SegmentExtent {
file_offset,
length: bytes.len() as u32,
segment_offset: 0,
segment_length: bytes.len() as u32,
hash: *blake3::hash(bytes).as_bytes(),
}
}
#[test]
fn manifest_round_trip_is_deterministic() {
let manifest = TableManifest {
files: vec![
ManifestFile {
path: ".wt.data".to_string(),
length: 3,
extents: vec![extent(0, b"abc")],
},
ManifestFile {
path: "primary.wt.idx".to_string(),
length: 0,
extents: Vec::new(),
},
],
};
let encoded = manifest.encode().unwrap();
assert_eq!(TableManifest::decode(&encoded).unwrap(), manifest);
assert_eq!(manifest.encode().unwrap(), encoded);
}
#[test]
fn manifest_rejects_corruption_and_unsafe_paths() {
let manifest = TableManifest {
files: vec![ManifestFile {
path: ".wt.data".to_string(),
length: 3,
extents: vec![extent(0, b"abc")],
}],
};
let mut encoded = manifest.encode().unwrap();
encoded[12] ^= 1;
assert!(TableManifest::decode(&encoded).is_err());
let unsafe_manifest = TableManifest {
files: vec![ManifestFile {
path: "../outside.wt.data".to_string(),
length: 0,
extents: Vec::new(),
}],
};
assert!(unsafe_manifest.encode().is_err());
}
#[test]
fn manifest_requires_exact_extent_coverage() {
let manifest = TableManifest {
files: vec![ManifestFile {
path: ".wt.data".to_string(),
length: 4,
extents: vec![extent(0, b"abc")],
}],
};
assert!(manifest.encode().is_err());
}
#[test]
fn manifest_requires_strictly_sorted_unique_paths() {
let file = |path: &str| ManifestFile {
path: path.to_string(),
length: 0,
extents: Vec::new(),
};
let unsorted = TableManifest {
files: vec![file("primary.wt.idx"), file(".wt.data")],
}
.encode()
.unwrap();
assert!(TableManifest::decode(&unsorted).is_err());
let duplicate = TableManifest {
files: vec![file(".wt.data"), file(".wt.data")],
}
.encode()
.unwrap();
assert!(TableManifest::decode(&duplicate).is_err());
}
#[test]
fn a_page_change_splits_a_large_segment_without_reuploading_it() {
let original = vec![7_u8; SEGMENT_TARGET];
let original_extent = extent(0, &original);
let changed_page = vec![9_u8; CHANGE_BLOCK_SIZE];
let replacement = extent(CHANGE_BLOCK_SIZE as u64, &changed_page);
let extents =
overlay_extent(core::slice::from_ref(&original_extent), replacement.clone()).unwrap();
assert_eq!(extents.len(), 3);
assert_eq!(extents[0].length as usize, CHANGE_BLOCK_SIZE);
assert_eq!(extents[1], replacement);
assert_eq!(extents[2].file_offset, (2 * CHANGE_BLOCK_SIZE) as u64);
assert_eq!(extents[2].segment_offset, (2 * CHANGE_BLOCK_SIZE) as u32);
assert_eq!(extents[2].file_end().unwrap(), SEGMENT_TARGET as u64);
assert_eq!(extents[0].hash, original_extent.hash);
assert_eq!(extents[2].hash, original_extent.hash);
}
#[test]
fn legacy_manifest_decodes_as_segment_extents() {
let contents = [vec![1_u8; SEGMENT_TARGET], vec![2_u8; 19]];
let mut payload = Vec::new();
payload.extend_from_slice(MANIFEST_MAGIC_V1);
payload.extend_from_slice(&1_u32.to_le_bytes());
let path = b".wt.data";
payload.extend_from_slice(&(path.len() as u16).to_le_bytes());
payload.extend_from_slice(path);
payload.extend_from_slice(&((SEGMENT_TARGET + 19) as u64).to_le_bytes());
payload.extend_from_slice(&2_u32.to_le_bytes());
for bytes in &contents {
payload.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
payload.extend_from_slice(blake3::hash(bytes).as_bytes());
}
let checksum = blake3::hash(&payload);
payload.extend_from_slice(checksum.as_bytes());
let manifest = TableManifest::decode(&payload).unwrap();
let file = &manifest.files[0];
assert_eq!(file.extents.len(), 2);
assert_eq!(file.extents[0].length as usize, SEGMENT_TARGET);
assert_eq!(file.extents[1].file_offset, SEGMENT_TARGET as u64);
assert!(manifest.encode().unwrap().starts_with(MANIFEST_MAGIC_V2));
}
}