hyperfile/
config.rs

1use std::fmt;
2use std::io::{Error, ErrorKind, Result};
3use log::warn;
4use serde::{Deserialize, Serialize};
5use crate::meta_format::BlockPtrFormat;
6use crate::staging::config::StagingConfig;
7use crate::data_cache::config::HyperFileDataCacheConfig;
8use crate::node_cache::config::HyperFileNodeCacheConfig;
9#[cfg(feature = "wal")]
10use crate::wal::config::HyperFileWalConfig;
11use crate::*;
12
13const MIN_ROOT_SIZE: usize = 56;
14const MIN_META_BLOCK_SIZE: usize = 4096;
15const MIN_DATA_BLOCK_SIZE: usize = 4096;
16/// Largest block-size shift a container may name. Not a policy — 1 GiB
17/// blocks are already absurd — but a bound, so that a byte from a container
18/// this build does not understand cannot become a shift overflow. The
19/// encoded field is a whole byte, so without this a stray value shifts by up
20/// to 255.
21const MAX_BLOCK_SHIFT: u32 = 30;
22
23pub(crate) const DEFAULT_ROOT_SIZE: usize = MIN_ROOT_SIZE;
24const DEFAULT_META_BLOCK_SIZE: usize = MIN_META_BLOCK_SIZE;
25const DEFAULT_DATA_BLOCK_SIZE: usize = MIN_DATA_BLOCK_SIZE;
26// `MicroGroup`, because that is what every file created so far actually
27// got: `HyperFile::new` hardcoded it and ignored this config. Now that it
28// honours the config, the default has to name the same format or existing
29// callers would silently start writing a different one.
30/// A checkpoint is written as `<segid>.<part>` objects, which is what lets memory
31/// pressure be answered with a partial segment instead of a checkpoint nobody asked
32/// for. See `HyperFileRuntimeConfig::parted_segment_enabled` for taking that
33/// behaviour back, and `BlockPtrFormat::PartedSegment` for the addressing.
34///
35/// A container carries its format for life, and an earlier build refuses one it does
36/// not know rather than misreading it — so a container created by this version is
37/// not readable by a `0.6` one.
38const DEFAULT_BLOCK_PTR_FORMAT: BlockPtrFormat = BlockPtrFormat::PartedSegment;
39
40#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
41pub struct HyperFileMetaConfig {
42    pub root_size: usize,
43    pub meta_block_size: usize,
44    pub data_block_size: usize,
45    pub block_ptr_format: BlockPtrFormat,
46}
47
48impl fmt::Display for HyperFileMetaConfig {
49    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
50        write!(f, "root size: {}, meta block size: {}, data block size: {}, block ptr format: {:?}",
51            self.root_size, self.meta_block_size, self.data_block_size, self.block_ptr_format)
52    }
53}
54
55impl HyperFileMetaConfig {
56    pub fn new(root_size: usize, meta_block_size: usize, data_block_size: usize, block_ptr_format: BlockPtrFormat) -> Self {
57        let root_size = if root_size != DEFAULT_ROOT_SIZE {
58            warn!("only support fixed root size == {} at this moment", DEFAULT_ROOT_SIZE);
59            DEFAULT_ROOT_SIZE
60        } else {
61            root_size
62        };
63        // enforce input min size and log2 aligned
64        let root_size = std::cmp::max(root_size, DEFAULT_ROOT_SIZE);
65        let meta_block_size = std::cmp::max(meta_block_size, DEFAULT_META_BLOCK_SIZE);
66        let data_block_size = std::cmp::max(data_block_size, DEFAULT_DATA_BLOCK_SIZE);
67        let meta_block_size = 1 << meta_block_size.checked_ilog2().unwrap();
68        let data_block_size = 1 << data_block_size.checked_ilog2().unwrap();
69        Self {
70            root_size: root_size,
71            meta_block_size: meta_block_size,
72            data_block_size: data_block_size,
73            block_ptr_format: block_ptr_format,
74        }
75    }
76
77    // encode as u32
78    // format:
79    //   root_mul (multiple of 8 bytes) | meta block shift | data block shift | block ptr format
80    pub fn as_u32(&self) -> u32 {
81        let root_multiple: u32 = (self.root_size / 8) as u32;
82        assert!(self.root_size % 8 == 0);
83        let meta_block_shift: u32 = self.meta_block_size.checked_ilog2().expect("invalid meta block size");
84        let data_block_shift: u32 = self.data_block_size.checked_ilog2().expect("invalid meta block size");
85        let block_ptr_format: u32 = self.block_ptr_format as u8 as u32;
86        root_multiple << 24 | meta_block_shift << 16 | data_block_shift << 8 | block_ptr_format
87    }
88
89    /// Decode from the `u32` an inode carries, rejecting anything this build
90    /// cannot represent.
91    ///
92    /// Every field is checked, because this value comes from storage and may
93    /// have been written by a version whose layout differs. A container from
94    /// before this field was populated carries zero, which decodes to a root
95    /// of 0 bytes and blocks of 1 byte — all three below the minimums that
96    /// were already named above. Read without checking, those propagate: the
97    /// caller's own config is overwritten by what the container says (see
98    /// `HyperFile::do_open`), so a one-byte block size then decides how every
99    /// read and write is cut up.
100    ///
101    /// Refusing is the point. A container this build does not understand
102    /// cannot be read correctly, and reporting the raw value at the first
103    /// step is much better than presenting a file whose size and block size
104    /// are quietly wrong.
105    pub fn try_from_u32(data: u32) -> Result<Self> {
106        let block_ptr_format = BlockPtrFormat::try_from_u8((data & 0xFF) as u8)?;
107        if block_ptr_format == BlockPtrFormat::Nop {
108            return Err(Error::new(ErrorKind::InvalidData,
109                "block ptr format is Nop, which no container is written with"));
110        }
111
112        let data_shift = (data >> 8) & 0xFF;
113        let meta_shift = (data >> 16) & 0xFF;
114        // Checked before shifting, not after: the shift itself would
115        // overflow.
116        for (what, shift, min) in [
117            ("data block size", data_shift, MIN_DATA_BLOCK_SIZE),
118            ("meta block size", meta_shift, MIN_META_BLOCK_SIZE),
119        ] {
120            let min_shift = min.ilog2();
121            if shift < min_shift || shift > MAX_BLOCK_SHIFT {
122                return Err(Error::new(ErrorKind::InvalidData, format!(
123                    "{} shift {} is outside {}..={}", what, shift, min_shift, MAX_BLOCK_SHIFT)));
124            }
125        }
126        let data_block_size = 1usize << data_shift;
127        let meta_block_size = 1usize << meta_shift;
128
129        let root_size: usize = (((data >> 24) & 0xFF) * 8) as usize;
130        if root_size < MIN_ROOT_SIZE {
131            return Err(Error::new(ErrorKind::InvalidData, format!(
132                "root size {} is below the minimum {}", root_size, MIN_ROOT_SIZE)));
133        }
134
135        Ok(Self {
136            root_size,
137            meta_block_size,
138            data_block_size,
139            block_ptr_format,
140        })
141    }
142
143    /// Decode from a `u32` this crate produced itself, panicking on one it
144    /// cannot represent. Anything read from a container should go through
145    /// [`Self::try_from_u32`].
146    pub fn from_u32(data: u32) -> Self {
147        Self::try_from_u32(data).expect("meta config")
148    }
149}
150
151impl Default for HyperFileMetaConfig {
152    fn default() -> Self {
153        Self {
154            root_size: DEFAULT_ROOT_SIZE,
155            meta_block_size: DEFAULT_META_BLOCK_SIZE,
156            data_block_size: DEFAULT_DATA_BLOCK_SIZE,
157            block_ptr_format: DEFAULT_BLOCK_PTR_FORMAT,
158        }
159    }
160}
161
162#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
163pub struct HyperFileRuntimeConfig {
164    // forward origin
165    pub forward_origin_concurrency: usize,
166    pub forward_origin_threshold: usize,
167    pub forward_origin_chunk_size: usize,
168    // segment
169    pub segment_buffer_size: usize,
170    pub segment_mpu_chunk_size: usize,
171    /// The most parts of one segment's multipart upload in flight at once; the next
172    /// part starts as one finishes. Default 8. Zero is taken as one.
173    ///
174    /// A segment longer than `segment_mpu_chunk_size` goes up in parts of that size.
175    /// Each part in flight is a request with its own connection and TLS and HTTP
176    /// buffers, and with `wal` also a copy of its range of the segment. In the default
177    /// build a part is a view of the segment's blocks, not a copy.
178    ///
179    /// It does not bound the segment itself, which is held in memory until the upload
180    /// ends whatever this is.
181    #[serde(default = "default_segment_mpu_max_concurrency")]
182    pub segment_mpu_max_concurrency: usize,
183    /// The most bytes one stored object of a checkpoint may hold. Default, and
184    /// upper limit, 1 GiB: what a block pointer can address within one object.
185    ///
186    /// A checkpoint larger than this is written as several objects under the
187    /// parted block-pointer format — data-only parts first, then the part holding
188    /// the index and the inode, which is what makes it a checkpoint — and is
189    /// published as one. Under the older formats, and with `wal`, a checkpoint is
190    /// always one object, and a flush that would need more is refused before
191    /// anything is written.
192    ///
193    /// Values above the limit are taken as the limit; below 16 KiB, as 16 KiB.
194    #[serde(default = "default_segment_max_object_bytes")]
195    pub segment_max_object_bytes: usize,
196    // data cache & dirty cache
197    pub data_cache_blocks: usize,
198    pub data_cache_dirty_max_bytes_threshold: usize,
199    pub data_cache_dirty_max_blocks_threshold: usize,
200    pub data_cache_dirty_max_flush_interval: u64,
201    // bmap node cache
202    pub node_cache_blocks: usize,
203    /// Cap on the number of bytes a single S3 GET issued by the
204    /// read path's coalescing logic may cover. Larger contiguous
205    /// regions are split into this-sized sub-ranges.
206    #[serde(default = "default_read_get_max_bytes")]
207    pub read_get_max_bytes: usize,
208
209    /// Whether this container writes partial segments when memory pressure asks
210    /// for it, instead of publishing a checkpoint nobody asked for.
211    ///
212    /// On by default, which changes nothing for a container whose format is not
213    /// [`crate::meta_format::BlockPtrFormat::PartedSegment`] — a partial segment is
214    /// addressable only under that format, and the format is fixed when the
215    /// container is created. So asking for the format is asking for the behaviour,
216    /// and this exists to take it back: a container can be created with the format
217    /// and made to behave exactly as before by clearing this.
218    ///
219    /// What it changes: a dirty-data threshold, or the flush interval, writes the
220    /// dirty set out as a partial segment and keeps accumulating, rather than
221    /// publishing. Under a transaction or `WalRecoveryMode::Barrier`, where
222    /// crossing a threshold is refused with `OutOfMemory` because nothing may
223    /// publish, it writes a partial instead — which is the case this exists for.
224    ///
225    /// What it does not change: an explicit `flush`, `fdatasync`, `release` or
226    /// `commit_txn` still writes a consistency point, and with no partial
227    /// outstanding it writes exactly the single segment it always did.
228    #[serde(default = "default_parted_segment_enabled")]
229    pub parted_segment_enabled: bool,
230
231    /// Least dirty data worth writing as a partial segment. Below it, a threshold
232    /// crossing behaves as it always did.
233    ///
234    /// A partial segment costs an object and a break in what a sequential read can
235    /// coalesce, so a small one buys little memory for a lasting cost in
236    /// fragmentation. Defaults to `data_cache_dirty_max_bytes_threshold`'s own
237    /// default, which is the amount that triggers a write in the first place.
238    #[serde(default = "default_data_cache_dirty_min_bytes_to_part")]
239    pub data_cache_dirty_min_bytes_to_part: usize,
240    /// Backpressure cap on in-flight S3 GETs spawned by a single
241    /// `fs_read` / `fh_read` after Level-A coalescing.
242    #[serde(default = "default_read_max_concurrency")]
243    pub read_max_concurrency: usize,
244    /// How to resolve flush conflicts when another writer has modified
245    /// the same file between our read and our write. See
246    /// `FlushConflictPolicy` for details.
247    #[serde(default)]
248    pub flush_conflict_policy: FlushConflictPolicy,
249}
250
251fn default_read_get_max_bytes() -> usize { DEFAULT_READ_GET_MAX_BYTES }
252fn default_segment_mpu_max_concurrency() -> usize { DEFAULT_SEGMENT_MPU_MAX_CONCURRENCY }
253fn default_segment_max_object_bytes() -> usize { DEFAULT_SEGMENT_MAX_OBJECT_BYTES }
254fn default_parted_segment_enabled() -> bool { true }
255fn default_data_cache_dirty_min_bytes_to_part() -> usize { DEFAULT_MAX_DIRTY_DATA_BYTES_THRESHOLD }
256fn default_read_max_concurrency() -> usize { DEFAULT_READ_MAX_CONCURRENCY }
257
258/// Policy that controls how `flush` handles a concurrent modification
259/// detected at the storage layer (S3 PutObject returns 412 Precondition
260/// Failed because another writer committed since we read the ETag).
261///
262/// Default is `RetryLastWriterWins` to preserve the pre-existing
263/// single-writer / relaxed behavior.
264#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
265pub enum FlushConflictPolicy {
266    /// Retry on conflict by refreshing local state from the persisted
267    /// inode and re-flushing. In a multi-writer scenario this means the
268    /// later writer silently overwrites the earlier writer's change.
269    /// Convenient for single-writer workloads; NOT safe if your workload
270    /// cannot tolerate silent overwrites.
271    RetryLastWriterWins,
272
273    /// Return `std::io::ErrorKind::AlreadyExists` on the first conflict
274    /// without retrying. The caller decides whether to re-read, merge,
275    /// or abort. Use this for workflows that need explicit conflict
276    /// detection (e.g. ensuring no lost updates).
277    FailFast,
278}
279
280impl Default for FlushConflictPolicy {
281    fn default() -> Self {
282        Self::RetryLastWriterWins
283    }
284}
285
286impl Default for HyperFileRuntimeConfig {
287    fn default() -> Self {
288        Self {
289            forward_origin_concurrency: DEFAULT_FORWARD_ORIGIN_CONCURRENCY,
290            forward_origin_threshold: DEFAULT_FORWARD_ORIGIN_THRESHOLD,
291            forward_origin_chunk_size: DEFAULT_FORWARD_ORIGIN_CHUNK_SIZE,
292            segment_buffer_size: DEFAULT_SEGMENT_BUFFER_SIZE,
293            segment_mpu_chunk_size: DEFAULT_SEGMENT_MPU_CHUNK_SIZE,
294            segment_mpu_max_concurrency: DEFAULT_SEGMENT_MPU_MAX_CONCURRENCY,
295            segment_max_object_bytes: DEFAULT_SEGMENT_MAX_OBJECT_BYTES,
296            data_cache_blocks: DEFAULT_DATA_CACHE_BLOCKS,
297            data_cache_dirty_max_bytes_threshold: DEFAULT_MAX_DIRTY_DATA_BYTES_THRESHOLD,
298            data_cache_dirty_max_blocks_threshold: DEFAULT_MAX_DIRTY_DATA_BLOCKS_THRESHOLD,
299            data_cache_dirty_max_flush_interval: DEFAULT_MAX_DIRTY_DATA_FLUSH_INTERVAL,
300            node_cache_blocks: DEFAULT_NODE_CACHE_BLOCKS,
301            flush_conflict_policy: FlushConflictPolicy::default(),
302            read_get_max_bytes: DEFAULT_READ_GET_MAX_BYTES,
303            parted_segment_enabled: true,
304            data_cache_dirty_min_bytes_to_part: DEFAULT_MAX_DIRTY_DATA_BYTES_THRESHOLD,
305            read_max_concurrency: DEFAULT_READ_MAX_CONCURRENCY,
306        }
307    }
308}
309
310impl HyperFileRuntimeConfig {
311    pub fn default_large() -> Self {
312        Self {
313            forward_origin_concurrency: DEFAULT_FORWARD_ORIGIN_CONCURRENCY,
314            forward_origin_threshold: DEFAULT_FORWARD_ORIGIN_THRESHOLD,
315            forward_origin_chunk_size: DEFAULT_FORWARD_ORIGIN_CHUNK_SIZE,
316            segment_buffer_size: DEFAULT_LARGE_SEGMENT_BUFFER_SIZE,
317            segment_mpu_chunk_size: DEFAULT_SEGMENT_MPU_CHUNK_SIZE,
318            segment_mpu_max_concurrency: DEFAULT_SEGMENT_MPU_MAX_CONCURRENCY,
319            segment_max_object_bytes: DEFAULT_SEGMENT_MAX_OBJECT_BYTES,
320            data_cache_blocks: DEFAULT_LARGE_DATA_CACHE_BLOCKS,
321            data_cache_dirty_max_bytes_threshold: DEFAULT_LARGE_MAX_DIRTY_DATA_BYTES_THRESHOLD,
322            data_cache_dirty_max_blocks_threshold: DEFAULT_LARGE_MAX_DIRTY_DATA_BLOCKS_THRESHOLD,
323            data_cache_dirty_max_flush_interval: DEFAULT_MAX_DIRTY_DATA_FLUSH_INTERVAL,
324            node_cache_blocks: DEFAULT_MAX_NODE_CACHE_BLOCKS,
325            flush_conflict_policy: FlushConflictPolicy::default(),
326            read_get_max_bytes: DEFAULT_READ_GET_MAX_BYTES,
327            parted_segment_enabled: true,
328            data_cache_dirty_min_bytes_to_part: DEFAULT_MAX_DIRTY_DATA_BYTES_THRESHOLD,
329            read_max_concurrency: DEFAULT_READ_MAX_CONCURRENCY,
330        }
331    }
332
333    pub fn default_middle() -> Self {
334        Self {
335            forward_origin_concurrency: DEFAULT_FORWARD_ORIGIN_CONCURRENCY,
336            forward_origin_threshold: DEFAULT_FORWARD_ORIGIN_THRESHOLD,
337            forward_origin_chunk_size: DEFAULT_FORWARD_ORIGIN_CHUNK_SIZE,
338            segment_buffer_size: DEFAULT_MIDDLE_SEGMENT_BUFFER_SIZE,
339            segment_mpu_chunk_size: DEFAULT_SEGMENT_MPU_CHUNK_SIZE,
340            segment_mpu_max_concurrency: DEFAULT_SEGMENT_MPU_MAX_CONCURRENCY,
341            segment_max_object_bytes: DEFAULT_SEGMENT_MAX_OBJECT_BYTES,
342            data_cache_blocks: DEFAULT_MIDDLE_DATA_CACHE_BLOCKS,
343            data_cache_dirty_max_bytes_threshold: DEFAULT_MIDDLE_MAX_DIRTY_DATA_BYTES_THRESHOLD,
344            data_cache_dirty_max_blocks_threshold: DEFAULT_MIDDLE_MAX_DIRTY_DATA_BLOCKS_THRESHOLD,
345            data_cache_dirty_max_flush_interval: DEFAULT_MAX_DIRTY_DATA_FLUSH_INTERVAL,
346            node_cache_blocks: DEFAULT_MAX_NODE_CACHE_BLOCKS,
347            flush_conflict_policy: FlushConflictPolicy::default(),
348            read_get_max_bytes: DEFAULT_READ_GET_MAX_BYTES,
349            parted_segment_enabled: true,
350            data_cache_dirty_min_bytes_to_part: DEFAULT_MAX_DIRTY_DATA_BYTES_THRESHOLD,
351            read_max_concurrency: DEFAULT_READ_MAX_CONCURRENCY,
352        }
353    }
354}
355
356/// Central config per hyper file
357#[derive(Clone, Default, Debug, PartialEq, Deserialize, Serialize)]
358pub struct HyperFileConfig {
359	/// metadata config for hyper file
360	pub meta: HyperFileMetaConfig,
361	/// staging config for hyper file
362	pub staging: StagingConfig,
363	/// hyper file runtime tunables
364	#[serde(default)]
365	pub runtime: HyperFileRuntimeConfig,
366    #[cfg(feature = "wal")]
367	/// wal config for hyper file
368	pub wal: HyperFileWalConfig,
369    /// data cache config for hyper file
370    pub data_cache: HyperFileDataCacheConfig,
371    /// node cache config for hyper file
372    pub node_cache: HyperFileNodeCacheConfig,
373}
374
375impl HyperFileConfig {
376    pub fn from_json_string(s: &str) -> Result<Self> {
377        serde_json::from_str(s)
378            .map_err(|e| {
379                let err_msg = format!("{}", e);
380                Error::new(ErrorKind::InvalidInput, err_msg)
381            })
382    }
383
384    pub fn to_json_string(&self, pretty: bool) -> String {
385        if pretty {
386            return serde_json::to_string_pretty(self).unwrap();
387        }
388        serde_json::to_string(self).unwrap()
389    }
390}
391
392pub struct HyperFileConfigBuilder {
393	pub(crate) config: HyperFileConfig,
394}
395
396impl HyperFileConfigBuilder {
397	/// Get a new hyper file config builder with all default values,
398	/// by default, it is a read-only hyper file with 4KiB block size.
399	pub fn new() -> Self {
400		let config = HyperFileConfig::default();
401		Self {
402			config
403		}
404	}
405
406	pub fn from(config: &HyperFileConfig) -> Self {
407		Self {
408			config: config.to_owned(),
409		}
410	}
411
412	pub fn with_meta_config(mut self, meta: &HyperFileMetaConfig) -> Self {
413		self.config.meta = meta.to_owned();
414		self
415	}
416
417	pub fn with_staging_config(mut self, staging: &StagingConfig) -> Self {
418		self.config.staging = staging.to_owned();
419		self
420	}
421
422	pub fn with_runtime_config(mut self, runtime: &HyperFileRuntimeConfig) -> Self {
423		self.config.runtime = runtime.to_owned();
424		self
425	}
426
427    #[cfg(feature = "wal")]
428	pub fn with_wal_config(mut self, wal: &HyperFileWalConfig) -> Self {
429		self.config.wal = wal.to_owned();
430		self
431	}
432
433	pub fn with_data_cache_config(mut self, cache: &HyperFileDataCacheConfig) -> Self {
434		self.config.data_cache = cache.to_owned();
435		self
436	}
437
438	pub fn with_node_cache_config(mut self, cache: &HyperFileNodeCacheConfig) -> Self {
439		self.config.node_cache = cache.to_owned();
440		self
441	}
442
443	pub fn build(&self) -> HyperFileConfig {
444		self.config.clone()
445	}
446}
447
448#[cfg(test)]
449mod tests {
450    use super::*;
451
452    /// A runtime config written before the part bound and the object size limit
453    /// existed still reads, and gets their defaults -- in every profile, which each
454    /// set them.
455    #[test]
456    fn runtime_config_without_the_part_bound_reads_with_the_default() {
457        for (name, cfg) in [
458            ("default", HyperFileRuntimeConfig::default()),
459            ("large", HyperFileRuntimeConfig::default_large()),
460            ("middle", HyperFileRuntimeConfig::default_middle()),
461        ] {
462            assert_eq!(cfg.segment_mpu_max_concurrency, DEFAULT_SEGMENT_MPU_MAX_CONCURRENCY, "{}", name);
463            assert_eq!(cfg.segment_max_object_bytes, DEFAULT_SEGMENT_MAX_OBJECT_BYTES, "{}", name);
464            let mut v = serde_json::to_value(&cfg).unwrap();
465            assert!(v.as_object_mut().unwrap().remove("segment_mpu_max_concurrency").is_some(), "{}", name);
466            assert!(v.as_object_mut().unwrap().remove("segment_max_object_bytes").is_some(), "{}", name);
467            let back: HyperFileRuntimeConfig = serde_json::from_value(v).expect(name);
468            assert_eq!(back, cfg, "{}", name);
469        }
470    }
471
472    #[test]
473    fn meta_config_default_round_trip() {
474        let cfg = HyperFileMetaConfig::default();
475        let encoded = cfg.as_u32();
476        let decoded = HyperFileMetaConfig::from_u32(encoded);
477        assert_eq!(cfg, decoded);
478    }
479
480    #[test]
481    fn meta_config_custom_round_trip() {
482        let cfg = HyperFileMetaConfig::new(56, 8192, 65536, BlockPtrFormat::MicroGroup);
483        let encoded = cfg.as_u32();
484        let decoded = HyperFileMetaConfig::from_u32(encoded);
485        assert_eq!(decoded.root_size, 56);
486        assert_eq!(decoded.meta_block_size, 8192);
487        assert_eq!(decoded.data_block_size, 65536);
488        assert_eq!(decoded.block_ptr_format, BlockPtrFormat::MicroGroup);
489    }
490
491    #[test]
492    fn meta_config_enforces_min_sizes() {
493        // Passing values below minimum should be clamped up
494        let cfg = HyperFileMetaConfig::new(56, 1, 1, BlockPtrFormat::Flat);
495        assert!(cfg.meta_block_size >= MIN_META_BLOCK_SIZE);
496        assert!(cfg.data_block_size >= MIN_DATA_BLOCK_SIZE);
497    }
498
499    #[test]
500    fn meta_config_power_of_two_alignment() {
501        // 5000 is not power-of-two; should be rounded down to 4096
502        let cfg = HyperFileMetaConfig::new(56, 5000, 5000, BlockPtrFormat::Flat);
503        assert_eq!(cfg.meta_block_size, 4096);
504        assert_eq!(cfg.data_block_size, 4096);
505    }
506
507    #[test]
508    fn meta_config_json_round_trip() {
509        let cfg = HyperFileConfig::default();
510        let json = cfg.to_json_string(false);
511        let decoded = HyperFileConfig::from_json_string(&json).unwrap();
512        assert_eq!(cfg, decoded);
513    }
514
515    #[test]
516    fn meta_config_json_pretty() {
517        let cfg = HyperFileConfig::default();
518        let json = cfg.to_json_string(true);
519        assert!(json.contains('\n'));
520        let decoded = HyperFileConfig::from_json_string(&json).unwrap();
521        assert_eq!(cfg, decoded);
522    }
523
524    #[test]
525    fn meta_config_from_invalid_json() {
526        let result = HyperFileConfig::from_json_string("not json");
527        assert!(result.is_err());
528    }
529
530    /// A container written before `i_meta_config` was populated carries zero.
531    /// That used to decode without complaint into a root of 0 bytes and
532    /// blocks of 1 byte, and since the container's config overwrites the
533    /// caller's, the 1-byte block size then decided how reads were cut up.
534    #[test]
535    fn meta_config_rejects_an_unpopulated_field() {
536        let e = HyperFileMetaConfig::try_from_u32(0).unwrap_err();
537        assert_eq!(e.kind(), ErrorKind::InvalidData);
538        // Nop is reached first, being the low byte.
539        assert!(format!("{}", e).contains("Nop"), "{}", e);
540    }
541
542    /// Every field is checked, not just the one that happens to be wrong in
543    /// the containers that prompted this.
544    #[test]
545    fn meta_config_rejects_each_field_out_of_range() {
546        let good = HyperFileMetaConfig::default();
547        let ok = good.as_u32();
548        assert_eq!(HyperFileMetaConfig::try_from_u32(ok).unwrap(), good,
549            "a config this crate encoded must decode back");
550
551        // Block ptr format this build does not know.
552        assert!(HyperFileMetaConfig::try_from_u32((ok & !0xFF) | 0x07).is_err());
553
554        // Data block shift below the minimum, and absurdly high. The high
555        // case is the one that would shift by more than a usize has bits.
556        let below = (ok & !0x0000FF00) | (11 << 8);
557        assert!(HyperFileMetaConfig::try_from_u32(below).is_err(), "shift 11 accepted");
558        let absurd = (ok & !0x0000FF00) | (200 << 8);
559        assert!(HyperFileMetaConfig::try_from_u32(absurd).is_err(), "shift 200 accepted");
560
561        // Meta block shift, same two ends.
562        assert!(HyperFileMetaConfig::try_from_u32((ok & !0x00FF0000) | (11 << 16)).is_err());
563        assert!(HyperFileMetaConfig::try_from_u32((ok & !0x00FF0000) | (200 << 16)).is_err());
564
565        // Root size below the minimum.
566        assert!(HyperFileMetaConfig::try_from_u32(ok & !0xFF000000).is_err());
567    }
568
569    /// The sizes hypercli round-trips must stay acceptable, so the guard
570    /// rejects the unrepresentable rather than the merely unusual.
571    #[test]
572    fn meta_config_accepts_the_sizes_in_use() {
573        for data_block_size in [4096usize, 65536, 524288] {
574            let c = HyperFileMetaConfig::new(
575                DEFAULT_ROOT_SIZE, DEFAULT_META_BLOCK_SIZE, data_block_size,
576                DEFAULT_BLOCK_PTR_FORMAT);
577            let back = HyperFileMetaConfig::try_from_u32(c.as_u32())
578                .expect("a config in use must decode");
579            assert_eq!(back.data_block_size, data_block_size);
580            assert_eq!(back, c);
581        }
582    }
583}
hyperfile/config.rs

hyperfile/
config.rs
