1use std::path::Path;
5use std::sync::{Arc, Mutex};
6
7use store_io_buf::BufPool;
8use store_io_core::decide::decide;
9use store_io_core::error::{Error, ErrorContext, Named, NotWrittenCause, Op, OsError};
10use store_io_core::id::{Generation, RegionId, VolumeId};
11use store_io_core::observe::Observer;
12use store_io_format::meta::{
13 RegionKind, RegionMeta, RegionState, TableEntry, VolumeMeta, decode_table,
14};
15use store_io_format::pair::Winner;
16use store_io_platform::{FileMode, IoOp, Platform, Queue, QueueConfig};
17
18use crate::domain::Domain;
19use crate::exec::{Lane, QueueSet, Ran, run};
20use crate::frontier::AppendFrontier;
21use crate::gate::Gate;
22use crate::layout::Layout;
23use crate::slots::{PairRead, read_pair};
24use crate::space::Accounting;
25use crate::store::*;
26
27impl<P: Platform> Store<P>
28where
29 P::Queue: Send,
30{
31 pub fn open(platform: P, path: &Path, opts: StoreOptions) -> Result<Self, Error> {
52 Self::open_inner(platform, path, opts, false)
53 }
54
55 pub fn open_readonly(platform: P, path: &Path, opts: StoreOptions) -> Result<Self, Error> {
62 Self::open_inner(platform, path, opts, true)
63 }
64
65 fn open_inner(
66 platform: P,
67 path: &Path,
68 opts: StoreOptions,
69 read_only: bool,
70 ) -> Result<Self, Error> {
71 let dir = platform
72 .open_dir(path, false)
73 .map_err(|e| not_found_or(Op::Open, e, Named::Store))?;
74 let name = container_name()?;
75 let mode = if read_only {
79 FileMode::ReadOnly
80 } else {
81 FileMode::ReadWrite
82 };
83 let file = platform
84 .open_file(&dir, &name, mode)
85 .map_err(|e| not_found_or(Op::Open, e, Named::Store))?;
86 let lock = if read_only {
87 platform.lock_shared(&file)
88 } else {
89 platform.lock_exclusive(&file)
90 }
91 .map_err(|raw| {
92 if store_io_core::errno::is_lock_held(raw) {
95 Error::Locked
96 } else {
97 io(Op::Lock, raw)
98 }
99 })?;
100 let evidence = platform.probe(&dir, &file).map_err(|e| {
104 if read_only {
105 io(Op::Probe, e)
106 } else {
107 Error::DurabilityUnknown {
108 op: Op::Probe,
109 raw: Some(e),
110 ctx: ErrorContext::default(),
111 }
112 }
113 })?;
114 let decision = decide(&evidence, &opts.trust);
115 if !read_only {
116 refuse(&decision)?;
117 platform
121 .flush_all(&file)
122 .map_err(|e| Error::DurabilityUnknown {
123 op: Op::FlushAll,
124 raw: Some(e),
125 ctx: ErrorContext::default(),
126 })?;
127 }
128 let policy = policy_of(&decision);
129 let lanes = if read_only || !policy.flush {
130 1
131 } else {
132 flush_slots(&platform, &opts)
133 };
134 let queues = make_queues(&platform, &opts, lanes)?;
135 let container_len = platform.size(&file).map_err(|e| io(Op::Open, e))?;
136 let boot_pool = make_pool(&evidence, &Layout::v1(16, 16), &opts, false)?;
139 let (volume, layout, found, volume_pair) = {
140 let mut q = queues.get(0);
141 find_volume(&mut q, &file, &boot_pool)?
142 };
143 let pool = make_pool(&evidence, &layout, &opts, true)?;
144 let vmeta = match found {
145 Found::V1(_) if !read_only => {
148 return Err(Error::Unsupported {
149 what: store_io_core::error::Capability::FormatVersion,
150 });
151 }
152 Found::V1(vm) => vm,
153 Found::V2(vm) => {
154 let opened = {
155 let mut q = queues.get(0);
156 crate::v2::open_metadata::<P>(
157 &file,
158 &mut q,
159 &pool,
160 &layout,
161 volume,
162 vm,
163 volume_pair,
164 container_len,
165 read_only,
166 &policy,
167 opts.append_window,
168 &mut || {
169 let cfg = QueueConfig {
170 depth: opts.queue_depth.max(1),
171 parallel: true,
172 pool_threads: opts.t3_pool_threads,
173 ..QueueConfig::default()
174 };
175 platform.queue(cfg).ok().map(|q| (q, cfg.depth as usize))
176 },
177 )?
178 };
179 let report = StoreReport {
180 evidence,
181 decision,
182 block_size: layout.block(),
183 table_capacity: layout.pages as usize
184 * usize::from(store_io_format::meta2::entries_per_page(layout.log2_table)),
185 volume,
186 owner_generation: opened.owner_generation,
187 read_only: opened.read_only,
188 io: platform.io_path(),
189 unavailable: opened.unavailable,
190 format: store_io_format::slot::FORMAT_2,
191 metadata: opened.report,
192 flushes_in_flight: 1,
193 };
194 return Ok(Self::assemble(
195 platform,
196 dir,
197 file,
198 lock,
199 volume,
200 layout,
201 policy,
202 pool,
203 queues,
204 opened.meta,
205 opened.regions,
206 report,
207 opened.read_only,
208 opened.owner_generation,
209 opts,
210 ));
211 }
212 };
213 let tat = pair_table(volume, &layout);
214 let mut q = queues.get(0);
215 let table_pair = read_pair(&mut q, &file, &pool, &tat)?;
216 let table_bytes = winner_payload(&table_pair)?;
217 let table = decode_table(&table_bytes, layout.block()).map_err(|_| corrupt())?;
218 let next_region_id = table.next_region_id;
219 let entries: Vec<TableEntry> = table.iter().collect();
220 let mut regions = Vec::new();
221 let mut unavailable = Vec::new();
224 for e in entries.iter().filter(|e| e.state == RegionState::Ready) {
225 let mut skip = |reason| {
226 unavailable.push(UnavailableRegion {
227 name: e.name.as_str().to_owned(),
228 id: RegionId::from_raw(u64::from(e.region_id)),
229 kind: e.kind,
230 reason,
231 });
232 };
233 let end = layout
234 .extent(e.data_size)
235 .and_then(|l| e.offset.checked_add(l))
236 .ok_or_else(corrupt)?;
237 if end > container_len {
238 skip(Unavailable::ContainerShort {
240 needed: end,
241 actual: container_len,
242 });
243 continue;
244 }
245 let at = pair_region(volume, &layout, e);
246 let header = read_pair(&mut q, &file, &pool, &at)?;
247 let (Some(bytes), Some((_, info))) =
248 (header.payload.clone(), header.report.winner_info())
249 else {
250 skip(Unavailable::HeaderUnreadable);
251 continue;
252 };
253 let Ok(rm) = RegionMeta::decode(&bytes) else {
254 skip(Unavailable::HeaderInvalid);
255 continue;
256 };
257 if rm.region_id != e.region_id || rm.kind != e.kind {
258 skip(Unavailable::HeaderMismatch);
259 continue;
260 }
261 if rm.state != RegionState::Ready {
262 skip(Unavailable::HeaderReleased);
263 continue;
264 }
265 let Some(generation) = Generation::from_raw(info.generation) else {
266 skip(Unavailable::HeaderInvalid);
267 continue;
268 };
269 let frontier = (e.kind == RegionKind::Append).then(|| {
270 AppendFrontier::new(
271 0,
272 e.data_size,
273 u32::from(layout.log2_block),
274 crate::frontier::ring_for(
275 opts.append_window,
276 e.data_size,
277 u32::from(layout.log2_block),
278 ),
279 )
280 });
281 regions.push(Arc::new(LiveRegion {
282 id: RegionId::from_raw(u64::from(e.region_id)),
283 name: e.name,
284 kind: e.kind,
285 generation,
286 map: crate::regionmap::RegionMap::single(
287 e.offset + 2 * layout.block(),
288 e.data_size,
289 ),
290 header_at: [e.offset, e.offset + layout.block()],
291 header: Mutex::new(header),
292 frontier,
293 positioned: std::sync::atomic::AtomicBool::new(false),
294 force_sync: Arc::new(std::sync::atomic::AtomicBool::new(false)),
295 read_only: std::sync::atomic::AtomicBool::new(false),
296 superseded: false,
297 foreign: false,
298 gate: Gate::new(),
299 }));
300 }
301 drop(q);
302 let meta = Meta {
303 volume: vmeta,
304 volume_pair,
305 table_pair,
306 next_region_id,
307 entries,
308 container_len,
309 v2: None,
310 unavailable: Vec::new(),
311 repairs: Vec::new(),
312 };
313 debug_assert!(read_only, "a writable format-1 open was refused");
316 let owner_generation = 0;
317 let report = StoreReport {
318 evidence,
319 decision,
320 block_size: layout.block(),
321 table_capacity: layout.table_capacity(),
322 volume,
323 owner_generation,
324 read_only,
325 io: platform.io_path(),
326 unavailable,
327 format: store_io_format::slot::FORMAT_1,
328 metadata: MetadataReport::default(),
329 flushes_in_flight: 1,
330 };
331 Ok(Self::assemble(
332 platform,
333 dir,
334 file,
335 lock,
336 volume,
337 layout,
338 policy,
339 pool,
340 queues,
341 meta,
342 regions,
343 report,
344 read_only,
345 owner_generation,
346 opts,
347 ))
348 }
349
350 #[allow(clippy::too_many_arguments)]
351 pub(crate) fn assemble(
352 platform: P,
353 dir: P::Dir,
354 file: P::File,
355 lock: P::Lock,
356 volume: VolumeId,
357 layout: Layout,
358 policy: Policy,
359 pool: BufPool,
360 mut queues: QueueSet<P::Queue>,
361 meta: Meta,
362 regions: Vec<Arc<LiveRegion>>,
363 report: StoreReport,
364 read_only: bool,
365 owner_generation: u64,
366 opts: StoreOptions,
367 ) -> Self {
368 let acct = match meta.v2.as_deref() {
369 Some(v2) => Accounting::from_usage(&v2.usage),
370 None => Accounting::from_entries(&meta.entries, layout.block()),
371 };
372 let lanes = if read_only || !policy.flush {
378 1
379 } else {
380 queues.flush_lanes()
381 };
382 let flush_files = flush_handles(&platform, &file, lanes);
383 queues.keep_flush_lanes(1 + flush_files.len());
384 let mut report = report;
385 report.flushes_in_flight = 1 + flush_files.len() as u8;
386 let mut meta = meta;
387 meta.unavailable = report.unavailable.clone();
388 let frozen = std::sync::atomic::AtomicBool::new(report.metadata.frozen);
389 let block = layout.block() as usize;
390 let ordinary = crate::store::io_max(&opts).max(pool.min_len());
391 let chunk = (ordinary / block * block).max(block);
392 Self {
393 inner: Arc::new(Inner {
394 domain: Domain::with_slots(1 + flush_files.len()),
395 flush_files: flush_files.into_boxed_slice(),
396 platform,
397 _dir: dir,
398 file,
399 _lock: lock,
400 volume,
401 layout,
402 policy,
403 pool,
404 queues,
405 meta: Mutex::new(meta),
406 regions: Mutex::new(regions),
407 report,
408 frozen,
409 read_only,
410 owner_generation,
411 opts,
412 ordinary,
413 chunk,
414 acct: Mutex::new(acct),
415 epoch: EPOCH.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
416 observer: Observer::new(),
417 seen: Mutex::new(crate::stats::Seen::default()),
418 submitters: Mutex::new(crate::stats::Submitters::default()),
419 flusher: std::sync::OnceLock::new(),
420 }),
421 }
422 }
423}
424
425fn flush_handles<P: Platform>(platform: &P, file: &P::File, lanes: usize) -> Vec<P::File> {
432 (1..lanes).map_while(|_| platform.reopen(file)).collect()
433}
434
435enum Found {
437 V1(VolumeMeta),
438 V2(store_io_format::meta2::VolumeMeta2),
439}
440
441fn find_volume<Q: Queue>(
444 q: &mut Lane<Q>,
445 file: &Q::File,
446 pool: &BufPool,
447) -> Result<(VolumeId, Layout, Found, PairRead), Error> {
448 let mut media_error = false;
451 for log2 in crate::layout::MIN_LOG2_BLOCK..=crate::layout::MAX_LOG2_BLOCK {
452 let size = 1u64 << log2;
453 for probe_at in [0, size] {
454 let buf = pool.take(4096).map_err(|_| Error::NotWritten {
456 cause: NotWrittenCause::PoolExhausted,
457 ctx: ErrorContext::default(),
458 })?;
459 let io = |raw| Error::Io {
464 op: Op::Read,
465 raw,
466 ctx: ErrorContext::default(),
467 };
468 let (got, result) = match run(
469 q,
470 IoOp::Read {
471 file,
472 offset: probe_at,
473 buf,
474 },
475 ) {
476 Ok(Ran::Done { buf, result }) => (buf, result),
477 Ok(Ran::Abandoned(raw)) | Err((_, Some(raw))) => return Err(io(raw)),
479 Err((_, None)) => {
480 return Err(Error::NotWritten {
481 cause: NotWrittenCause::QueueFull,
482 ctx: ErrorContext::default(),
483 });
484 }
485 };
486 match result {
487 Err(raw) if !crate::slots::unreadable(raw) => return Err(io(raw)),
488 Ok(4096) => {}
489 Err(raw) => {
490 media_error |= store_io_core::errno::is_media_error(raw);
491 continue;
492 }
493 Ok(_) => continue,
494 }
495 let Some(b) = got else { continue };
496 if b.as_slice()[..8] != store_io_format::slot::MAGIC {
497 continue;
498 }
499 let mut uuid = [0u8; 16];
500 uuid.copy_from_slice(&b.as_slice()[32..48]);
501 if b.as_slice()[78] != log2 {
502 continue;
503 }
504 if !header_verifies(b.as_slice()) {
507 continue;
508 }
509 let format = u16::from_le_bytes([b.as_slice()[14], b.as_slice()[15]]);
510 let v2 = format == store_io_format::slot::FORMAT_2;
511 let layout0 = Layout {
512 format: if v2 {
513 format
514 } else {
515 store_io_format::slot::FORMAT_1
516 },
517 ..Layout::v1(log2, log2)
518 };
519 let at = pair_volume(VolumeId::from_bytes(uuid), &layout0);
520 let pair = read_pair(q, file, pool, &at)?;
521 media_error |= pair.media_error;
522 let bytes = match winner_payload(&pair) {
523 Ok(b) => b,
524 Err(Error::Unsupported { what }) => return Err(Error::Unsupported { what }),
525 Err(_) => continue,
526 };
527 let newer = Error::Unsupported {
531 what: store_io_core::error::Capability::FormatVersion,
532 };
533 if v2 {
534 let vm = crate::v2::decode_volume(&bytes)?;
535 if vm.base.log2_block_size != log2 {
536 continue;
537 }
538 let layout = Layout {
539 log2_block: log2,
540 log2_table: vm.base.log2_table_slot_size,
541 format,
542 pages: vm.table_pages,
543 };
544 return Ok((VolumeId::from_bytes(uuid), layout, Found::V2(vm), pair));
545 }
546 if bytes.len() > store_io_format::meta::VOLUME_META_LEN {
547 return Err(newer);
548 }
549 let vm = VolumeMeta::decode(&bytes).map_err(|e| match e {
550 store_io_format::meta::MetaError::Field("features") => newer,
551 _ => corrupt(),
552 })?;
553 if vm.log2_block_size != log2 {
554 continue;
555 }
556 let layout = Layout::v1(log2, vm.log2_table_slot_size);
557 if vm.table_offset != layout.table_slots().0 {
558 return Err(corrupt());
559 }
560 return Ok((VolumeId::from_bytes(uuid), layout, Found::V1(vm), pair));
561 }
562 }
563 Err(Error::Corruption {
569 kind: if media_error {
570 store_io_core::error::CorruptionKind::MediaError
571 } else {
572 store_io_core::error::CorruptionKind::HeaderCrc
573 },
574 ctx: ErrorContext::default(),
575 })
576}
577
578static EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
580
581fn not_found_or(op: Op, raw: OsError, what: Named) -> Error {
582 if store_io_core::errno::is_not_found(raw) {
583 Error::NotFound { what }
584 } else {
585 io(op, raw)
586 }
587}
588
589pub(crate) fn winner_payload(p: &PairRead) -> Result<Vec<u8>, Error> {
590 match p.report.winner {
591 Winner::Confirmed(_) | Winner::Unconfirmed(_) => p.payload.clone().ok_or_else(corrupt),
592 Winner::Refused => Err(Error::Unsupported {
593 what: store_io_core::error::Capability::FormatVersion,
594 }),
595 Winner::Fork => Err(Error::Corruption {
596 kind: store_io_core::error::CorruptionKind::Fork,
597 ctx: ErrorContext::default(),
598 }),
599 Winner::Lost if p.media_error => Err(Error::Corruption {
603 kind: store_io_core::error::CorruptionKind::MediaError,
604 ctx: ErrorContext::default(),
605 }),
606 Winner::Empty | Winner::Lost => Err(Error::Corruption {
607 kind: store_io_core::error::CorruptionKind::HeaderCrc,
608 ctx: ErrorContext::default(),
609 }),
610 }
611}
612
613fn header_verifies(b: &[u8]) -> bool {
617 let len = usize::from(u16::from_le_bytes([b[12], b[13]]));
618 if len > b.len() {
619 return u16::from_le_bytes([b[14], b[15]]) != store_io_format::slot::FORMAT_2;
620 }
621 let Some(h) = b.get(..len.max(store_io_format::slot::HEADER_LEN)) else {
622 return false;
623 };
624 let stored = u32::from_le_bytes([h[8], h[9], h[10], h[11]]);
625 let mut h = h.to_vec();
626 h[8..12].fill(0);
627 store_io_format::crc32c::crc32c(&h) == stored
628}