1use std::path::Path;
5
6use store_io_buf::BufPool;
7use store_io_core::decide::decide;
8use store_io_core::error::{Error, ErrorContext, Named, Op};
9use store_io_core::id::VolumeId;
10use store_io_format::meta::{VOLUME_META_LEN, VolumeMeta, encode_table};
11use store_io_platform::{FileMode, FileName, IoOp, Platform, Queue};
12
13use crate::exec::{Lane, QueueSet, Ran, run};
14use crate::layout::{Layout, choose_log2_block};
15use crate::slots::{PairRead, plan_next, write_slot};
16use crate::store::*;
17
18impl<P: Platform> Store<P>
19where
20 P::Queue: Send,
21{
22 pub fn create(platform: P, path: &Path, opts: StoreOptions) -> Result<Self, Error> {
36 Self::create_named(platform, path, opts, &container_name()?)
37 }
38
39 pub(crate) fn create_named(
42 platform: P,
43 path: &Path,
44 opts: StoreOptions,
45 name: &store_io_platform::FileName,
46 ) -> Result<Self, Error> {
47 let dir = platform
48 .open_dir(path, true)
49 .map_err(|e| io(Op::Create, e))?;
50 let name = name.clone();
51 let (file, lock) = match platform.create_file(&dir, &name) {
52 Ok(file) => {
53 let lock = platform.lock_exclusive(&file).map_err(|raw| {
54 if store_io_core::errno::is_lock_held(raw) {
55 Error::Locked
56 } else {
57 io(Op::Lock, raw)
58 }
59 })?;
60 (file, lock)
61 }
62 Err(e) if store_io_core::errno::is_already_exists(e) => {
63 Self::reclaim_abandoned(&platform, &dir, &name, &opts)?
64 }
65 Err(e) => return Err(io(Op::Create, e)),
66 };
67 let built = Self::build_new(&platform, &dir, &file, &opts);
68 match built {
69 Ok((meta, report, layout, policy, pool, queues)) => {
70 if let Err(e) = platform.sync_dir(&dir) {
71 return Err(Error::DurabilityUnknown {
72 op: Op::SyncDir,
73 raw: Some(e),
74 ctx: ErrorContext::default(),
75 });
76 }
77 let volume = report.volume;
78 Ok(Self::assemble(
79 platform,
80 dir,
81 file,
82 lock,
83 volume,
84 layout,
85 policy,
86 pool,
87 queues,
88 meta,
89 Vec::new(),
90 report,
91 false,
92 1,
93 opts,
94 ))
95 }
96 Err(e) => {
97 drop(lock);
98 drop(file);
99 let _removed = platform.unlink(&dir, &name);
100 let _synced = platform.sync_dir(&dir);
101 Err(e)
102 }
103 }
104 }
105
106 fn reclaim_abandoned(
120 platform: &P,
121 dir: &P::Dir,
122 name: &FileName,
123 opts: &StoreOptions,
124 ) -> Result<(P::File, P::Lock), Error> {
125 let exists = || Error::AlreadyExists { what: Named::Store };
126 let file = platform
127 .open_file(dir, name, FileMode::ReadWrite)
128 .map_err(|_| exists())?;
129 let lock = platform.lock_exclusive(&file).map_err(|raw| {
132 if store_io_core::errno::is_lock_held(raw) {
133 exists()
134 } else {
135 io(Op::Lock, raw)
136 }
137 })?;
138 let v2 = opts.container_format == store_io_format::slot::FORMAT_2;
143 let v1_metadata = 4u64 << crate::layout::MAX_LOG2_BLOCK;
144 let bound = if v2 {
145 (crate::layout::MIN_LOG2_BLOCK..=crate::layout::MAX_LOG2_BLOCK)
146 .map(|b| {
147 let t = opts.log2_table_slot.clamp(b, 16);
148 crate::v2::layout_for(b, t, opts.region_capacity).data_start()
149 })
150 .max()
151 .unwrap_or(v1_metadata)
152 } else {
153 v1_metadata
154 };
155 if platform.size(&file).map_err(|e| io(Op::Open, e))? > bound {
156 return Err(exists());
157 }
158 let evidence = platform.probe(dir, &file).map_err(|e| io(Op::Probe, e))?;
159 let largest_metadata = if v2 {
160 let log2_block = choose_log2_block(&evidence, opts.block_size);
161 let log2_table = opts.log2_table_slot.clamp(log2_block, 16);
162 crate::v2::layout_for(log2_block, log2_table, opts.region_capacity).data_start()
163 } else {
164 v1_metadata
165 };
166 if platform.size(&file).map_err(|e| io(Op::Open, e))? > largest_metadata {
167 return Err(exists());
168 }
169 let boot_pool = make_pool(
170 &evidence,
171 &Layout::v1(crate::layout::MAX_LOG2_BLOCK, crate::layout::MAX_LOG2_BLOCK),
172 opts,
173 false,
174 )?;
175 let queues = make_queues(platform, opts, 1)?;
176 let len = platform.size(&file).map_err(|e| io(Op::Open, e))?;
177 let mut leftovers = false;
178 let holds = {
179 let mut q = queues.get(0);
180 holds_slot_header(
181 &mut q,
182 &file,
183 &boot_pool,
184 len,
185 v2.then_some(opts.log2_table_slot),
186 &mut leftovers,
187 )
188 };
189 if holds != Some(false) || (len > v1_metadata && !leftovers) {
193 return Err(exists());
194 }
195 platform.set_len(&file, 0).map_err(|e| io(Op::Create, e))?;
196 Ok((file, lock))
197 }
198
199 #[allow(clippy::type_complexity)]
200 fn build_new(
201 platform: &P,
202 dir: &P::Dir,
203 file: &P::File,
204 opts: &StoreOptions,
205 ) -> Result<
206 (
207 Meta,
208 StoreReport,
209 Layout,
210 Policy,
211 BufPool,
212 QueueSet<P::Queue>,
213 ),
214 Error,
215 > {
216 let evidence = platform.probe(dir, file).map_err(|e| io(Op::Probe, e))?;
217 let decision = decide(&evidence, &opts.trust);
218 refuse(&decision)?;
219 let log2_block = choose_log2_block(&evidence, opts.block_size);
220 let log2_table = opts.log2_table_slot.clamp(log2_block, 16);
221 let v2 = opts.container_format == store_io_format::slot::FORMAT_2;
222 let layout = if v2 {
223 crate::v2::layout_for(log2_block, log2_table, opts.region_capacity)
224 } else {
225 Layout::v1(log2_block, log2_table)
226 };
227 let policy = policy_of(&decision);
228 let pool = make_pool(&evidence, &layout, opts, true)?;
229 let lanes = if policy.flush {
230 flush_slots(platform, opts)
231 } else {
232 1
233 };
234 let queues = make_queues(platform, opts, lanes)?;
235 let volume = VolumeId::from_bytes(random_uuid());
236 let created = std::time::SystemTime::now()
237 .duration_since(std::time::UNIX_EPOCH)
238 .map_or(0, |d| d.as_secs());
239 if v2 {
240 let meta = {
241 let mut q = queues.get(0);
242 crate::v2::create_metadata(platform, file, &mut q, &pool, &layout, volume, created)?
243 };
244 let report = StoreReport {
245 evidence,
246 decision,
247 block_size: layout.block(),
248 table_capacity: layout.pages as usize
249 * usize::from(store_io_format::meta2::entries_per_page(layout.log2_table)),
250 volume,
251 owner_generation: 1,
252 read_only: false,
253 io: platform.io_path(),
254 unavailable: Vec::new(),
255 format: store_io_format::slot::FORMAT_2,
256 metadata: MetadataReport::default(),
257 flushes_in_flight: 1,
258 };
259 return Ok((meta, report, layout, policy, pool, queues));
260 }
261 let header_len = layout.data_start();
262 platform
263 .allocate(file, header_len)
264 .map_err(|e| space_err(Op::Allocate, e, 0))?;
265 let vmeta = VolumeMeta {
266 log2_block_size: layout.log2_block,
267 log2_table_slot_size: layout.log2_table,
268 device_index: 0,
269 owner_generation: 1,
270 table_offset: layout.table_slots().0,
271 created_unix_s: created,
272 features: 0,
273 };
274 let mut vbuf = [0u8; VOLUME_META_LEN];
275 let _len = vmeta.encode(&mut vbuf).map_err(|_| meta_err())?;
276 let mut tbuf = vec![0u8; layout.table() as usize];
277 let tlen = encode_table(&mut tbuf, 0, &[]).map_err(|_| meta_err())?;
278 let vat = pair_volume(volume, &layout);
279 let tat = pair_table(volume, &layout);
280 let empty = PairRead::blank();
281 let mut q = queues.get(0);
282 let next_v = plan_next(&empty, &vbuf, 1)?;
283 let volume_slot = write_slot(&mut q, file, &pool, &vat, &next_v, false)?;
284 let next_t = plan_next(&empty, &tbuf[..tlen], 1)?;
285 let table_slot = write_slot(&mut q, file, &pool, &tat, &next_t, false)?;
286 drop(q);
287 platform
288 .flush_all(file)
289 .map_err(|e| Error::DurabilityUnknown {
290 op: Op::FlushAll,
291 raw: Some(e),
292 ctx: ErrorContext::default(),
293 })?;
294 let volume_pair = empty.after(volume_slot);
296 let table_pair = empty.after(table_slot);
297 let report = StoreReport {
298 evidence,
299 decision,
300 block_size: layout.block(),
301 table_capacity: layout.table_capacity(),
302 volume,
303 owner_generation: 1,
304 read_only: false,
305 io: platform.io_path(),
306 unavailable: Vec::new(),
307 format: store_io_format::slot::FORMAT_1,
308 metadata: MetadataReport::default(),
309 flushes_in_flight: 1,
310 };
311 let meta = Meta {
312 volume: vmeta,
313 volume_pair,
314 table_pair,
315 next_region_id: 0,
316 entries: Vec::new(),
317 container_len: header_len,
318 v2: None,
319 unavailable: Vec::new(),
320 repairs: Vec::new(),
321 };
322 Ok((meta, report, layout, policy, pool, queues))
323 }
324}
325
326fn holds_slot_header<Q: Queue>(
342 q: &mut Lane<Q>,
343 file: &Q::File,
344 pool: &BufPool,
345 len: u64,
346 v2: Option<u8>,
347 leftovers: &mut bool,
348) -> Option<bool> {
349 use store_io_format::slot::{Expect, SlotIndex, SlotStatus, validate};
350 const SCAN: u64 = 4096;
351 let mut at = 0u64;
352 while at < len {
353 let want = SCAN.min(len - at) as usize;
354 let buf = pool.take(SCAN as usize).ok()?;
356 let Ran::Done { buf, result } = run(
357 q,
358 IoOp::Read {
359 file,
360 offset: at,
361 buf,
362 },
363 )
364 .ok()?
365 else {
366 return None;
367 };
368 let mut b = buf?;
369 if result.ok()? != want {
370 return None;
371 }
372 b.as_mut_slice()[want..].fill(0);
373 let expect = Expect {
374 volume_uuid: [0; 16],
375 object_id: 0,
376 slot_offset: at,
377 slot_index: SlotIndex::A,
378 log2_slot_size: 12,
379 format_major: store_io_format::slot::FORMAT_1,
380 numbered: false,
381 };
382 match validate(b.as_slice(), &expect) {
383 SlotStatus::Blank
384 | SlotStatus::BadMagic
385 | SlotStatus::BadHeaderCrc
386 | SlotStatus::BadLength
387 | SlotStatus::ReadError => {}
388 _ if v2.is_some_and(|t| crashed_create_leftover(b.as_slice(), at, t)) => {
389 *leftovers = true;
390 }
391 _ => return Some(true),
392 }
393 at += SCAN;
394 }
395 Some(false)
396}
397
398fn crashed_create_leftover(b: &[u8], at: u64, log2_table_slot: u8) -> bool {
406 let u16_at = |i: usize| u16::from_le_bytes([b[i], b[i + 1]]);
407 let u32_at = |i: usize| u32::from_le_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]]);
408 let u64_at = |i: usize| {
409 let mut x = [0u8; 8];
410 x.copy_from_slice(&b[i..i + 8]);
411 u64::from_le_bytes(x)
412 };
413 let header_len = usize::from(u16_at(12));
414 if u16_at(14) != store_io_format::slot::FORMAT_2
417 || u64_at(56) != 1
418 || header_len != 128
419 || u64_at(64) != at
420 || b[76] != 0
421 {
422 return false;
423 }
424 let object = u64_at(48);
425 let payload_len = u32_at(72) as usize;
426 let Some(payload) = header_len
427 .checked_add(payload_len)
428 .and_then(|end| b.get(header_len..end))
429 else {
430 return false;
431 };
432 let torn = store_io_format::crc32c::crc32c(payload) != u32_at(28);
433 if object == 0 {
434 return torn && at == 0;
436 }
437 let page = object & !(1u64 << 61);
438 if object >> 61 != 1 || page >= u64::from(store_io_format::meta2::MAX_TABLE_PAGES) {
439 return false;
440 }
441 let placed = (crate::layout::MIN_LOG2_BLOCK..=crate::layout::MAX_LOG2_BLOCK).any(|lb| {
443 let t = 1u64 << log2_table_slot.clamp(lb, 16);
444 at == (2u64 << lb) + 2 * page * t
445 });
446 placed
447 && (torn
448 || store_io_format::meta2::decode_page(payload, page as u32, u16::MAX)
449 .is_ok_and(|p| p.header.count == 0))
450}