1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
//! Releasing a region in container format 2 (spec §12): with the region
//! drained and its lock held (the header mutex, before the metadata lock,
//! D-113), and the metadata lock held throughout, the region's page is
//! written without its entry (the commit; every other entry byte for byte,
//! `next_seq` never reused), then its header says Released, its data
//! extents are deallocated, and free space is the complement again.
use std::sync::{Arc, PoisonError};
use store_io_core::error::{Error, NotWrittenCause, Op};
use store_io_core::id::Generation;
use store_io_format::judge::Standing;
use store_io_format::meta::RegionState;
use store_io_format::meta2::{
PAGE_HEADER_LEN, PageHeader, decode_region_header, encode_page, encode_region_header,
};
use store_io_format::slot::FORMAT_2;
use store_io_platform::Platform;
use crate::gate::Retire;
use crate::io::{ctx, not_written};
use crate::slots::{PairAt, PairRead, plan_next};
use crate::store::*;
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
/// Releases a format-2 region. See [`Store::release_region`].
pub(crate) fn release_region2(&self, r: &Arc<LiveRegion>) -> Result<(), Error> {
let inner = &self.inner;
let c = ctx(r);
if inner.frozen.load(std::sync::atomic::Ordering::Acquire) {
return Err(not_written(NotWrittenCause::Frozen, c));
}
// Drained before any lock (a `Submitter` operation holds its pass
// until its thread processes the completion; see format 1).
self.retire(r, Retire::Release)?;
// The region lock, then the metadata lock, both held throughout.
let mut header = r.header.lock().unwrap_or_else(PoisonError::into_inner);
let mut meta = lock_meta(&inner.meta);
let id = r.id.get();
let seq = (id & 0xFFFF_FFFF) as u32;
// Every return before the commit gives the region back.
let refused = |e: Error| {
r.gate.restore();
e
};
let p = usize::try_from(id >> 32).map_err(|_| refused(corrupt()))?;
// A region an overlap or a name demoted rests on this one: released,
// the next open would serve it (spec §9 step 4).
if let Some(d) = meta
.v2
.as_deref()
.and_then(|v2| crate::repair::would_lift(self, v2, id))
{
return Err(refused(not_written(
NotWrittenCause::Overlap,
store_io_core::error::ErrorContext {
region: Some(d),
..store_io_core::error::ErrorContext::default()
},
)));
}
// 1. The commit: the page without the entry.
let (payload, removed) = {
let v2 = meta.v2.as_deref().ok_or_else(|| refused(corrupt()))?;
let page = v2.pages.get(p).ok_or_else(|| refused(corrupt()))?;
let ph = page.header.ok_or_else(|| refused(corrupt()))?;
// A page holding an entry past its own write is written only to
// remove that entry (`Page2::ahead`).
if page.standing != Standing::Confirmed || page.ahead().next().is_some() {
return Err(refused(not_written(NotWrittenCause::Frozen, c)));
}
let Some(at) = page.entries.iter().position(|e| e.seq == seq) else {
return Err(refused(corrupt()));
};
let mut raws = page.entries.clone();
let removed = raws.remove(at);
let next = PageHeader {
page_index: ph.page_index,
// Never decreases: the id is never handed out again.
next_seq: ph.next_seq,
count: raws.len() as u16,
};
let mut buf = vec![0u8; PAGE_HEADER_LEN + 96 * raws.len()];
let n = encode_page(&mut buf, &next, &raws).map_err(|_| refused(meta_err()))?;
buf.truncate(n);
(buf, removed)
};
let meta_seq = crate::v2::next_meta_seq_of(&meta).map_err(refused)?;
{
let mut q = inner.queues.get(0);
if let Err(e) = crate::v2::write_page::<P>(
&mut q,
&inner.file,
&inner.pool,
&inner.layout,
inner.volume,
&inner.policy,
&inner.domain,
&mut meta,
p,
&payload,
meta_seq,
inner.owner_generation,
) {
// Not submitted: nothing changed, the region serves again.
// Submitted: the domain is poisoned (durability unknown).
if inner.domain.poisoned().is_none() {
r.gate.restore();
}
self.wake_frontiers();
return Err(e);
}
}
// Committed: memory follows, before the header and the space.
let block = inner.layout.block();
let committed = usize::from(removed.extent_count);
let extents = committed_extents(&header, inner.layout.log2_block, committed);
let mut footprint = vec![(removed.header_a, block), (removed.header_b, block)];
footprint.extend(extents.iter().copied());
if let Some(v2) = meta.v2.as_deref_mut() {
for range in &footprint {
if let Some(i) = v2.allocated.iter().position(|a| a == range) {
let _gone = v2.allocated.swap_remove(i);
}
}
let physical = removed.capacity.saturating_add(2 * block);
if let Some(i) = v2
.usage
.iter()
.position(|u| *u == (removed.tag, removed.data_size, physical))
{
let _gone = v2.usage.swap_remove(i);
}
}
crate::space::lock_acct(&inner.acct).remove_usage(
removed.tag,
removed.data_size,
removed.capacity.saturating_add(2 * block),
);
inner
.regions
.lock()
.unwrap_or_else(PoisonError::into_inner)
.retain(|x| x.id != r.id);
// Free space: the complement, recomputed now (the metadata lock,
// held until return, keeps it from reuse until the space is given
// back below).
let len = meta.container_len;
if let Some(v2) = meta.v2.as_deref_mut() {
v2.rebuild_free(len);
}
// 2. The header says Released: should the commit's page ever be
// rolled back (its newer copy found damaged), the region is then
// unavailable, not served. A header write that failed before it
// was submitted (refused, a lane that could not be cleared, a
// header pair that would not encode or plan) leaves the data in
// place instead (that rollback would find the region whole); one
// that failed after submission has poisoned the store, and fails
// the call. Either way the release is committed.
match self.rewrite_header2(r, &mut header, RegionState::Released, 0, committed, None) {
Ok(_) => {}
Err(_) if inner.domain.poisoned().is_none() => return Ok(()),
Err(e) => return Err(e),
}
// 3. The data extents, as format 1 does: every one tried.
self.release_extents(extents.iter().copied(), c)
}
/// Recycles a format-2 region's header (spec §12): one header write
/// under the region lock, Ready, its region generation one more, its
/// entry's extents (never an extra one) and everything else kept.
/// Returns the new region generation.
pub(crate) fn recycle_header2(&self, r: &LiveRegion) -> Result<Generation, Error> {
let mut header = r.header.lock().unwrap_or_else(PoisonError::into_inner);
// The entry's extent count, read under the metadata lock (taken
// after the region lock, D-113).
let committed = {
let meta = lock_meta(&self.inner.meta);
let id = r.id.get();
let p = usize::try_from(id >> 32).map_err(|_| corrupt())?;
let seq = (id & 0xFFFF_FFFF) as u32;
meta.v2
.as_deref()
.and_then(|v| v.pages.get(p))
.and_then(|page| page.entries.iter().find(|e| e.seq == seq))
.map(|e| usize::from(e.extent_count))
.ok_or_else(corrupt)?
};
self.rewrite_header2(r, &mut header, RegionState::Ready, 1, committed, None)
}
/// Writes the region's header again with `state` and its region
/// generation moved on by `generation_step`: its first `committed`
/// extents (the entry's count, never an extra one a header may list),
/// then `append` (a growth beyond capacity: the header is one extent
/// ahead of its entry until the page commits), its fill and key, its
/// may-ignore flags and its object as they were. The caller holds the
/// region lock (`header`).
pub(crate) fn rewrite_header2(
&self,
r: &LiveRegion,
header: &mut PairRead,
state: RegionState,
generation_step: u64,
committed: usize,
append: Option<(u64, u64)>,
) -> Result<Generation, Error> {
let inner = &self.inner;
let c = ctx(r);
let metadata = || Error::Corruption {
kind: store_io_core::error::CorruptionKind::Metadata,
ctx: c,
};
let bytes = header.payload.as_deref().ok_or_else(metadata)?;
let view = decode_region_header(bytes, inner.layout.log2_block, u64::MAX)
.map_err(|_| metadata())?;
let mut extents: Vec<(u64, u64)> = view.extents().take(committed).collect();
extents.extend(append);
let mut h = view.header;
h.state = state;
h.extent_count = extents.len() as u16;
h.region_generation = h
.region_generation
.checked_add(generation_step)
.ok_or_else(metadata)?;
// Converted before the write, so nothing can fail after it.
let generation = Generation::from_raw(h.region_generation).ok_or_else(metadata)?;
let object = view.object().to_vec();
let mut buf = vec![0u8; inner.layout.block() as usize];
let n = encode_region_header(&mut buf, &h, &extents, &object).map_err(|_| metadata())?;
let next = plan_next(header, &buf[..n], inner.owner_generation)?;
let [a, b] = r.header_at;
let at = PairAt {
volume: *inner.volume.as_bytes(),
object: inner.layout.region_object(r.id.get()),
a,
b,
log2: inner.layout.log2_block,
format: FORMAT_2,
numbered: false,
};
let mut lane = self.lane(r, Op::Write)?;
let written = durable_slot::<P>(
&mut lane,
&inner.file,
&inner.pool,
&at,
header,
&next,
&inner.policy,
&inner.domain,
)
.inspect_err(|_| self.wake_frontiers())?;
*header = written;
Ok(generation)
}
}
/// The first `committed` extents the header in `pair` lists (the entry's;
/// a slot region has none), as the open's footprint holds them.
fn committed_extents(pair: &PairRead, log2_block: u8, committed: usize) -> Vec<(u64, u64)> {
pair.payload
.as_deref()
.and_then(|b| decode_region_header(b, log2_block, u64::MAX).ok())
.map(|v| v.extents().take(committed).collect())
.unwrap_or_default()
}