1pub mod constants;
8
9use anyhow::{bail, Context, Result};
10use constants::*;
11use std::collections::{HashMap, HashSet};
12
13use crate::orchestrator::BLOB_SIZE_C_BIT;
14use std::time::{SystemTime, UNIX_EPOCH};
15
16use crate::piv::PivBackend;
17
18#[derive(Debug, Clone)]
24pub struct Object {
25 pub(crate) index: u8,
27 pub(crate) object_size: usize,
29
30 pub(crate) yblob_magic: u32,
32 pub(crate) object_count: u8,
33 pub(crate) store_key_slot: u8,
34 pub(crate) age: u32,
36
37 pub(crate) chunk_pos: u8,
39 pub(crate) next_chunk: u8,
40
41 pub blob_mtime: u32,
44 pub blob_size: u32,
46 pub blob_key_slot: u8,
48 pub blob_plain_size: u32,
50 pub is_compressed: bool,
52 pub blob_name: String,
53
54 pub(crate) payload: Vec<u8>,
56
57 pub(crate) dirty: bool,
59}
60
61impl Object {
62 pub fn from_bytes(index: u8, data: &[u8]) -> Result<Self> {
64 let object_size = data.len();
65 if !(OBJECT_MIN_SIZE..=OBJECT_MAX_SIZE).contains(&object_size) {
66 bail!(
67 "object {index}: invalid size {object_size} \
68 (expected {OBJECT_MIN_SIZE}..={OBJECT_MAX_SIZE})"
69 );
70 }
71
72 let magic = read_u32_le(data, MAGIC_O)?;
73 if magic != YBLOB_MAGIC {
74 bail!("object {index}: bad magic 0x{magic:08x} (expected 0x{YBLOB_MAGIC:08x})");
75 }
76
77 let object_count = data[OBJECT_COUNT_O];
78 let store_key_slot = data[STORE_KEY_SLOT_O];
79 let age = read_u24_le(data, OBJECT_AGE_O)?;
80
81 if age == 0 {
82 return Ok(Self {
84 index,
85 object_size,
86 yblob_magic: magic,
87 object_count,
88 store_key_slot,
89 age: 0,
90 chunk_pos: 0,
91 next_chunk: 0,
92 blob_mtime: 0,
93 blob_size: 0,
94 blob_key_slot: 0,
95 blob_plain_size: 0,
96 is_compressed: false,
97 blob_name: String::new(),
98 payload: Vec::new(),
99 dirty: false,
100 });
101 }
102
103 let chunk_pos = data[CHUNK_POS_O];
104 let next_chunk = data[NEXT_CHUNK_O];
105
106 let (
107 blob_mtime,
108 blob_size,
109 blob_key_slot,
110 blob_plain_size,
111 is_compressed,
112 blob_name,
113 payload_start,
114 ) = if chunk_pos == 0 {
115 let mtime = read_u32_le(data, BLOB_MTIME_O)?;
116 let bsize = read_u24_le(data, BLOB_SIZE_O)?;
117 let bkslot = data[BLOB_KEY_SLOT_O];
118 let raw_plain = read_u24_le(data, BLOB_PLAIN_SIZE_O)?;
119 let compressed = raw_plain & BLOB_SIZE_C_BIT as u32 != 0;
120 let plain = raw_plain & !(BLOB_SIZE_C_BIT as u32);
121 let nlen = data[BLOB_NAME_LEN_O] as usize;
122 if BLOB_NAME_O + nlen > object_size {
123 bail!("object {index}: name length {nlen} overflows object");
124 }
125 let name = std::str::from_utf8(&data[BLOB_NAME_O..BLOB_NAME_O + nlen])
126 .with_context(|| format!("object {index}: blob name is not valid UTF-8"))?
127 .to_owned();
128 (
129 mtime,
130 bsize,
131 bkslot,
132 plain,
133 compressed,
134 name,
135 BLOB_NAME_O + nlen,
136 )
137 } else {
138 (0, 0, 0, 0, false, String::new(), CONTINUATION_PAYLOAD_O)
139 };
140
141 let raw_payload = &data[payload_start..];
146 let payload = raw_payload.to_vec();
147
148 Ok(Self {
149 index,
150 object_size,
151 yblob_magic: magic,
152 object_count,
153 store_key_slot,
154 age,
155 chunk_pos,
156 next_chunk,
157 blob_mtime,
158 blob_size,
159 blob_key_slot,
160 blob_plain_size,
161 is_compressed,
162 blob_name,
163 payload,
164 dirty: false,
165 })
166 }
167
168 pub fn to_bytes(&self) -> Vec<u8> {
176 if self.age == 0 {
177 let mut buf = vec![0u8; OBJECT_MIN_SIZE];
179 write_u32_le(&mut buf, MAGIC_O, self.yblob_magic);
180 buf[OBJECT_COUNT_O] = self.object_count;
181 buf[STORE_KEY_SLOT_O] = self.store_key_slot;
182 return buf;
184 }
185
186 if self.chunk_pos == 0 {
187 let name_bytes = self.blob_name.as_bytes();
189 let total = (BLOB_NAME_O + name_bytes.len() + self.payload.len())
190 .clamp(OBJECT_MIN_SIZE, OBJECT_MAX_SIZE);
191 let mut buf = vec![0u8; total];
192 write_u32_le(&mut buf, MAGIC_O, self.yblob_magic);
193 buf[OBJECT_COUNT_O] = self.object_count;
194 buf[STORE_KEY_SLOT_O] = self.store_key_slot;
195 write_u24_le(&mut buf, OBJECT_AGE_O, self.age);
196 buf[CHUNK_POS_O] = self.chunk_pos;
197 buf[NEXT_CHUNK_O] = self.next_chunk;
198 write_u32_le(&mut buf, BLOB_MTIME_O, self.blob_mtime);
199 write_u24_le(&mut buf, BLOB_SIZE_O, self.blob_size);
200 buf[BLOB_KEY_SLOT_O] = self.blob_key_slot;
201 write_u24_le(
202 &mut buf,
203 BLOB_PLAIN_SIZE_O,
204 self.blob_plain_size | self.c_bit(),
205 );
206 buf[BLOB_NAME_LEN_O] = name_bytes.len() as u8;
207 buf[BLOB_NAME_O..BLOB_NAME_O + name_bytes.len()].copy_from_slice(name_bytes);
208 let payload_start = BLOB_NAME_O + name_bytes.len();
209 let payload_end = (payload_start + self.payload.len()).min(total);
210 buf[payload_start..payload_end]
211 .copy_from_slice(&self.payload[..payload_end - payload_start]);
212 buf
213 } else {
214 let total = (CONTINUATION_PAYLOAD_O + self.payload.len())
216 .clamp(OBJECT_MIN_SIZE, OBJECT_MAX_SIZE);
217 let mut buf = vec![0u8; total];
218 write_u32_le(&mut buf, MAGIC_O, self.yblob_magic);
219 buf[OBJECT_COUNT_O] = self.object_count;
220 buf[STORE_KEY_SLOT_O] = self.store_key_slot;
221 write_u24_le(&mut buf, OBJECT_AGE_O, self.age);
222 buf[CHUNK_POS_O] = self.chunk_pos;
223 buf[NEXT_CHUNK_O] = self.next_chunk;
224 let payload_end = (CONTINUATION_PAYLOAD_O + self.payload.len()).min(total);
225 buf[CONTINUATION_PAYLOAD_O..payload_end]
226 .copy_from_slice(&self.payload[..payload_end - CONTINUATION_PAYLOAD_O]);
227 buf
228 }
229 }
230
231 pub fn reset(&mut self) {
237 let index = self.index;
238 let object_count = self.object_count;
239 let store_key_slot = self.store_key_slot;
240 *self = Self {
241 index,
242 object_size: OBJECT_MIN_SIZE, yblob_magic: crate::store::constants::YBLOB_MAGIC,
244 object_count,
245 store_key_slot,
246 age: 0,
247 chunk_pos: 0,
248 next_chunk: 0,
249 blob_mtime: 0,
250 blob_size: 0,
251 blob_key_slot: 0,
252 blob_plain_size: 0,
253 is_compressed: false,
254 blob_name: String::new(),
255 payload: Vec::new(),
256 dirty: true,
257 };
258 }
259
260 pub fn head_payload_capacity(_object_size: usize, name_len: usize) -> usize {
265 OBJECT_MAX_SIZE.saturating_sub(BLOB_NAME_O + name_len)
266 }
267
268 pub fn continuation_payload_capacity(_object_size: usize) -> usize {
273 OBJECT_MAX_SIZE.saturating_sub(CONTINUATION_PAYLOAD_O)
274 }
275
276 fn c_bit(&self) -> u32 {
279 BLOB_SIZE_C_BIT as u32 * self.is_compressed as u32
280 }
281
282 pub fn is_empty(&self) -> bool {
283 self.age == 0
284 }
285
286 pub fn is_head(&self) -> bool {
287 self.age != 0 && self.chunk_pos == 0
288 }
289
290 pub fn is_encrypted(&self) -> bool {
291 self.blob_key_slot != 0
292 }
293
294 pub fn set_payload(&mut self, payload: Vec<u8>) {
299 self.payload = payload;
300 }
301
302 pub fn index(&self) -> u8 {
303 self.index
304 }
305
306 pub fn age(&self) -> u32 {
307 self.age
308 }
309
310 pub fn object_count(&self) -> u8 {
312 self.object_count
313 }
314
315 pub fn chunk_pos(&self) -> u8 {
316 self.chunk_pos
317 }
318
319 pub fn next_chunk(&self) -> u8 {
320 self.next_chunk
321 }
322
323 pub fn payload(&self) -> &[u8] {
325 &self.payload
326 }
327
328 pub fn payload_len(&self) -> usize {
330 self.payload.len()
331 }
332
333 pub fn object_size(&self) -> usize {
334 self.object_size
335 }
336}
337
338pub struct ObjectParams {
344 pub index: u8,
345 pub age: u32,
346 pub chunk_pos: u8,
347 pub next_chunk: u8,
348}
349
350pub struct Store {
356 pub reader: String,
357 pub object_size: usize,
358 pub object_count: u8,
359 pub store_key_slot: u8,
360 pub objects: Vec<Object>,
361 pub store_age: u32,
363 pub missing: Vec<u8>,
366}
367
368fn is_absent(e: &anyhow::Error) -> bool {
370 matches!(
371 e.downcast_ref::<crate::errors::CardError>(),
372 Some(crate::errors::CardError::Status { sw: 0x6A82, .. })
373 )
374}
375
376pub const MAX_AGE: u32 = 0xFF_FFFF;
378
379impl Store {
380 pub fn from_device(reader: &str, piv: &dyn PivBackend) -> Result<Self> {
384 let mut store = Self::from_device_raw(reader, piv)?;
385 store.sanitize();
386 Ok(store)
387 }
388
389 pub fn from_device_raw(reader: &str, piv: &dyn PivBackend) -> Result<Self> {
392 let first_id = OBJECT_ID_ZERO;
393 let raw = piv
394 .read_object(reader, first_id)
395 .with_context(|| format!("reading object 0x{first_id:06x}"))?;
396
397 let first =
398 Object::from_bytes(0, &raw).with_context(|| "parsing object 0 (store header)")?;
399
400 let object_count = first.object_count;
401 let object_size = raw.len();
402 let store_key_slot = first.store_key_slot;
403
404 let mut objects = vec![first.clone()];
405 let mut missing = Vec::new();
406 for i in 1..object_count {
407 let id = OBJECT_ID_ZERO + i as u32;
408 let raw = match piv.read_object(reader, id) {
409 Ok(raw) => raw,
410 Err(e) if is_absent(&e) => {
413 let mut obj = first.clone();
414 obj.index = i;
415 obj.reset();
416 objects.push(obj);
417 missing.push(i);
418 continue;
419 }
420 Err(e) => return Err(e).with_context(|| format!("reading object 0x{id:06x}")),
421 };
422 let obj = Object::from_bytes(i, &raw).with_context(|| format!("parsing object {i}"))?;
423 objects.push(obj);
424 }
425
426 let store_age = objects.iter().map(|o| o.age).max().unwrap_or(0);
427
428 Ok(Self {
429 reader: reader.to_owned(),
430 object_size,
431 object_count,
432 store_key_slot,
433 objects,
434 store_age,
435 missing,
436 })
437 }
438
439 pub fn format(
445 reader: &str,
446 piv: &dyn PivBackend,
447 object_count: u8,
448 store_key_slot: u8,
449 management_key: &str,
450 ) -> Result<Self> {
451 let mut store = Self {
453 reader: reader.to_owned(),
454 object_size: OBJECT_MIN_SIZE,
455 object_count,
456 store_key_slot,
457 objects: Vec::with_capacity(object_count as usize),
458 store_age: 0,
459 missing: Vec::new(),
460 };
461 for i in 0..object_count {
462 store.objects.push(store.make_object(ObjectParams {
464 index: i,
465 age: 0,
466 chunk_pos: 0,
467 next_chunk: 0,
468 }));
469 }
470 store.write_dirty(piv, management_key, false, &[])?;
472 Ok(store)
473 }
474
475 pub fn chain_is_complete(&self, head_index: u8) -> bool {
481 let head = &self.objects[head_index as usize];
482 if !head.is_head() {
483 return false;
484 }
485 let mut idx = head_index;
486 let mut pos = 0u8;
487 for _ in 0..self.objects.len() {
489 let next = self.objects[idx as usize].next_chunk;
490 if next == idx {
491 return true;
492 }
493 let Some(obj) = self.objects.get(next as usize) else {
494 return false;
495 };
496 if obj.is_empty()
497 || obj.object_count != self.object_count
498 || obj.chunk_pos == 0
499 || Some(obj.chunk_pos) != pos.checked_add(1)
500 || head.age.checked_add(u32::from(obj.chunk_pos)) != Some(obj.age)
501 {
502 return false;
503 }
504 pos = obj.chunk_pos;
505 idx = next;
506 }
507 false
508 }
509
510 pub fn sanitize(&mut self) -> usize {
515 let mut kept: HashMap<String, (u8, u32)> = HashMap::new();
517 for obj in self.objects.iter().filter(|o| o.is_head()) {
518 if obj.object_count != self.object_count {
521 continue;
522 }
523 if !self.chain_is_complete(obj.index) {
524 continue;
525 }
526 match kept.get(&obj.blob_name) {
527 Some(&(_, age)) if age >= obj.age => {}
528 _ => {
529 kept.insert(obj.blob_name.clone(), (obj.index, obj.age));
530 }
531 }
532 }
533
534 let mut reachable: HashSet<u8> = HashSet::new();
535 for &(head_idx, _) in kept.values() {
536 reachable.extend(self.chunk_chain(head_idx));
537 }
538
539 let to_reset: Vec<u8> = self
540 .objects
541 .iter()
542 .filter(|o| !o.is_empty() && !reachable.contains(&o.index))
543 .map(|o| o.index)
544 .collect();
545 for &idx in &to_reset {
546 self.objects[idx as usize].reset();
547 }
548 to_reset.len()
549 }
550
551 pub fn sync(&mut self, piv: &dyn PivBackend, management_key: &str) -> Result<()> {
555 self.sync_ordered(piv, management_key, &[])
556 }
557
558 pub fn sync_ordered(
562 &mut self,
563 piv: &dyn PivBackend,
564 management_key: &str,
565 order: &[u8],
566 ) -> Result<()> {
567 self.write_dirty(piv, management_key, true, order)
568 }
569
570 fn write_dirty(
571 &mut self,
572 piv: &dyn PivBackend,
573 management_key: &str,
574 note_partial: bool,
575 order: &[u8],
576 ) -> Result<()> {
577 use indicatif::{ProgressBar, ProgressStyle};
578
579 let mut dirty: Vec<u8> = order
580 .iter()
581 .copied()
582 .filter(|&idx| self.objects[idx as usize].dirty)
583 .collect();
584 dirty.extend(
585 self.objects
586 .iter()
587 .filter(|o| o.dirty && !order.contains(&o.index))
588 .map(|o| o.index),
589 );
590
591 let pb = ProgressBar::new(dirty.len() as u64);
592 pb.set_style(
593 ProgressStyle::with_template("Writing objects: [{bar:30}] {pos}/{len}")
594 .unwrap()
595 .progress_chars("=>-"),
596 );
597
598 for (written, idx) in dirty.iter().enumerate() {
599 let obj = &mut self.objects[*idx as usize];
600 let id = OBJECT_ID_ZERO + obj.index as u32;
601 let data = obj.to_bytes();
602 let result = piv.write_object(&self.reader, id, &data, management_key);
603 if note_partial && written > 0 {
604 result.context("the store may be partially updated; run `yb fsck`")?;
605 } else {
606 result?;
607 }
608 obj.dirty = false;
609 pb.inc(1);
610 }
611 pb.finish_and_clear();
612 Ok(())
613 }
614
615 pub fn alloc_free(&self) -> Option<u8> {
617 self.objects.iter().find(|o| o.is_empty()).map(|o| o.index)
618 }
619
620 pub fn free_count(&self) -> usize {
622 self.objects.iter().filter(|o| o.is_empty()).count()
623 }
624
625 pub fn find_head(&self, name: &str) -> Option<&Object> {
627 self.objects
628 .iter()
629 .find(|o| o.is_head() && o.blob_name == name)
630 }
631
632 pub fn chunk_chain(&self, head_index: u8) -> Vec<u8> {
638 let mut chain = vec![head_index];
639 let mut seen: HashSet<u8> = HashSet::from([head_index]);
640 let mut idx = head_index;
641 loop {
642 let next = self.objects[idx as usize].next_chunk;
643 if next == idx {
644 break;
645 }
646 if seen.contains(&next) || next as usize >= self.objects.len() {
647 break;
650 }
651 seen.insert(next);
652 chain.push(next);
653 idx = next;
654 }
655 chain
656 }
657
658 pub fn now_unix() -> u32 {
660 SystemTime::now()
661 .duration_since(UNIX_EPOCH)
662 .map(|d| d.as_secs() as u32)
663 .unwrap_or(0)
664 }
665
666 pub fn ensure_age_room(&self, count: usize) -> Result<()> {
670 if u64::from(self.store_age) + count as u64 > u64::from(MAX_AGE) {
671 return Err(crate::errors::YbError::new(
672 "this store's age counter is exhausted: it cannot record more writes",
673 )
674 .why(format!(
675 "ages are 24-bit, and the store is at age {}",
676 self.store_age
677 ))
678 .fix("fetch the blobs, run `yb format` (which resets the ages), store them again")
679 .into());
680 }
681 Ok(())
682 }
683
684 pub fn next_age(&mut self) -> u32 {
686 self.store_age += 1;
687 self.store_age
688 }
689
690 pub fn make_object(&self, p: ObjectParams) -> Object {
696 Object {
697 index: p.index,
698 object_size: self.object_size,
699 yblob_magic: crate::store::constants::YBLOB_MAGIC,
700 object_count: self.object_count,
701 store_key_slot: self.store_key_slot,
702 age: p.age,
703 chunk_pos: p.chunk_pos,
704 next_chunk: p.next_chunk,
705 blob_mtime: 0,
706 blob_size: 0,
707 blob_key_slot: 0,
708 blob_plain_size: 0,
709 is_compressed: false,
710 blob_name: String::new(),
711 payload: Vec::new(),
712 dirty: true,
713 }
714 }
715}
716
717pub(crate) fn read_u32_le(buf: &[u8], offset: usize) -> Result<u32> {
722 let end = offset + 4;
723 if end > buf.len() {
724 bail!(
725 "read_u32_le: offset {offset}+4 out of bounds (buf len {})",
726 buf.len()
727 );
728 }
729 Ok(u32::from_le_bytes(buf[offset..end].try_into().unwrap()))
730}
731
732pub(crate) fn read_u24_le(buf: &[u8], offset: usize) -> Result<u32> {
733 let end = offset + 3;
734 if end > buf.len() {
735 bail!(
736 "read_u24_le: offset {offset}+3 out of bounds (buf len {})",
737 buf.len()
738 );
739 }
740 let b = &buf[offset..end];
741 Ok(b[0] as u32 | ((b[1] as u32) << 8) | ((b[2] as u32) << 16))
742}
743
744pub(crate) fn write_u32_le(buf: &mut [u8], offset: usize, v: u32) {
745 buf[offset..offset + 4].copy_from_slice(&v.to_le_bytes());
746}
747
748pub(crate) fn write_u24_le(buf: &mut [u8], offset: usize, v: u32) {
749 buf[offset] = (v & 0xff) as u8;
750 buf[offset + 1] = ((v >> 8) & 0xff) as u8;
751 buf[offset + 2] = ((v >> 16) & 0xff) as u8;
752}
753
754#[cfg(test)]
759mod tests;