iroh_bytes/store/fs/
migrate_redb_v1_v2.rs

1use std::path::{Path, PathBuf};
2
3use anyhow::Result;
4use redb_v1::ReadableTable;
5use tempfile::NamedTempFile;
6use tracing::info;
7
8pub fn run(source: impl AsRef<Path>) -> Result<redb::Database> {
9    let source = source.as_ref();
10    let dir = source.parent().expect("database is not in root");
11    // create the new database in a tempfile in the same directory as the old db
12    let target = NamedTempFile::with_prefix_in("blobs.db.migrate", dir)?;
13    let target = target.into_temp_path();
14    info!("migrate {} to {}", source.display(), target.display());
15    let old_db = redb_v1::Database::open(source)?;
16    let new_db = redb::Database::create(&target)?;
17
18    let rtx = old_db.begin_read()?;
19    let wtx = new_db.begin_write()?;
20
21    {
22        let old_blobs = rtx.open_table(old::BLOBS_TABLE)?;
23        let mut new_blobs = wtx.open_table(new::BLOBS_TABLE)?;
24        let len = old_blobs.len()?;
25        info!("migrate blobs table ({len} rows)");
26        for (i, entry) in old_blobs.iter()?.enumerate() {
27            let (key, value) = entry?;
28            let key: crate::Hash = key.value().into();
29            let value = value.value();
30            if i > 0 && i % 1000 == 0 {
31                info!("    row {i:>6} of {len}");
32            }
33            new_blobs.insert(key, value)?;
34        }
35        info!("migrate blobs table done");
36        let old_tags = rtx.open_table(old::TAGS_TABLE)?;
37        let mut new_tags = wtx.open_table(new::TAGS_TABLE)?;
38        let len = old_tags.len()?;
39        info!("migrate tags table ({len} rows)");
40        for (i, entry) in old_tags.iter()?.enumerate() {
41            let (key, value) = entry?;
42            let key = key.value();
43            let value: crate::HashAndFormat = value.value().into();
44            if i > 0 && i % 1000 == 0 {
45                info!("    row {i:>6} of {len}");
46            }
47            new_tags.insert(key, value)?;
48        }
49        info!("migrate tags table done");
50        let old_inline_data = rtx.open_table(old::INLINE_DATA_TABLE)?;
51        let mut new_inline_data = wtx.open_table(new::INLINE_DATA_TABLE)?;
52        let len = old_inline_data.len()?;
53        info!("migrate inline data table ({len} rows)");
54        for (i, entry) in old_inline_data.iter()?.enumerate() {
55            let (key, value) = entry?;
56            let key: crate::Hash = key.value().into();
57            let value = value.value();
58            if i > 0 && i % 1000 == 0 {
59                info!("    row {i:>6} of {len}");
60            }
61            new_inline_data.insert(key, value)?;
62        }
63        info!("migrate inline data table done");
64        let old_inline_outboard = rtx.open_table(old::INLINE_OUTBOARD_TABLE)?;
65        let mut new_inline_outboard = wtx.open_table(new::INLINE_OUTBOARD_TABLE)?;
66        let len = old_inline_outboard.len()?;
67        info!("migrate inline outboard table ({len} rows)");
68        for (i, entry) in old_inline_outboard.iter()?.enumerate() {
69            let (key, value) = entry?;
70            let key: crate::Hash = key.value().into();
71            let value = value.value();
72            if i > 0 && i % 1000 == 0 {
73                info!("    row {i:>6} of {len}");
74            }
75            new_inline_outboard.insert(key, value)?;
76        }
77        info!("migrate inline outboard table done");
78    }
79
80    wtx.commit()?;
81    drop(rtx);
82    drop(old_db);
83    drop(new_db);
84
85    let backup_path: PathBuf = {
86        let mut p = source.to_owned().into_os_string();
87        p.push(".backup-redb-v1");
88        p.into()
89    };
90    info!("rename {} to {}", source.display(), backup_path.display());
91    std::fs::rename(source, &backup_path)?;
92    info!("rename {} to {}", target.display(), source.display());
93    target.persist_noclobber(source)?;
94    info!("opening migrated database from {}", source.display());
95    let db = redb::Database::open(source)?;
96    Ok(db)
97}
98
99mod new {
100    pub(super) use super::super::tables::*;
101}
102
103mod old {
104    use super::super::EntryState;
105    use crate::util::Tag;
106    use bytes::Bytes;
107    use iroh_base::hash::BlobFormat;
108    use postcard::experimental::max_size::MaxSize;
109    use redb_v1::{RedbKey, RedbValue, TableDefinition, TypeName};
110    use serde::{Deserialize, Deserializer, Serialize, Serializer};
111    use smallvec::SmallVec;
112
113    pub const BLOBS_TABLE: TableDefinition<Hash, EntryState> = TableDefinition::new("blobs-0");
114
115    pub const TAGS_TABLE: TableDefinition<Tag, HashAndFormat> = TableDefinition::new("tags-0");
116
117    pub const INLINE_DATA_TABLE: TableDefinition<Hash, &[u8]> =
118        TableDefinition::new("inline-data-0");
119
120    pub const INLINE_OUTBOARD_TABLE: TableDefinition<Hash, &[u8]> =
121        TableDefinition::new("inline-outboard-0");
122
123    impl redb_v1::RedbValue for EntryState {
124        type SelfType<'a> = EntryState;
125
126        type AsBytes<'a> = SmallVec<[u8; 128]>;
127
128        fn fixed_width() -> Option<usize> {
129            None
130        }
131
132        fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
133        where
134            Self: 'a,
135        {
136            postcard::from_bytes(data).unwrap()
137        }
138
139        fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
140        where
141            Self: 'a,
142            Self: 'b,
143        {
144            postcard::to_extend(value, SmallVec::new()).unwrap()
145        }
146
147        fn type_name() -> TypeName {
148            TypeName::new("EntryState")
149        }
150    }
151
152    impl RedbValue for HashAndFormat {
153        type SelfType<'a> = Self;
154
155        type AsBytes<'a> = [u8; Self::POSTCARD_MAX_SIZE];
156
157        fn fixed_width() -> Option<usize> {
158            Some(Self::POSTCARD_MAX_SIZE)
159        }
160
161        fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
162        where
163            Self: 'a,
164        {
165            let t: &'a [u8; Self::POSTCARD_MAX_SIZE] = data.try_into().unwrap();
166            postcard::from_bytes(t.as_slice()).unwrap()
167        }
168
169        fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
170        where
171            Self: 'a,
172            Self: 'b,
173        {
174            let mut res = [0u8; 33];
175            postcard::to_slice(&value, &mut res).unwrap();
176            res
177        }
178
179        fn type_name() -> TypeName {
180            TypeName::new("iroh_base::HashAndFormat")
181        }
182    }
183
184    impl RedbValue for Tag {
185        type SelfType<'a> = Self;
186
187        type AsBytes<'a> = bytes::Bytes;
188
189        fn fixed_width() -> Option<usize> {
190            None
191        }
192
193        fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
194        where
195            Self: 'a,
196        {
197            Self(Bytes::copy_from_slice(data))
198        }
199
200        fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
201        where
202            Self: 'a,
203            Self: 'b,
204        {
205            value.0.clone()
206        }
207
208        fn type_name() -> TypeName {
209            TypeName::new("Tag")
210        }
211    }
212
213    impl RedbKey for Tag {
214        fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering {
215            data1.cmp(data2)
216        }
217    }
218
219    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
220    pub struct Hash([u8; 32]);
221
222    impl Serialize for Hash {
223        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
224        where
225            S: Serializer,
226        {
227            self.0.serialize(serializer)
228        }
229    }
230
231    impl<'de> Deserialize<'de> for Hash {
232        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
233        where
234            D: Deserializer<'de>,
235        {
236            let data: [u8; 32] = Deserialize::deserialize(deserializer)?;
237            Ok(Self(data))
238        }
239    }
240
241    impl MaxSize for Hash {
242        const POSTCARD_MAX_SIZE: usize = 32;
243    }
244
245    impl From<Hash> for crate::Hash {
246        fn from(value: Hash) -> Self {
247            value.0.into()
248        }
249    }
250
251    impl RedbValue for Hash {
252        type SelfType<'a> = Self;
253
254        type AsBytes<'a> = &'a [u8; 32];
255
256        fn fixed_width() -> Option<usize> {
257            Some(32)
258        }
259
260        fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
261        where
262            Self: 'a,
263        {
264            let contents: &'a [u8; 32] = data.try_into().unwrap();
265            Hash(*contents)
266        }
267
268        fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a>
269        where
270            Self: 'a,
271            Self: 'b,
272        {
273            &value.0
274        }
275
276        fn type_name() -> TypeName {
277            TypeName::new("iroh_base::Hash")
278        }
279    }
280
281    impl RedbKey for Hash {
282        fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering {
283            data1.cmp(data2)
284        }
285    }
286
287    /// A hash and format pair
288    #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, MaxSize)]
289    pub struct HashAndFormat {
290        /// The hash
291        pub hash: Hash,
292        /// The format
293        pub format: BlobFormat,
294    }
295
296    impl From<HashAndFormat> for crate::HashAndFormat {
297        fn from(value: HashAndFormat) -> Self {
298            crate::HashAndFormat {
299                hash: value.hash.into(),
300                format: value.format,
301            }
302        }
303    }
304    impl Serialize for HashAndFormat {
305        fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
306        where
307            S: Serializer,
308        {
309            (self.hash, self.format).serialize(serializer)
310        }
311    }
312
313    impl<'de> Deserialize<'de> for HashAndFormat {
314        fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
315        where
316            D: Deserializer<'de>,
317        {
318            let (hash, format) = <(Hash, BlobFormat)>::deserialize(deserializer)?;
319            Ok(Self { hash, format })
320        }
321    }
322}
iroh_bytes/store/fs/migrate_redb_v1_v2.rs

iroh_bytes/store/fs/
migrate_redb_v1_v2.rs
