alopex_core/storage/format/
reader.rs

1//! ファイルリーダーの共通トレイトと入力ソース定義。
2//!
3//! この段階ではインターフェースのみを提供し、各プラットフォームごとの実装は
4//! 別タスク (3.1 Native / 3.2 WASM) で追加する。
5
6use std::pin::Pin;
7
8use crate::storage::checksum;
9use crate::storage::compression;
10use crate::storage::format::{
11    FileFooter, FileHeader, FileVersion, FormatError, SectionEntry, SectionIndex, FOOTER_SIZE,
12    HEADER_SIZE,
13};
14
15#[cfg(not(target_arch = "wasm32"))]
16use memmap2::Mmap;
17#[cfg(not(target_arch = "wasm32"))]
18use std::fs::File;
19#[cfg(not(target_arch = "wasm32"))]
20use std::path::{Path, PathBuf};
21
22#[cfg(target_arch = "wasm32")]
23use std::vec::Vec;
24
25/// ファイルソース抽象化。
26pub enum FileSource {
27    /// ファイルパス（Native用）。
28    #[cfg(not(target_arch = "wasm32"))]
29    Path(PathBuf),
30    /// バイトバッファ（WASM用）。
31    #[cfg(target_arch = "wasm32")]
32    Buffer(Vec<u8>),
33    /// IndexedDBキー（WASM + feature）。
34    #[cfg(all(target_arch = "wasm32", feature = "wasm-indexeddb"))]
35    IndexedDb {
36        db_name: String,
37        key: String,
38        /// ファイル全体のサイズ（バイト）。
39        length: u64,
40        /// 範囲読み込みローダー。
41        loader: Box<dyn RangeLoader>,
42    },
43}
44
45/// prefetch_sections の戻り値に用いるFuture型。
46#[cfg(not(target_arch = "wasm32"))]
47pub type PrefetchFuture<'a> =
48    Pin<Box<dyn std::future::Future<Output = Result<(), FormatError>> + Send + 'a>>;
49/// prefetch_sections の戻り値に用いるFuture型（WASM版、Send制約なし）。
50#[cfg(target_arch = "wasm32")]
51pub type PrefetchFuture<'a> =
52    Pin<Box<dyn std::future::Future<Output = Result<(), FormatError>> + 'a>>;
53
54/// WASM向けの範囲読み込みローダー。
55#[cfg(target_arch = "wasm32")]
56pub trait RangeLoader: Send + Sync {
57    /// `[offset, offset+length)` のバイト列を取得する。
58    fn load_range(&self, offset: u64, length: u64) -> Result<Vec<u8>, FormatError>;
59}
60
61/// メモリ上のバッファから範囲読み込みするデフォルト実装。
62#[cfg(target_arch = "wasm32")]
63pub struct BufferRangeLoader {
64    data: Vec<u8>,
65}
66
67#[cfg(target_arch = "wasm32")]
68impl BufferRangeLoader {
69    /// 新規バッファローダーを作成する。
70    pub fn new(data: Vec<u8>) -> Self {
71        Self { data }
72    }
73}
74
75#[cfg(target_arch = "wasm32")]
76impl RangeLoader for BufferRangeLoader {
77    fn load_range(&self, offset: u64, length: u64) -> Result<Vec<u8>, FormatError> {
78        let offset = offset as usize;
79        let length = length as usize;
80        let end = offset
81            .checked_add(length)
82            .ok_or(FormatError::IncompleteWrite)?;
83        if end > self.data.len() {
84            return Err(FormatError::IncompleteWrite);
85        }
86        Ok(self.data[offset..end].to_vec())
87    }
88}
89
90/// プラットフォーム共通のファイルリーダートレイト。
91pub trait FileReader {
92    /// ファイルを開き、ヘッダー/フッター/セクションインデックスを初期化する。
93    fn open(source: FileSource) -> Result<Self, FormatError>
94    where
95        Self: Sized;
96
97    /// ヘッダーへの参照を返す。
98    fn header(&self) -> &FileHeader;
99
100    /// フッターへの参照を返す。
101    fn footer(&self) -> &FileFooter;
102
103    /// セクションインデックスへの参照を返す。
104    fn section_index(&self) -> &SectionIndex;
105
106    /// 指定セクションを解凍済みバイト列で読み取る。
107    fn read_section(&self, section_id: u32) -> Result<Vec<u8>, FormatError>;
108
109    /// 指定セクションを圧縮状態のまま読み取る。
110    fn read_section_raw(&self, section_id: u32) -> Result<Vec<u8>, FormatError>;
111
112    /// 指定セクションのチェックサムを検証する（圧縮後データを対象）。
113    fn validate_section(&self, section_id: u32) -> Result<(), FormatError>;
114
115    /// 全セクションの整合性を検証する。
116    fn validate_all(&self) -> Result<(), FormatError>;
117
118    /// 指定セクションを事前読み込みする（WASMではIndexedDBからの範囲読み込みを想定）。
119    fn prefetch_sections<'a>(&'a self, section_ids: &'a [u32]) -> PrefetchFuture<'a>;
120}
121
122/// ネイティブ向けファイルリーダー（mmap）。
123#[cfg(not(target_arch = "wasm32"))]
124pub struct AlopexFileReader {
125    mmap: Mmap,
126    header: FileHeader,
127    footer: FileFooter,
128    section_index: SectionIndex,
129}
130
131#[cfg(not(target_arch = "wasm32"))]
132impl AlopexFileReader {
133    fn map_file(path: &Path) -> Result<Mmap, FormatError> {
134        let file = File::open(path).map_err(|_| FormatError::IncompleteWrite)?;
135        unsafe { Mmap::map(&file).map_err(|_| FormatError::IncompleteWrite) }
136    }
137
138    fn read_footer(mmap: &Mmap) -> Result<FileFooter, FormatError> {
139        if mmap.len() < FOOTER_SIZE {
140            return Err(FormatError::IncompleteWrite);
141        }
142        let start = mmap.len() - FOOTER_SIZE;
143        let mut buf = [0u8; FOOTER_SIZE];
144        buf.copy_from_slice(&mmap[start..]);
145        FileFooter::from_bytes(&buf)
146    }
147
148    fn read_header(mmap: &Mmap) -> Result<FileHeader, FormatError> {
149        if mmap.len() < HEADER_SIZE {
150            return Err(FormatError::IncompleteWrite);
151        }
152        let mut buf = [0u8; HEADER_SIZE];
153        buf.copy_from_slice(&mmap[..HEADER_SIZE]);
154        let header = FileHeader::from_bytes(&buf)?;
155        header.check_compatibility(&FileVersion::CURRENT)?;
156        Ok(header)
157    }
158
159    fn read_section_index(mmap: &Mmap, footer: &FileFooter) -> Result<SectionIndex, FormatError> {
160        let offset = footer.section_index_offset as usize;
161        if offset >= mmap.len() {
162            return Err(FormatError::IncompleteWrite);
163        }
164        // まずcountを読むために最低4バイトが必要。
165        if mmap.len() < offset + 4 {
166            return Err(FormatError::IncompleteWrite);
167        }
168        let count = u32::from_le_bytes(
169            mmap[offset..offset + 4]
170                .try_into()
171                .expect("slice length checked"),
172        );
173        let expected = 4usize + count as usize * SectionEntry::SIZE;
174        if mmap.len() < offset + expected {
175            return Err(FormatError::IncompleteWrite);
176        }
177        SectionIndex::from_bytes(&mmap[offset..offset + expected])
178    }
179
180    fn entry(&self, section_id: u32) -> Result<&SectionEntry, FormatError> {
181        self.section_index
182            .find_by_id(section_id)
183            .ok_or(FormatError::IncompleteWrite)
184    }
185}
186
187#[cfg(not(target_arch = "wasm32"))]
188impl FileReader for AlopexFileReader {
189    fn open(source: FileSource) -> Result<Self, FormatError> {
190        let FileSource::Path(path) = source;
191        let mmap = Self::map_file(&path)?;
192        let footer = Self::read_footer(&mmap)?;
193        let section_index = Self::read_section_index(&mmap, &footer)?;
194        let header = Self::read_header(&mmap)?;
195        Ok(Self {
196            mmap,
197            header,
198            footer,
199            section_index,
200        })
201    }
202
203    fn header(&self) -> &FileHeader {
204        &self.header
205    }
206
207    fn footer(&self) -> &FileFooter {
208        &self.footer
209    }
210
211    fn section_index(&self) -> &SectionIndex {
212        &self.section_index
213    }
214
215    fn read_section(&self, section_id: u32) -> Result<Vec<u8>, FormatError> {
216        let entry = self.entry(section_id)?;
217        let raw = self.read_section_raw(section_id)?;
218        if raw.len() as u64 != entry.compressed_length {
219            return Err(FormatError::IncompleteWrite);
220        }
221        checksum::verify(&raw, self.header.checksum_algorithm, entry.checksum as u64)?;
222        let decompressed = compression::decompress(&raw, entry.compression)?;
223        if decompressed.len() as u64 != entry.uncompressed_length {
224            return Err(FormatError::IncompleteWrite);
225        }
226        Ok(decompressed)
227    }
228
229    fn read_section_raw(&self, section_id: u32) -> Result<Vec<u8>, FormatError> {
230        let entry = self.entry(section_id)?;
231        let offset = entry.offset as usize;
232        let end = offset
233            .checked_add(entry.compressed_length as usize)
234            .ok_or(FormatError::IncompleteWrite)?;
235        if end > self.mmap.len() || end - offset != entry.compressed_length as usize {
236            return Err(FormatError::IncompleteWrite);
237        }
238        Ok(self.mmap[offset..end].to_vec())
239    }
240
241    fn validate_section(&self, section_id: u32) -> Result<(), FormatError> {
242        let entry = self.entry(section_id)?;
243        let raw = self.read_section_raw(section_id)?;
244        checksum::verify(&raw, self.header.checksum_algorithm, entry.checksum as u64)
245    }
246
247    fn validate_all(&self) -> Result<(), FormatError> {
248        for entry in &self.section_index.entries {
249            self.validate_section(entry.section_id)?;
250        }
251        Ok(())
252    }
253
254    fn prefetch_sections<'a>(&'a self, _section_ids: &'a [u32]) -> PrefetchFuture<'a> {
255        // mmap上でアクセスしてページをウォームアップする。
256        let section_ids = _section_ids.to_vec();
257        Box::pin(async move {
258            for id in section_ids {
259                let _ = self.read_section_raw(id)?;
260            }
261            Ok(())
262        })
263    }
264}
265
266/// WASM向けの読み取り挙動設定。
267#[cfg(target_arch = "wasm32")]
268pub struct WasmReaderConfig {
269    /// このサイズ未満なら全体をバッファにロードする。
270    pub full_load_threshold_bytes: usize,
271    /// 大容量ファイル用の範囲ローダー（IndexedDB等）。
272    pub range_loader: Option<Box<dyn RangeLoader>>,
273}
274
275#[cfg(target_arch = "wasm32")]
276impl Default for WasmReaderConfig {
277    fn default() -> Self {
278        Self {
279            full_load_threshold_bytes: 100 * 1024 * 1024, // 100MB
280            range_loader: None,
281        }
282    }
283}
284
285/// WASM向けファイルリーダー（バッファ/IndexedDB）。
286#[cfg(target_arch = "wasm32")]
287pub struct AlopexFileReader {
288    buffer: Option<Vec<u8>>,
289    loader: Option<Box<dyn RangeLoader>>,
290    #[allow(dead_code)]
291    length: u64,
292    header: FileHeader,
293    footer: FileFooter,
294    section_index: SectionIndex,
295    #[allow(dead_code)]
296    config: WasmReaderConfig,
297}
298
299#[cfg(target_arch = "wasm32")]
300impl AlopexFileReader {
301    /// コンフィグ付きでファイルを開く。
302    pub fn open_with_config(
303        source: FileSource,
304        config: WasmReaderConfig,
305    ) -> Result<Self, FormatError> {
306        match source {
307            FileSource::Buffer(buf) => Self::from_buffer(buf, config),
308            #[cfg(feature = "wasm-indexeddb")]
309            FileSource::IndexedDb {
310                length,
311                db_name: _,
312                key: _,
313                loader,
314            } => {
315                let mut cfg = config;
316                cfg.range_loader = Some(loader);
317                Self::from_indexed_db(length, cfg)
318            }
319        }
320    }
321
322    fn from_buffer(buffer: Vec<u8>, config: WasmReaderConfig) -> Result<Self, FormatError> {
323        // 閾値超過時、range_loaderがあればIndexedDB経路へフォールバック。
324        if buffer.len() > config.full_load_threshold_bytes {
325            let mut cfg = config;
326            if let Some(loader) = cfg.range_loader.take() {
327                return Self::from_indexed_db(
328                    buffer.len() as u64,
329                    WasmReaderConfig {
330                        range_loader: Some(loader),
331                        ..cfg
332                    },
333                );
334            } else {
335                return Err(FormatError::IncompleteWrite);
336            }
337        }
338
339        if buffer.len() < HEADER_SIZE + FOOTER_SIZE {
340            return Err(FormatError::IncompleteWrite);
341        }
342
343        let footer = Self::read_footer(&buffer)?;
344        let section_index = Self::read_section_index(&buffer, &footer)?;
345        let header = Self::read_header(&buffer)?;
346
347        Ok(Self {
348            buffer: Some(buffer),
349            loader: None,
350            length: (HEADER_SIZE + FOOTER_SIZE + section_index.serialized_size()) as u64,
351            header,
352            footer,
353            section_index,
354            config,
355        })
356    }
357
358    fn from_indexed_db(length: u64, mut config: WasmReaderConfig) -> Result<Self, FormatError> {
359        let loader = config
360            .range_loader
361            .take()
362            .ok_or(FormatError::IncompleteWrite)?;
363
364        // サイズが閾値未満なら全体をロードしてバッファ経路に切替
365        if (length as usize) <= config.full_load_threshold_bytes {
366            let full = loader.load_range(0, length)?;
367            return Self::from_buffer(
368                full,
369                WasmReaderConfig {
370                    range_loader: Some(loader),
371                    ..config
372                },
373            );
374        }
375
376        // フッターを末尾から取得
377        let footer_start = length
378            .checked_sub(FOOTER_SIZE as u64)
379            .ok_or(FormatError::IncompleteWrite)?;
380        let footer_bytes = loader.load_range(footer_start, FOOTER_SIZE as u64)?;
381        let footer_array: [u8; FOOTER_SIZE] = footer_bytes
382            .try_into()
383            .map_err(|_| FormatError::IncompleteWrite)?;
384        let footer = FileFooter::from_bytes(&footer_array)?;
385
386        // セクションインデックスを取得
387        let index_offset = footer.section_index_offset;
388        // 最低でもcount + entries分だけ読む
389        let count_bytes = loader.load_range(index_offset, 4)?;
390        let count_arr: [u8; 4] = count_bytes
391            .try_into()
392            .map_err(|_| FormatError::IncompleteWrite)?;
393        let count = u32::from_le_bytes(count_arr);
394        let total_size = 4 + count as usize * SectionEntry::SIZE;
395        let index_bytes = loader.load_range(index_offset, total_size as u64)?;
396        let section_index = SectionIndex::from_bytes(&index_bytes)?;
397
398        // ヘッダー取得
399        let header_bytes = loader.load_range(0, HEADER_SIZE as u64)?;
400        let header_array: [u8; HEADER_SIZE] = header_bytes
401            .try_into()
402            .map_err(|_| FormatError::IncompleteWrite)?;
403        let header = FileHeader::from_bytes(&header_array)?;
404        header.check_compatibility(&FileVersion::CURRENT)?;
405
406        Ok(Self {
407            buffer: None,
408            loader: Some(loader),
409            length,
410            header,
411            footer,
412            section_index,
413            config,
414        })
415    }
416
417    fn read_footer(buffer: &[u8]) -> Result<FileFooter, FormatError> {
418        if buffer.len() < FOOTER_SIZE {
419            return Err(FormatError::IncompleteWrite);
420        }
421        let start = buffer.len() - FOOTER_SIZE;
422        let mut buf = [0u8; FOOTER_SIZE];
423        buf.copy_from_slice(&buffer[start..]);
424        FileFooter::from_bytes(&buf)
425    }
426
427    fn read_header(buffer: &[u8]) -> Result<FileHeader, FormatError> {
428        if buffer.len() < HEADER_SIZE {
429            return Err(FormatError::IncompleteWrite);
430        }
431        let mut buf = [0u8; HEADER_SIZE];
432        buf.copy_from_slice(&buffer[..HEADER_SIZE]);
433        let header = FileHeader::from_bytes(&buf)?;
434        header.check_compatibility(&FileVersion::CURRENT)?;
435        Ok(header)
436    }
437
438    fn read_section_index(buffer: &[u8], footer: &FileFooter) -> Result<SectionIndex, FormatError> {
439        let offset = footer.section_index_offset as usize;
440        if buffer.len() < offset + 4 {
441            return Err(FormatError::IncompleteWrite);
442        }
443        let count = u32::from_le_bytes(
444            buffer[offset..offset + 4]
445                .try_into()
446                .expect("slice length checked"),
447        );
448        let expected = 4usize + count as usize * SectionEntry::SIZE;
449        if buffer.len() < offset + expected {
450            return Err(FormatError::IncompleteWrite);
451        }
452        SectionIndex::from_bytes(&buffer[offset..offset + expected])
453    }
454
455    fn entry(&self, section_id: u32) -> Result<&SectionEntry, FormatError> {
456        self.section_index
457            .find_by_id(section_id)
458            .ok_or(FormatError::IncompleteWrite)
459    }
460}
461
462#[cfg(target_arch = "wasm32")]
463impl FileReader for AlopexFileReader {
464    fn open(source: FileSource) -> Result<Self, FormatError>
465    where
466        Self: Sized,
467    {
468        Self::open_with_config(source, WasmReaderConfig::default())
469    }
470
471    fn header(&self) -> &FileHeader {
472        &self.header
473    }
474
475    fn footer(&self) -> &FileFooter {
476        &self.footer
477    }
478
479    fn section_index(&self) -> &SectionIndex {
480        &self.section_index
481    }
482
483    fn read_section(&self, section_id: u32) -> Result<Vec<u8>, FormatError> {
484        let entry = self.entry(section_id)?;
485        let raw = self.read_section_raw(section_id)?;
486        if raw.len() as u64 != entry.compressed_length {
487            return Err(FormatError::IncompleteWrite);
488        }
489        checksum::verify(&raw, self.header.checksum_algorithm, entry.checksum as u64)?;
490        let decompressed = compression::decompress(&raw, entry.compression)?;
491        if decompressed.len() as u64 != entry.uncompressed_length {
492            return Err(FormatError::IncompleteWrite);
493        }
494        Ok(decompressed)
495    }
496
497    fn read_section_raw(&self, section_id: u32) -> Result<Vec<u8>, FormatError> {
498        let entry = self.entry(section_id)?;
499        match &self.buffer {
500            Some(buf) => {
501                let offset = entry.offset as usize;
502                let end = offset
503                    .checked_add(entry.compressed_length as usize)
504                    .ok_or(FormatError::IncompleteWrite)?;
505                if end > buf.len() || end - offset != entry.compressed_length as usize {
506                    return Err(FormatError::IncompleteWrite);
507                }
508                Ok(buf[offset..end].to_vec())
509            }
510            None => {
511                let loader = self.loader.as_ref().ok_or(FormatError::IncompleteWrite)?;
512                loader.load_range(entry.offset, entry.compressed_length)
513            }
514        }
515    }
516
517    fn validate_section(&self, section_id: u32) -> Result<(), FormatError> {
518        let entry = self.entry(section_id)?;
519        let raw = self.read_section_raw(section_id)?;
520        checksum::verify(&raw, self.header.checksum_algorithm, entry.checksum as u64)
521    }
522
523    fn validate_all(&self) -> Result<(), FormatError> {
524        for entry in &self.section_index.entries {
525            self.validate_section(entry.section_id)?;
526        }
527        Ok(())
528    }
529
530    fn prefetch_sections<'a>(&'a self, _section_ids: &'a [u32]) -> PrefetchFuture<'a> {
531        let section_ids = _section_ids.to_vec();
532        Box::pin(async move {
533            for id in section_ids {
534                let _ = self.read_section_raw(id)?;
535            }
536            Ok(())
537        })
538    }
539}
alopex_core/storage/format/reader.rs

alopex_core/storage/format/
reader.rs
