geographdb_core/storage/
sectioned.rs

1//! Sectioned file storage container
2//!
3//! Provides a safe, append-only container for multiple data sections
4//! within a single file. Uses explicit LE encoding/decoding with CRC32
5//! checksums for data integrity.
6
7use anyhow::{anyhow, Context, Result};
8use crc32fast::Hasher as Crc32Hasher;
9use std::collections::BTreeMap;
10use std::fs::File;
11use std::io::{Read, Seek, SeekFrom, Write};
12use std::path::Path;
13
14// Debug macro - only compiles in when debug-prints feature is enabled
15#[cfg(feature = "debug-prints")]
16macro_rules! debug_print {
17    ($($arg:tt)*) => {
18        eprintln!($($arg)*);
19    };
20}
21
22#[cfg(not(feature = "debug-prints"))]
23macro_rules! debug_print {
24    ($($arg:tt)*) => {
25        ()
26    };
27}
28
29// ============================================================================
30// CONSTANTS
31// ============================================================================
32
33/// File magic number: "GEODB" + 3 null bytes
34pub const FILE_MAGIC: [u8; 8] = *b"GEODB\0\0\0";
35
36/// Current file format version
37pub const FORMAT_VERSION: u32 = 1;
38
39/// Header size: 128 bytes
40pub const HEADER_SIZE: u64 = 128;
41pub const HEADER_SIZE_USIZE: usize = 128;
42
43/// Section entry size: 64 bytes per entry
44pub const SECTION_ENTRY_SIZE: u64 = 64;
45pub const SECTION_ENTRY_SIZE_USIZE: usize = 64;
46
47/// Maximum section name length (UTF-8 bytes)
48pub const MAX_SECTION_NAME_LEN: usize = 32;
49
50// ============================================================================
51// HEADER STRUCT
52// ============================================================================
53
54/// GeoFileHeader - EXACTLY 128 bytes
55///
56/// Byte layout:
57/// Offset  Size    Field
58/// ------  ------  -----
59/// 0x00    8       magic
60/// 0x08    4       version
61/// 0x0C    4       flags
62/// 0x10    8       section_table_offset
63/// 0x18    8       section_count
64/// 0x20    8       next_data_offset
65/// 0x28    8       created_at_epoch
66/// 0x30    8       modified_at_epoch
67/// 0x38    72      reserved
68/// -----   ----
69/// Total:  128
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub struct GeoFileHeader {
72    pub magic: [u8; 8],
73    pub version: u32,
74    pub flags: u32,
75    pub section_table_offset: u64,
76    pub section_count: u64,
77    pub next_data_offset: u64,
78    pub created_at_epoch: u64,
79    pub modified_at_epoch: u64,
80    pub reserved: [u8; 72],
81}
82
83impl Default for GeoFileHeader {
84    fn default() -> Self {
85        let now = std::time::SystemTime::now()
86            .duration_since(std::time::UNIX_EPOCH)
87            .unwrap_or_default()
88            .as_secs();
89
90        Self {
91            magic: FILE_MAGIC,
92            version: FORMAT_VERSION,
93            flags: 0,
94            section_table_offset: HEADER_SIZE,
95            section_count: 0,
96            next_data_offset: HEADER_SIZE,
97            created_at_epoch: now,
98            modified_at_epoch: 0,
99            reserved: [0u8; 72],
100        }
101    }
102}
103
104// ============================================================================
105// SECTION ENTRY STRUCT
106// ============================================================================
107
108/// SectionEntry - for on-disk section table entry
109///
110/// Byte layout:
111/// Offset  Size    Field
112/// ------  ------  -----
113/// 0x00    32      name (UTF-8, zero-padded)
114/// 0x20    8       offset
115/// 0x28    8       length
116/// 0x30    8       capacity
117/// 0x38    4       flags
118/// 0x3C    4       checksum
119/// 0x40    24      reserved
120/// -----   ----
121/// Total:  64
122#[derive(Debug, Clone, PartialEq, Eq)]
123pub struct SectionEntry {
124    pub name: String,
125    pub offset: u64,
126    pub length: u64,
127    pub capacity: u64,
128    pub flags: u32,
129    pub checksum: u32,
130}
131
132// ============================================================================
133// SECTION STRUCT (RUNTIME)
134// ============================================================================
135
136/// Section - runtime metadata
137#[derive(Debug, Clone, PartialEq, Eq)]
138pub struct Section {
139    pub name: String,
140    pub offset: u64,
141    pub length: u64,
142    pub capacity: u64,
143    pub flags: u32,
144    pub checksum: u32,
145}
146
147// ============================================================================
148// MAIN STORAGE STRUCT
149// ============================================================================
150
151/// Sectioned file storage container
152///
153/// File layout (Phase A - Append-Only with Dead Tables):
154/// ```text
155/// [0..128)                    - header
156/// [various offsets]            - live section payloads (may have gaps)
157/// [header.section_table_offset..) - current live section table
158/// [section_table_offset..file_len) - dead bytes (old tables) may exist
159/// ```
160impl std::fmt::Debug for SectionedStorage {
161    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
162        f.debug_struct("SectionedStorage")
163            .field("path", &self.path)
164            .field("header", &self.header)
165            .field("section_count", &self.sections.len())
166            .field("dirty", &self.dirty)
167            .finish()
168    }
169}
170
171pub struct SectionedStorage {
172    file: File,
173    path: std::path::PathBuf,
174    header: GeoFileHeader,
175    sections: BTreeMap<String, Section>,
176    dirty: bool,
177}
178
179// ============================================================================
180// ENCODING/DECODING HELPERS
181// ============================================================================
182
183pub fn encode_header(header: &GeoFileHeader) -> [u8; HEADER_SIZE_USIZE] {
184    let mut buf = [0u8; HEADER_SIZE_USIZE];
185
186    // 0x00: magic (8)
187    buf[0..8].copy_from_slice(&header.magic);
188
189    // 0x08: version (4)
190    buf[8..12].copy_from_slice(&header.version.to_le_bytes());
191
192    // 0x0C: flags (4)
193    buf[12..16].copy_from_slice(&header.flags.to_le_bytes());
194
195    // 0x10: section_table_offset (8)
196    buf[16..24].copy_from_slice(&header.section_table_offset.to_le_bytes());
197
198    // 0x18: section_count (8)
199    buf[24..32].copy_from_slice(&header.section_count.to_le_bytes());
200
201    // 0x20: next_data_offset (8)
202    buf[32..40].copy_from_slice(&header.next_data_offset.to_le_bytes());
203
204    // 0x28: created_at_epoch (8)
205    buf[40..48].copy_from_slice(&header.created_at_epoch.to_le_bytes());
206
207    // 0x30: modified_at_epoch (8)
208    buf[48..56].copy_from_slice(&header.modified_at_epoch.to_le_bytes());
209
210    // 0x38: reserved (72) - already zeroed
211
212    buf
213}
214
215pub fn decode_header(buf: &[u8; HEADER_SIZE_USIZE]) -> Result<GeoFileHeader> {
216    // Verify magic first
217    let magic: [u8; 8] = buf[0..8]
218        .try_into()
219        .map_err(|_| anyhow!("Magic slice has wrong size"))?;
220
221    if magic != FILE_MAGIC {
222        return Err(anyhow!(
223            "Invalid magic: expected {:?}, got {:?}",
224            FILE_MAGIC,
225            magic
226        ));
227    }
228
229    let version = u32::from_le_bytes(buf[8..12].try_into()?);
230    if version != FORMAT_VERSION {
231        return Err(anyhow!(
232            "Unsupported version: expected {}, got {}",
233            FORMAT_VERSION,
234            version
235        ));
236    }
237
238    Ok(GeoFileHeader {
239        magic,
240        version,
241        flags: u32::from_le_bytes(buf[12..16].try_into()?),
242        section_table_offset: u64::from_le_bytes(buf[16..24].try_into()?),
243        section_count: u64::from_le_bytes(buf[24..32].try_into()?),
244        next_data_offset: u64::from_le_bytes(buf[32..40].try_into()?),
245        created_at_epoch: u64::from_le_bytes(buf[40..48].try_into()?),
246        modified_at_epoch: u64::from_le_bytes(buf[48..56].try_into()?),
247        reserved: {
248            let mut arr = [0u8; 72];
249            arr.copy_from_slice(&buf[56..128]);
250            arr
251        },
252    })
253}
254
255fn encode_section_entry_name(name: &str) -> [u8; MAX_SECTION_NAME_LEN] {
256    let mut buf = [0u8; MAX_SECTION_NAME_LEN];
257    let name_bytes = name.as_bytes();
258    let len = name_bytes.len().min(MAX_SECTION_NAME_LEN);
259    buf[..len].copy_from_slice(&name_bytes[..len]);
260    buf
261}
262
263fn decode_section_entry_name(buf: &[u8]) -> Result<String> {
264    let len = buf.iter().position(|&b| b == 0).unwrap_or(buf.len());
265    String::from_utf8(buf[..len].to_vec()).context("Section name is not valid UTF-8")
266}
267
268pub fn encode_section_entry(entry: &SectionEntry) -> [u8; SECTION_ENTRY_SIZE_USIZE] {
269    let mut buf = [0u8; SECTION_ENTRY_SIZE_USIZE];
270
271    // 0x00: name (32) - zero-padded
272    buf[0..32].copy_from_slice(&encode_section_entry_name(&entry.name));
273
274    // 0x20: offset (8)
275    buf[32..40].copy_from_slice(&entry.offset.to_le_bytes());
276
277    // 0x28: length (8)
278    buf[40..48].copy_from_slice(&entry.length.to_le_bytes());
279
280    // 0x30: capacity (8)
281    buf[48..56].copy_from_slice(&entry.capacity.to_le_bytes());
282
283    // 0x38: flags (4)
284    buf[56..60].copy_from_slice(&entry.flags.to_le_bytes());
285
286    // 0x3C: checksum (4)
287    buf[60..64].copy_from_slice(&entry.checksum.to_le_bytes());
288
289    // 0x40: reserved (24) - already zeroed
290
291    buf
292}
293
294pub fn decode_section_entry(buf: &[u8; SECTION_ENTRY_SIZE_USIZE]) -> Result<SectionEntry> {
295    let name = decode_section_entry_name(&buf[0..32])?;
296
297    Ok(SectionEntry {
298        name,
299        offset: u64::from_le_bytes(buf[32..40].try_into()?),
300        length: u64::from_le_bytes(buf[40..48].try_into()?),
301        capacity: u64::from_le_bytes(buf[48..56].try_into()?),
302        flags: u32::from_le_bytes(buf[56..60].try_into()?),
303        checksum: u32::from_le_bytes(buf[60..64].try_into()?),
304    })
305}
306
307pub fn compute_checksum(data: &[u8]) -> u32 {
308    let mut hasher = Crc32Hasher::new();
309    hasher.update(data);
310    hasher.finalize()
311}
312
313// ============================================================================
314// SECTIONED STORAGE IMPLEMENTATION
315// ============================================================================
316
317impl SectionedStorage {
318    /// Create a new sectioned file
319    pub fn create(path: &Path) -> Result<Self> {
320        let mut file = std::fs::OpenOptions::new()
321            .read(true)
322            .write(true)
323            .create(true)
324            .truncate(true)
325            .open(path)
326            .context("Failed to create sectioned file")?;
327
328        let header = GeoFileHeader::default();
329        let header_bytes = encode_header(&header);
330        file.write_all(&header_bytes)
331            .context("Failed to write header")?;
332        file.sync_all().context("Failed to sync file")?;
333
334        Ok(Self {
335            file,
336            path: path.to_path_buf(),
337            header,
338            sections: BTreeMap::new(),
339            dirty: false,
340        })
341    }
342
343    /// Check if a file is a sectioned file without opening it
344    ///
345    /// This is a lightweight check that only reads the file header
346    /// to verify the magic number. It doesn't validate the entire file structure.
347    pub fn is_sectioned_file(path: &Path) -> bool {
348        if !path.exists() {
349            return false;
350        }
351
352        // Try to read just the header
353        match std::fs::File::open(path) {
354            Ok(mut file) => {
355                let mut header_buf = [0u8; HEADER_SIZE_USIZE];
356                if file.read_exact(&mut header_buf).is_err() {
357                    return false;
358                }
359                // Check magic number
360                match decode_header(&header_buf) {
361                    Ok(header) => header.magic == FILE_MAGIC,
362                    Err(_) => false,
363                }
364            }
365            Err(_) => false,
366        }
367    }
368
369    /// Open an existing sectioned file
370    ///
371    /// Reads header and section table, validates structure integrity.
372    /// Returns error if validation fails.
373    pub fn open(path: &Path) -> Result<Self> {
374        let mut file = std::fs::OpenOptions::new()
375            .read(true)
376            .write(true)
377            .open(path)
378            .context("Failed to open sectioned file")?;
379
380        // Read header
381        let mut header_buf = [0u8; HEADER_SIZE_USIZE];
382        file.read_exact(&mut header_buf)
383            .context("Failed to read header")?;
384        let header = decode_header(&header_buf)?;
385
386        // Read section table
387        let mut sections = BTreeMap::new();
388
389        if header.section_count > 0 {
390            file.seek(SeekFrom::Start(header.section_table_offset))
391                .context("Failed to seek to section table")?;
392
393            for _ in 0..header.section_count {
394                let mut entry_buf = [0u8; SECTION_ENTRY_SIZE_USIZE];
395                file.read_exact(&mut entry_buf)
396                    .context("Failed to read section entry")?;
397                let entry = decode_section_entry(&entry_buf)?;
398
399                sections.insert(
400                    entry.name.clone(),
401                    Section {
402                        name: entry.name,
403                        offset: entry.offset,
404                        length: entry.length,
405                        capacity: entry.capacity,
406                        flags: entry.flags,
407                        checksum: entry.checksum,
408                    },
409                );
410            }
411        }
412
413        let mut storage = Self {
414            file,
415            path: path.to_path_buf(),
416            header,
417            sections,
418            dirty: false,
419        };
420
421        // CRITICAL: Validate before returning
422        storage.validate()?;
423
424        Ok(storage)
425    }
426
427    /// Create a new section with IMMEDIATE physical reservation
428    ///
429    /// This method:
430    /// 1. Validates the name
431    /// 2. Gets current file length
432    /// 3. Computes allocation_base = max(next_data_offset, file_len)
433    /// 4. Allocates from allocation_base
434    /// 5. Physically extends file to reserve capacity
435    /// 6. Updates header.next_data_offset
436    ///
437    /// Fails if:
438    /// - Name is empty
439    /// - Name exceeds 32 UTF-8 bytes
440    /// - Section with same name exists
441    /// - Addition would overflow
442    /// - File extension fails
443    pub fn create_section(&mut self, name: &str, capacity: u64, flags: u32) -> Result<()> {
444        // Validate name: non-empty
445        if name.is_empty() {
446            return Err(anyhow!("Section name cannot be empty"));
447        }
448
449        // Validate name: UTF-8 byte length
450        let name_bytes = name.as_bytes();
451        if name_bytes.len() > MAX_SECTION_NAME_LEN {
452            return Err(anyhow!(
453                "Section name too long: {} bytes > {}",
454                name_bytes.len(),
455                MAX_SECTION_NAME_LEN
456            ));
457        }
458
459        // Check for duplicate
460        if self.sections.contains_key(name) {
461            return Err(anyhow!("Section '{}' already exists", name));
462        }
463
464        // CRITICAL: Get current file length (may be > next_data_offset due to table)
465        let file_len = self
466            .file
467            .metadata()
468            .context("Failed to get file metadata")?
469            .len();
470
471        // CRITICAL: Allocation must start after BOTH next_data_offset AND current EOF
472        let allocation_base = self.header.next_data_offset.max(file_len);
473
474        // Calculate new next_data_offset with overflow check
475        let new_next_data_offset = allocation_base.checked_add(capacity).ok_or_else(|| {
476            anyhow!(
477                "Data offset overflow: allocation_base {} + capacity {}",
478                allocation_base,
479                capacity
480            )
481        })?;
482
483        // Physically extend file to reserve capacity
484        self.file.set_len(new_next_data_offset).with_context(|| {
485            format!(
486                "Failed to reserve {} bytes for section '{}' at offset {}",
487                capacity, name, allocation_base
488            )
489        })?;
490
491        // Allocate from computed base
492        let offset = allocation_base;
493
494        // Update header to track new append position
495        self.header.next_data_offset = new_next_data_offset;
496
497        // Add section (length=0, checksum=0 until written)
498        self.sections.insert(
499            name.to_string(),
500            Section {
501                name: name.to_string(),
502                offset,
503                length: 0,
504                capacity,
505                flags,
506                checksum: 0,
507            },
508        );
509
510        self.dirty = true;
511        Ok(())
512    }
513
514    /// Write data to an existing section
515    ///
516    /// **TASK 4 CAPACITY BEHAVIOR**: Fails loudly with explicit error on overflow.
517    /// Never silently loses data.
518    ///
519    /// Fails if:
520    /// - Section doesn't exist
521    /// - data.len() > section.capacity (explicit capacity error)
522    pub fn write_section(&mut self, name: &str, data: &[u8]) -> Result<()> {
523        debug_print!(
524            "[WRITE_SECTION] Writing section '{}': {} bytes",
525            name,
526            data.len()
527        );
528        let section = self
529            .sections
530            .get(name)
531            .ok_or_else(|| anyhow!("Section '{}' not found", name))?;
532
533        if data.len() as u64 > section.capacity {
534            // TASK 4: Explicit capacity error with full context
535            return Err(anyhow!(
536                "Section '{}' overflow: attempted to write {} bytes, but capacity is {} bytes ({} bytes over limit)\n\
537                 Section details: offset={}, length={}, capacity={}\n\
538                 To fix: Increase section capacity during database creation or migrate to a larger capacity.",
539                name,
540                data.len(),
541                section.capacity,
542                data.len() as u64 - section.capacity,
543                section.offset,
544                section.length,
545                section.capacity
546            ));
547        }
548
549        let checksum = compute_checksum(data);
550
551        self.file
552            .seek(SeekFrom::Start(section.offset))
553            .context("Failed to seek to section")?;
554        self.file
555            .write_all(data)
556            .context("Failed to write section data")?;
557
558        // Update section metadata
559        if let Some(s) = self.sections.get_mut(name) {
560            s.length = data.len() as u64;
561            s.checksum = checksum;
562            debug_print!(
563                "[WRITE_SECTION] Updated section '{}' metadata: length={}, checksum={}",
564                name,
565                s.length,
566                s.checksum
567            );
568        }
569
570        self.dirty = true;
571        Ok(())
572    }
573
574    /// Read data from a section
575    ///
576    /// Fails if:
577    /// - Section doesn't exist
578    /// - Checksum mismatch
579    pub fn read_section(&mut self, name: &str) -> Result<Vec<u8>> {
580        let section = self
581            .sections
582            .get(name)
583            .ok_or_else(|| anyhow!("Section '{}' not found", name))?;
584
585        if section.length == 0 {
586            return Ok(Vec::new());
587        }
588
589        self.file
590            .seek(SeekFrom::Start(section.offset))
591            .context("Failed to seek to section")?;
592
593        let mut buffer = vec![0u8; section.length as usize];
594        self.file
595            .read_exact(&mut buffer)
596            .context("Failed to read section data")?;
597
598        // Verify checksum
599        let computed = compute_checksum(&buffer);
600        if computed != section.checksum {
601            return Err(anyhow!(
602                "Checksum mismatch for section '{}': stored {}, computed {}",
603                name,
604                section.checksum,
605                computed
606            ));
607        }
608
609        Ok(buffer)
610    }
611
612    /// Get section metadata without reading data
613    pub fn get_section(&self, name: &str) -> Option<&Section> {
614        self.sections.get(name)
615    }
616
617    /// List all sections
618    pub fn list_sections(&self) -> Vec<Section> {
619        self.sections.values().cloned().collect()
620    }
621
622    /// Get the number of sections
623    pub fn section_count(&self) -> usize {
624        self.sections.len()
625    }
626
627    /// Get the file path
628    pub fn path(&self) -> &Path {
629        &self.path
630    }
631
632    /// Get a reference to the header
633    pub fn header(&self) -> &GeoFileHeader {
634        &self.header
635    }
636
637    /// Flush section table to EOF
638    ///
639    /// Phase A behavior:
640    /// - Appends a FRESH section table at current EOF
641    /// - Updates header.section_table_offset to point to new table
642    /// - Old section tables become dead bytes in file
643    /// - File size grows with each flush
644    ///
645    /// File layout after flush():
646    /// ```text
647    /// [0..128)                    - header
648    /// [128..next_data_offset)      - section payload area
649    /// [section_table_offset..EOF)  - current section table
650    /// ```
651    ///
652    /// Because `create_section()` guarantees EOF >= next_data_offset,
653    /// the section table is always placed after all section payloads.
654    ///
655    /// Compaction (reclaiming dead tables) is deferred to a later phase.
656    pub fn flush(&mut self) -> Result<()> {
657        let now = std::time::SystemTime::now()
658            .duration_since(std::time::UNIX_EPOCH)
659            .unwrap_or_default()
660            .as_secs();
661
662        // Seek to end of file for new table
663        let table_offset = self
664            .file
665            .seek(SeekFrom::End(0))
666            .context("Failed to seek to EOF for section table")?;
667
668        debug_print!(
669            "[FLUSH_DEBUG] Writing section table at offset {}",
670            table_offset
671        );
672        debug_print!("[FLUSH_DEBUG] Section count: {}", self.sections.len());
673
674        // Write all section entries
675        for section in self.sections.values() {
676            debug_print!(
677                "[FLUSH_DEBUG]   Section {}: offset={}, length={}",
678                section.name,
679                section.offset,
680                section.length
681            );
682            let entry = SectionEntry {
683                name: section.name.clone(),
684                offset: section.offset,
685                length: section.length,
686                capacity: section.capacity,
687                flags: section.flags,
688                checksum: section.checksum,
689            };
690            let entry_bytes = encode_section_entry(&entry);
691            self.file
692                .write_all(&entry_bytes)
693                .context("Failed to write section entry")?;
694        }
695
696        // Update header
697        self.header.section_table_offset = table_offset;
698        self.header.section_count = self.sections.len() as u64;
699        self.header.modified_at_epoch = now;
700
701        // Write header at offset 0
702        self.file
703            .seek(SeekFrom::Start(0))
704            .context("Failed to seek to header")?;
705        let header_bytes = encode_header(&self.header);
706        self.file
707            .write_all(&header_bytes)
708            .context("Failed to write header")?;
709
710        self.file.sync_all().context("Failed to sync file")?;
711
712        self.dirty = false;
713        Ok(())
714    }
715
716    /// Validate file structure integrity
717    ///
718    /// IMPORTANT: If `self.dirty == true`, returns an error.
719    /// Unflushed state cannot be validated against disk.
720    pub fn validate(&mut self) -> Result<()> {
721        // Cannot validate dirty/unflushed state
722        if self.dirty {
723            return Err(anyhow!(
724                "cannot validate dirty/unflushed state; flush first"
725            ));
726        }
727
728        // 1. File metadata basics
729        let metadata = self
730            .file
731            .metadata()
732            .context("Failed to get file metadata")?;
733        let file_len = metadata.len();
734
735        // 2. File must be at least header size
736        if file_len < HEADER_SIZE {
737            return Err(anyhow!(
738                "File too small: {} < header size {}",
739                file_len,
740                HEADER_SIZE
741            ));
742        }
743
744        // 3. Re-read header from disk to detect corruption
745        let mut header_buf = [0u8; HEADER_SIZE_USIZE];
746        self.file
747            .seek(SeekFrom::Start(0))
748            .context("Failed to seek to header for validation")?;
749        self.file
750            .read_exact(&mut header_buf)
751            .context("Failed to read header for validation")?;
752        let disk_header = decode_header(&header_buf)?;
753
754        // 4. Copy validated header to self
755        self.header = disk_header.clone();
756
757        // 5. Verify section_table_offset is in bounds
758        if disk_header.section_table_offset < HEADER_SIZE {
759            return Err(anyhow!(
760                "Section table offset {} before data area {}",
761                disk_header.section_table_offset,
762                HEADER_SIZE
763            ));
764        }
765        if disk_header.section_table_offset > file_len {
766            return Err(anyhow!(
767                "Section table offset {} beyond file length {}",
768                disk_header.section_table_offset,
769                file_len
770            ));
771        }
772
773        // 6. PHYSICAL RESERVATION CHECK
774        if file_len < disk_header.next_data_offset {
775            return Err(anyhow!(
776                "File truncated: length {} < next_data_offset {} (physical reservation missing)",
777                file_len,
778                disk_header.next_data_offset
779            ));
780        }
781
782        // 7. Verify next_data_offset <= section_table_offset
783        if disk_header.next_data_offset > disk_header.section_table_offset {
784            return Err(anyhow!(
785                "Data area overlaps table: next_data_offset {} > section_table_offset {}",
786                disk_header.next_data_offset,
787                disk_header.section_table_offset
788            ));
789        }
790
791        // 8. Verify section table fits in file
792        if disk_header.section_count > 0 {
793            let table_size = disk_header
794                .section_count
795                .checked_mul(SECTION_ENTRY_SIZE)
796                .ok_or_else(|| anyhow!("Section count overflow"))?;
797            let table_end = disk_header
798                .section_table_offset
799                .checked_add(table_size)
800                .ok_or_else(|| anyhow!("Section table end overflow"))?;
801            if table_end > file_len {
802                return Err(anyhow!(
803                    "Section table extends beyond file: offset {} + size {} = {} > length {}",
804                    disk_header.section_table_offset,
805                    table_size,
806                    table_end,
807                    file_len
808                ));
809            }
810        }
811
812        // 9. Verify section names are non-empty
813        for name in self.sections.keys() {
814            if name.is_empty() {
815                return Err(anyhow!("Section name is empty"));
816            }
817        }
818
819        // 10. Verify each section (sorted by offset for overlap detection)
820        let mut prev_end = HEADER_SIZE;
821        let mut sorted_sections: Vec<_> = self.sections.iter().collect();
822        sorted_sections.sort_by_key(|(_, section)| section.offset);
823        for (name, section) in sorted_sections {
824            // 10a. Offset in data area (before current table)
825            if section.offset < HEADER_SIZE {
826                return Err(anyhow!(
827                    "Section '{}' offset {} before data area {}",
828                    name,
829                    section.offset,
830                    HEADER_SIZE
831                ));
832            }
833            if section.offset >= disk_header.section_table_offset {
834                return Err(anyhow!(
835                    "Section '{}' offset {} at or after current table {}",
836                    name,
837                    section.offset,
838                    disk_header.section_table_offset
839                ));
840            }
841
842            // 10b. capacity >= length
843            if section.capacity < section.length {
844                return Err(anyhow!(
845                    "Section '{}' capacity {} < length {}",
846                    name,
847                    section.capacity,
848                    section.length
849                ));
850            }
851
852            // 10c. Section fits before current table
853            let section_end = section
854                .offset
855                .checked_add(section.capacity)
856                .ok_or_else(|| anyhow!("Section '{}' end overflow", name))?;
857            if section_end > disk_header.section_table_offset {
858                return Err(anyhow!(
859                    "Section '{}' (offset {} + capacity {} = {}) extends beyond current table start {}",
860                    name,
861                    section.offset,
862                    section.capacity,
863                    section_end,
864                    disk_header.section_table_offset
865                ));
866            }
867
868            // 10d. No overlap with previous section
869            if section.offset < prev_end {
870                return Err(anyhow!(
871                    "Section '{}' (offset {}) overlaps previous section (ends at {})",
872                    name,
873                    section.offset,
874                    prev_end
875                ));
876            }
877            prev_end = section_end;
878
879            // 10e. Verify checksum only if length > 0
880            if section.length > 0 {
881                self.file
882                    .seek(SeekFrom::Start(section.offset))
883                    .context("Failed to seek to section for checksum validation")?;
884                let mut buf = vec![0u8; section.length as usize];
885                self.file
886                    .read_exact(&mut buf)
887                    .context("Failed to read section for checksum validation")?;
888                let computed = compute_checksum(&buf);
889                if computed != section.checksum {
890                    return Err(anyhow!(
891                        "Checksum mismatch for section '{}': stored {}, computed {}",
892                        name,
893                        section.checksum,
894                        computed
895                    ));
896                }
897            }
898        }
899
900        // 11. Verify next_data_offset doesn't overlap last section
901        if disk_header.next_data_offset < prev_end {
902            return Err(anyhow!(
903                "next_data_offset {} overlaps last section (ends at {})",
904                disk_header.next_data_offset,
905                prev_end
906            ));
907        }
908
909        // 12. Special case: empty file
910        if disk_header.section_count == 0 && disk_header.section_table_offset != HEADER_SIZE {
911            return Err(anyhow!(
912                "Empty file: section_table_offset should be {}, got {}",
913                HEADER_SIZE,
914                disk_header.section_table_offset
915            ));
916        }
917
918        Ok(())
919    }
920
921    /// Validate that required sections exist
922    pub fn validate_required_sections(&self, required: &[&str]) -> Result<()> {
923        for name in required {
924            if !self.sections.contains_key(*name) {
925                return Err(anyhow!("Required section '{}' is missing", name));
926            }
927        }
928        Ok(())
929    }
930
931    /// Resize a section to a new (larger) capacity
932    ///
933    /// This method:
934    /// 1. Reads the current section data
935    /// 2. Creates a new section with the larger capacity at EOF
936    /// 3. Copies the data to the new location
937    /// 4. Removes the old section (becomes dead space)
938    /// 5. Flushes to disk
939    ///
940    /// Fails if:
941    /// - Section doesn't exist
942    /// - New capacity is smaller than current data length
943    /// - Read/write operations fail
944    ///
945    /// Note: The old section's space becomes dead space in the file.
946    /// A future compaction operation could reclaim this space.
947    pub fn resize_section(&mut self, name: &str, new_capacity: u64) -> Result<()> {
948        // Clone needed fields before any mutable operations
949        let (offset, length, _capacity, flags, checksum) = {
950            let section = self
951                .sections
952                .get(name)
953                .ok_or_else(|| anyhow!("Section '{}' not found", name))?;
954
955            if new_capacity < section.length {
956                return Err(anyhow!(
957                    "Cannot resize section '{}' to {} bytes: current data length is {} bytes",
958                    name,
959                    new_capacity,
960                    section.length
961                ));
962            }
963
964            if new_capacity == section.capacity {
965                // Already at this capacity, nothing to do
966                return Ok(());
967            }
968
969            debug_print!(
970                "[RESIZE_SECTION] Resizing '{}' from {} to {} bytes",
971                name,
972                section.capacity,
973                new_capacity
974            );
975
976            (
977                section.offset,
978                section.length,
979                section.capacity,
980                section.flags,
981                section.checksum,
982            )
983        };
984
985        // Read current data
986        let current_data = if length > 0 {
987            self.file
988                .seek(SeekFrom::Start(offset))
989                .context("Failed to seek to section for resize")?;
990            let mut buffer = vec![0u8; length as usize];
991            self.file
992                .read_exact(&mut buffer)
993                .context("Failed to read section data for resize")?;
994            buffer
995        } else {
996            Vec::new()
997        };
998
999        // Get current file length and next_data_offset
1000        let file_len = self
1001            .file
1002            .metadata()
1003            .context("Failed to get file metadata")?
1004            .len();
1005        let allocation_base = self.header.next_data_offset.max(file_len);
1006
1007        // Calculate new next_data_offset
1008        let new_next_data_offset = allocation_base
1009            .checked_add(new_capacity)
1010            .ok_or_else(|| anyhow!("Data offset overflow during section resize"))?;
1011
1012        // Physically extend file to reserve new capacity
1013        self.file
1014            .set_len(new_next_data_offset)
1015            .context("Failed to extend file for section resize")?;
1016
1017        // Create new section at end of file
1018        let new_offset = allocation_base;
1019
1020        // Write data to new location
1021        if !current_data.is_empty() {
1022            self.file
1023                .seek(SeekFrom::Start(new_offset))
1024                .context("Failed to seek to new section location")?;
1025            self.file
1026                .write_all(&current_data)
1027                .context("Failed to write data to new section location")?;
1028        }
1029
1030        // Update section metadata (remove old, add new)
1031        self.sections.remove(name);
1032        self.sections.insert(
1033            name.to_string(),
1034            Section {
1035                name: name.to_string(),
1036                offset: new_offset,
1037                length,
1038                capacity: new_capacity,
1039                flags,
1040                checksum,
1041            },
1042        );
1043
1044        // Update header
1045        self.header.next_data_offset = new_next_data_offset;
1046        self.dirty = true;
1047
1048        debug_print!(
1049            "[RESIZE_SECTION] Section '{}' moved from offset {} to {}",
1050            name,
1051            offset,
1052            new_offset
1053        );
1054
1055        Ok(())
1056    }
1057}
1058
1059// ============================================================================
1060// UNIT TESTS
1061// ============================================================================
1062
1063#[cfg(test)]
1064mod tests {
1065    use super::*;
1066
1067    // === HEADER ENCODING/DECODING TESTS ===
1068
1069    #[test]
1070    fn test_header_exactly_128_bytes() {
1071        let header = GeoFileHeader::default();
1072        let encoded = encode_header(&header);
1073        assert_eq!(encoded.len(), 128, "Header must be exactly 128 bytes");
1074    }
1075
1076    #[test]
1077    fn test_header_reserved_field_size() {
1078        let header = GeoFileHeader::default();
1079        assert_eq!(header.reserved.len(), 72);
1080    }
1081
1082    #[test]
1083    fn test_header_roundtrip_preserves_all_fields() {
1084        let header = GeoFileHeader {
1085            flags: 0x12345678,
1086            section_table_offset: 0x1000,
1087            section_count: 5,
1088            next_data_offset: 0x2000,
1089            created_at_epoch: 1234567890,
1090            modified_at_epoch: 1234567900,
1091            ..Default::default()
1092        };
1093
1094        let encoded = encode_header(&header);
1095        let decoded = decode_header(&encoded).unwrap();
1096
1097        assert_eq!(decoded, header);
1098    }
1099
1100    #[test]
1101    fn test_decode_header_reserved_slice_correct() {
1102        let mut buf = [0u8; 128];
1103        buf[0..8].copy_from_slice(&FILE_MAGIC);
1104        buf[8..12].copy_from_slice(&1u32.to_le_bytes());
1105        // Set some bytes in reserved area
1106        buf[100] = 0x42;
1107        buf[127] = 0xFF;
1108
1109        let decoded = decode_header(&buf).unwrap();
1110
1111        // Reserved should capture bytes 56..127
1112        assert_eq!(decoded.reserved[100 - 56], 0x42);
1113        assert_eq!(decoded.reserved[127 - 56], 0xFF);
1114    }
1115
1116    #[test]
1117    fn test_invalid_magic_rejected() {
1118        let mut buf = [0u8; 128];
1119        buf[0..8].copy_from_slice(b"BADMAGIC");
1120        assert!(decode_header(&buf).is_err());
1121    }
1122
1123    // === SECTION ENTRY ENCODING/DECODING TESTS ===
1124
1125    #[test]
1126    fn test_section_entry_exactly_64_bytes() {
1127        let entry = SectionEntry {
1128            name: "test".to_string(),
1129            offset: 0,
1130            length: 0,
1131            capacity: 0,
1132            flags: 0,
1133            checksum: 0,
1134        };
1135        let encoded = encode_section_entry(&entry);
1136        assert_eq!(encoded.len(), 64, "Section entry must be exactly 64 bytes");
1137    }
1138
1139    #[test]
1140    fn test_section_entry_roundtrip() {
1141        let entry = SectionEntry {
1142            name: "test_section".to_string(),
1143            offset: 1024,
1144            length: 512,
1145            capacity: 1024,
1146            flags: 0x12345678,
1147            checksum: 0xABCDEF01,
1148        };
1149        let encoded = encode_section_entry(&entry);
1150        assert_eq!(encoded.len(), 64);
1151        let decoded = decode_section_entry(&encoded).unwrap();
1152        assert_eq!(entry.name, decoded.name);
1153        assert_eq!(entry.offset, decoded.offset);
1154        assert_eq!(entry.length, decoded.length);
1155        assert_eq!(entry.capacity, decoded.capacity);
1156        assert_eq!(entry.flags, decoded.flags);
1157        assert_eq!(entry.checksum, decoded.checksum);
1158    }
1159
1160    #[test]
1161    fn test_section_name_encoding() {
1162        let name = "cfg_data";
1163        let encoded = encode_section_entry_name(name);
1164        let decoded = decode_section_entry_name(&encoded).unwrap();
1165        assert_eq!(name, decoded);
1166    }
1167
1168    // === CHECKSUM TESTS ===
1169
1170    #[test]
1171    fn test_checksum_deterministic() {
1172        let data = b"test data";
1173        let crc1 = compute_checksum(data);
1174        let crc2 = compute_checksum(data);
1175        assert_eq!(crc1, crc2);
1176    }
1177
1178    #[test]
1179    fn test_checksum_detects_corruption() {
1180        let data1 = b"test data";
1181        let data2 = b"test datb"; // One byte changed
1182        assert_ne!(compute_checksum(data1), compute_checksum(data2));
1183    }
1184
1185    #[test]
1186    fn test_checksum_empty() {
1187        let data = b"";
1188        let crc = compute_checksum(data);
1189        // CRC32 of empty string is a known value
1190        assert_eq!(crc, 0);
1191    }
1192
1193    // === NAME VALIDATION TESTS ===
1194
1195    #[test]
1196    fn test_section_name_32_bytes_accepted() {
1197        let name_32_bytes = "12345678901234567890123456789012"; // 32 ASCII bytes
1198        assert_eq!(name_32_bytes.len(), 32);
1199
1200        let encoded = encode_section_entry_name(name_32_bytes);
1201        let decoded = decode_section_entry_name(&encoded).unwrap();
1202        assert_eq!(name_32_bytes, decoded);
1203    }
1204
1205    #[test]
1206    fn test_section_name_encoding_truncates_at_32() {
1207        let name_33_bytes = "123456789012345678901234567890123"; // 33 ASCII bytes
1208        assert_eq!(name_33_bytes.len(), 33);
1209
1210        let encoded = encode_section_entry_name(name_33_bytes);
1211        let decoded = decode_section_entry_name(&encoded).unwrap();
1212        assert_eq!(decoded.len(), 32); // Truncated
1213        assert_eq!(decoded, "12345678901234567890123456789012");
1214    }
1215}
geographdb_core/storage/sectioned.rs

geographdb_core/storage/
sectioned.rs
