geographdb_core/storage/
sectioned.rs

1//! Sectioned file storage container
2//!
3//! Provides a safe, append-only container for multiple data sections
4//! within a single file. Uses explicit LE encoding/decoding with CRC32
5//! checksums for data integrity.
6
7use anyhow::{anyhow, Context, Result};
8use crc32fast::Hasher as Crc32Hasher;
9use std::collections::BTreeMap;
10use std::fs::File;
11use std::io::{Read, Seek, SeekFrom, Write};
12use std::path::Path;
13
14// Debug macro - only compiles in when debug-prints feature is enabled
15#[cfg(feature = "debug-prints")]
16macro_rules! debug_print {
17    ($($arg:tt)*) => {
18        eprintln!($($arg)*);
19    };
20}
21
22#[cfg(not(feature = "debug-prints"))]
23macro_rules! debug_print {
24    ($($arg:tt)*) => {
25        ()
26    };
27}
28
29// ============================================================================
30// CONSTANTS
31// ============================================================================
32
33/// File magic number: "GEODB" + 3 null bytes
34pub const FILE_MAGIC: [u8; 8] = *b"GEODB\0\0\0";
35
36/// Current file format version
37pub const FORMAT_VERSION: u32 = 1;
38
39/// Header size: 128 bytes
40pub const HEADER_SIZE: u64 = 128;
41pub const HEADER_SIZE_USIZE: usize = 128;
42
43/// Section entry size: 64 bytes per entry
44pub const SECTION_ENTRY_SIZE: u64 = 64;
45pub const SECTION_ENTRY_SIZE_USIZE: usize = 64;
46
47/// Maximum section name length (UTF-8 bytes)
48pub const MAX_SECTION_NAME_LEN: usize = 32;
49
50// ============================================================================
51// HEADER STRUCT
52// ============================================================================
53
54/// GeoFileHeader - EXACTLY 128 bytes
55///
56/// Byte layout:
57/// Offset  Size    Field
58/// ------  ------  -----
59/// 0x00    8       magic
60/// 0x08    4       version
61/// 0x0C    4       flags
62/// 0x10    8       section_table_offset
63/// 0x18    8       section_count
64/// 0x20    8       next_data_offset
65/// 0x28    8       created_at_epoch
66/// 0x30    8       modified_at_epoch
67/// 0x38    72      reserved
68/// -----   ----
69/// Total:  128
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub struct GeoFileHeader {
72    pub magic: [u8; 8],
73    pub version: u32,
74    pub flags: u32,
75    pub section_table_offset: u64,
76    pub section_count: u64,
77    pub next_data_offset: u64,
78    pub created_at_epoch: u64,
79    pub modified_at_epoch: u64,
80    pub reserved: [u8; 72],
81}
82
83impl Default for GeoFileHeader {
84    fn default() -> Self {
85        let now = std::time::SystemTime::now()
86            .duration_since(std::time::UNIX_EPOCH)
87            .unwrap_or_default()
88            .as_secs();
89
90        Self {
91            magic: FILE_MAGIC,
92            version: FORMAT_VERSION,
93            flags: 0,
94            section_table_offset: HEADER_SIZE,
95            section_count: 0,
96            next_data_offset: HEADER_SIZE,
97            created_at_epoch: now,
98            modified_at_epoch: 0,
99            reserved: [0u8; 72],
100        }
101    }
102}
103
104// ============================================================================
105// SECTION ENTRY STRUCT
106// ============================================================================
107
108/// SectionEntry - for on-disk section table entry
109///
110/// Byte layout:
111/// Offset  Size    Field
112/// ------  ------  -----
113/// 0x00    32      name (UTF-8, zero-padded)
114/// 0x20    8       offset
115/// 0x28    8       length
116/// 0x30    8       capacity
117/// 0x38    4       flags
118/// 0x3C    4       checksum
119/// 0x40    24      reserved
120/// -----   ----
121/// Total:  64
122#[derive(Debug, Clone, PartialEq, Eq)]
123pub struct SectionEntry {
124    pub name: String,
125    pub offset: u64,
126    pub length: u64,
127    pub capacity: u64,
128    pub flags: u32,
129    pub checksum: u32,
130}
131
132// ============================================================================
133// SECTION STRUCT (RUNTIME)
134// ============================================================================
135
136/// Section - runtime metadata
137#[derive(Debug, Clone, PartialEq, Eq)]
138pub struct Section {
139    pub name: String,
140    pub offset: u64,
141    pub length: u64,
142    pub capacity: u64,
143    pub flags: u32,
144    pub checksum: u32,
145}
146
147// ============================================================================
148// MAIN STORAGE STRUCT
149// ============================================================================
150
151/// Sectioned file storage container
152///
153/// File layout (Phase A - Append-Only with Dead Tables):
154/// ```text
155/// [0..128)                    - header
156/// [various offsets]            - live section payloads (may have gaps)
157/// [header.section_table_offset..) - current live section table
158/// [section_table_offset..file_len) - dead bytes (old tables) may exist
159/// ```
160impl std::fmt::Debug for SectionedStorage {
161    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
162        f.debug_struct("SectionedStorage")
163            .field("path", &self.path)
164            .field("header", &self.header)
165            .field("section_count", &self.sections.len())
166            .field("dirty", &self.dirty)
167            .finish()
168    }
169}
170
171pub struct SectionedStorage {
172    file: File,
173    path: std::path::PathBuf,
174    header: GeoFileHeader,
175    sections: BTreeMap<String, Section>,
176    dirty: bool,
177}
178
179// ============================================================================
180// ENCODING/DECODING HELPERS
181// ============================================================================
182
183pub fn encode_header(header: &GeoFileHeader) -> [u8; HEADER_SIZE_USIZE] {
184    let mut buf = [0u8; HEADER_SIZE_USIZE];
185
186    // 0x00: magic (8)
187    buf[0..8].copy_from_slice(&header.magic);
188
189    // 0x08: version (4)
190    buf[8..12].copy_from_slice(&header.version.to_le_bytes());
191
192    // 0x0C: flags (4)
193    buf[12..16].copy_from_slice(&header.flags.to_le_bytes());
194
195    // 0x10: section_table_offset (8)
196    buf[16..24].copy_from_slice(&header.section_table_offset.to_le_bytes());
197
198    // 0x18: section_count (8)
199    buf[24..32].copy_from_slice(&header.section_count.to_le_bytes());
200
201    // 0x20: next_data_offset (8)
202    buf[32..40].copy_from_slice(&header.next_data_offset.to_le_bytes());
203
204    // 0x28: created_at_epoch (8)
205    buf[40..48].copy_from_slice(&header.created_at_epoch.to_le_bytes());
206
207    // 0x30: modified_at_epoch (8)
208    buf[48..56].copy_from_slice(&header.modified_at_epoch.to_le_bytes());
209
210    // 0x38: reserved (72) - already zeroed
211
212    buf
213}
214
215pub fn decode_header(buf: &[u8; HEADER_SIZE_USIZE]) -> Result<GeoFileHeader> {
216    // Verify magic first
217    let magic: [u8; 8] = buf[0..8]
218        .try_into()
219        .map_err(|_| anyhow!("Magic slice has wrong size"))?;
220
221    if magic != FILE_MAGIC {
222        return Err(anyhow!(
223            "Invalid magic: expected {:?}, got {:?}",
224            FILE_MAGIC,
225            magic
226        ));
227    }
228
229    let version = u32::from_le_bytes(buf[8..12].try_into()?);
230    if version != FORMAT_VERSION {
231        return Err(anyhow!(
232            "Unsupported version: expected {}, got {}",
233            FORMAT_VERSION,
234            version
235        ));
236    }
237
238    Ok(GeoFileHeader {
239        magic,
240        version,
241        flags: u32::from_le_bytes(buf[12..16].try_into()?),
242        section_table_offset: u64::from_le_bytes(buf[16..24].try_into()?),
243        section_count: u64::from_le_bytes(buf[24..32].try_into()?),
244        next_data_offset: u64::from_le_bytes(buf[32..40].try_into()?),
245        created_at_epoch: u64::from_le_bytes(buf[40..48].try_into()?),
246        modified_at_epoch: u64::from_le_bytes(buf[48..56].try_into()?),
247        reserved: {
248            let mut arr = [0u8; 72];
249            arr.copy_from_slice(&buf[56..128]);
250            arr
251        },
252    })
253}
254
255fn encode_section_entry_name(name: &str) -> [u8; MAX_SECTION_NAME_LEN] {
256    let mut buf = [0u8; MAX_SECTION_NAME_LEN];
257    let name_bytes = name.as_bytes();
258    let len = name_bytes.len().min(MAX_SECTION_NAME_LEN);
259    buf[..len].copy_from_slice(&name_bytes[..len]);
260    buf
261}
262
263fn decode_section_entry_name(buf: &[u8]) -> Result<String> {
264    let len = buf.iter().position(|&b| b == 0).unwrap_or(buf.len());
265    String::from_utf8(buf[..len].to_vec()).context("Section name is not valid UTF-8")
266}
267
268pub fn encode_section_entry(entry: &SectionEntry) -> [u8; SECTION_ENTRY_SIZE_USIZE] {
269    let mut buf = [0u8; SECTION_ENTRY_SIZE_USIZE];
270
271    // 0x00: name (32) - zero-padded
272    buf[0..32].copy_from_slice(&encode_section_entry_name(&entry.name));
273
274    // 0x20: offset (8)
275    buf[32..40].copy_from_slice(&entry.offset.to_le_bytes());
276
277    // 0x28: length (8)
278    buf[40..48].copy_from_slice(&entry.length.to_le_bytes());
279
280    // 0x30: capacity (8)
281    buf[48..56].copy_from_slice(&entry.capacity.to_le_bytes());
282
283    // 0x38: flags (4)
284    buf[56..60].copy_from_slice(&entry.flags.to_le_bytes());
285
286    // 0x3C: checksum (4)
287    buf[60..64].copy_from_slice(&entry.checksum.to_le_bytes());
288
289    // 0x40: reserved (24) - already zeroed
290
291    buf
292}
293
294pub fn decode_section_entry(buf: &[u8; SECTION_ENTRY_SIZE_USIZE]) -> Result<SectionEntry> {
295    let name = decode_section_entry_name(&buf[0..32])?;
296
297    Ok(SectionEntry {
298        name,
299        offset: u64::from_le_bytes(buf[32..40].try_into()?),
300        length: u64::from_le_bytes(buf[40..48].try_into()?),
301        capacity: u64::from_le_bytes(buf[48..56].try_into()?),
302        flags: u32::from_le_bytes(buf[56..60].try_into()?),
303        checksum: u32::from_le_bytes(buf[60..64].try_into()?),
304    })
305}
306
307pub fn compute_checksum(data: &[u8]) -> u32 {
308    let mut hasher = Crc32Hasher::new();
309    hasher.update(data);
310    hasher.finalize()
311}
312
313// ============================================================================
314// SECTIONED STORAGE IMPLEMENTATION
315// ============================================================================
316
317impl SectionedStorage {
318    /// Create a new sectioned file
319    pub fn create(path: &Path) -> Result<Self> {
320        let mut file = std::fs::OpenOptions::new()
321            .read(true)
322            .write(true)
323            .create(true)
324            .truncate(true)
325            .open(path)
326            .context("Failed to create sectioned file")?;
327
328        let header = GeoFileHeader::default();
329        let header_bytes = encode_header(&header);
330        file.write_all(&header_bytes)
331            .context("Failed to write header")?;
332        file.sync_all().context("Failed to sync file")?;
333
334        Ok(Self {
335            file,
336            path: path.to_path_buf(),
337            header,
338            sections: BTreeMap::new(),
339            dirty: false,
340        })
341    }
342
343    /// Check if a file is a sectioned file without opening it
344    ///
345    /// This is a lightweight check that only reads the file header
346    /// to verify the magic number. It doesn't validate the entire file structure.
347    pub fn is_sectioned_file(path: &Path) -> bool {
348        let mut is_sectioned = false;
349        if path.exists() {
350            // Try to read just the header
351            if let Ok(mut file) = std::fs::File::open(path) {
352                let mut header_buf = [0u8; HEADER_SIZE_USIZE];
353                if file.read_exact(&mut header_buf).is_ok() {
354                    // Check magic number
355                    if let Ok(header) = decode_header(&header_buf) {
356                        is_sectioned = header.magic == FILE_MAGIC;
357                    }
358                }
359            }
360        }
361        is_sectioned
362    }
363
364    /// Open an existing sectioned file
365    ///
366    /// Reads header and section table, validates structure integrity.
367    /// Returns error if validation fails.
368    pub fn open(path: &Path) -> Result<Self> {
369        let mut file = std::fs::OpenOptions::new()
370            .read(true)
371            .write(true)
372            .open(path)
373            .context("Failed to open sectioned file")?;
374
375        // Read header
376        let mut header_buf = [0u8; HEADER_SIZE_USIZE];
377        file.read_exact(&mut header_buf)
378            .context("Failed to read header")?;
379        let header = decode_header(&header_buf)?;
380
381        // Read section table
382        let mut sections = BTreeMap::new();
383
384        if header.section_count > 0 {
385            file.seek(SeekFrom::Start(header.section_table_offset))
386                .context("Failed to seek to section table")?;
387
388            for _ in 0..header.section_count {
389                let mut entry_buf = [0u8; SECTION_ENTRY_SIZE_USIZE];
390                file.read_exact(&mut entry_buf)
391                    .context("Failed to read section entry")?;
392                let entry = decode_section_entry(&entry_buf)?;
393
394                sections.insert(
395                    entry.name.clone(),
396                    Section {
397                        name: entry.name,
398                        offset: entry.offset,
399                        length: entry.length,
400                        capacity: entry.capacity,
401                        flags: entry.flags,
402                        checksum: entry.checksum,
403                    },
404                );
405            }
406        }
407
408        let mut storage = Self {
409            file,
410            path: path.to_path_buf(),
411            header,
412            sections,
413            dirty: false,
414        };
415
416        // CRITICAL: Validate before returning
417        storage.validate()?;
418
419        Ok(storage)
420    }
421
422    /// Create a new section with IMMEDIATE physical reservation
423    ///
424    /// This method:
425    /// 1. Validates the name
426    /// 2. Gets current file length
427    /// 3. Computes allocation_base = max(next_data_offset, file_len)
428    /// 4. Allocates from allocation_base
429    /// 5. Physically extends file to reserve capacity
430    /// 6. Updates header.next_data_offset
431    ///
432    /// Fails if:
433    /// - Name is empty
434    /// - Name exceeds 32 UTF-8 bytes
435    /// - Section with same name exists
436    /// - Addition would overflow
437    /// - File extension fails
438    pub fn create_section(&mut self, name: &str, capacity: u64, flags: u32) -> Result<()> {
439        // Validate name: non-empty
440        if name.is_empty() {
441            return Err(anyhow!("Section name cannot be empty"));
442        }
443
444        // Validate name: UTF-8 byte length
445        let name_bytes = name.as_bytes();
446        if name_bytes.len() > MAX_SECTION_NAME_LEN {
447            return Err(anyhow!(
448                "Section name too long: {} bytes > {}",
449                name_bytes.len(),
450                MAX_SECTION_NAME_LEN
451            ));
452        }
453
454        // Check for duplicate
455        if self.sections.contains_key(name) {
456            return Err(anyhow!("Section '{}' already exists", name));
457        }
458
459        // CRITICAL: Get current file length (may be > next_data_offset due to table)
460        let file_len = self
461            .file
462            .metadata()
463            .context("Failed to get file metadata")?
464            .len();
465
466        // CRITICAL: Allocation must start after BOTH next_data_offset AND current EOF
467        let allocation_base = self.header.next_data_offset.max(file_len);
468
469        // Calculate new next_data_offset with overflow check
470        let new_next_data_offset = allocation_base.checked_add(capacity).ok_or_else(|| {
471            anyhow!(
472                "Data offset overflow: allocation_base {} + capacity {}",
473                allocation_base,
474                capacity
475            )
476        })?;
477
478        // Physically extend file to reserve capacity
479        self.file.set_len(new_next_data_offset).with_context(|| {
480            format!(
481                "Failed to reserve {} bytes for section '{}' at offset {}",
482                capacity, name, allocation_base
483            )
484        })?;
485
486        // Allocate from computed base
487        let offset = allocation_base;
488
489        // Update header to track new append position
490        self.header.next_data_offset = new_next_data_offset;
491
492        // Add section (length=0, checksum=0 until written)
493        self.sections.insert(
494            name.to_string(),
495            Section {
496                name: name.to_string(),
497                offset,
498                length: 0,
499                capacity,
500                flags,
501                checksum: 0,
502            },
503        );
504
505        self.dirty = true;
506        Ok(())
507    }
508
509    /// Write data to an existing section
510    ///
511    /// **TASK 4 CAPACITY BEHAVIOR**: Fails loudly with explicit error on overflow.
512    /// Never silently loses data.
513    ///
514    /// Fails if:
515    /// - Section doesn't exist
516    /// - data.len() > section.capacity (explicit capacity error)
517    pub fn write_section(&mut self, name: &str, data: &[u8]) -> Result<()> {
518        debug_print!(
519            "[WRITE_SECTION] Writing section '{}': {} bytes",
520            name,
521            data.len()
522        );
523        let section = self
524            .sections
525            .get(name)
526            .ok_or_else(|| anyhow!("Section '{}' not found", name))?;
527
528        if data.len() as u64 > section.capacity {
529            // TASK 4: Explicit capacity error with full context
530            return Err(anyhow!(
531                "Section '{}' overflow: attempted to write {} bytes, but capacity is {} bytes ({} bytes over limit)\n\
532                 Section details: offset={}, length={}, capacity={}\n\
533                 To fix: Increase section capacity during database creation or migrate to a larger capacity.",
534                name,
535                data.len(),
536                section.capacity,
537                data.len() as u64 - section.capacity,
538                section.offset,
539                section.length,
540                section.capacity
541            ));
542        }
543
544        let checksum = compute_checksum(data);
545
546        self.file
547            .seek(SeekFrom::Start(section.offset))
548            .context("Failed to seek to section")?;
549        self.file
550            .write_all(data)
551            .context("Failed to write section data")?;
552
553        // Update section metadata
554        if let Some(s) = self.sections.get_mut(name) {
555            s.length = data.len() as u64;
556            s.checksum = checksum;
557            debug_print!(
558                "[WRITE_SECTION] Updated section '{}' metadata: length={}, checksum={}",
559                name,
560                s.length,
561                s.checksum
562            );
563        }
564
565        self.dirty = true;
566        Ok(())
567    }
568
569    /// Read data from a section
570    ///
571    /// Fails if:
572    /// - Section doesn't exist
573    /// - Checksum mismatch
574    pub fn read_section(&mut self, name: &str) -> Result<Vec<u8>> {
575        let section = self
576            .sections
577            .get(name)
578            .ok_or_else(|| anyhow!("Section '{}' not found", name))?;
579
580        if section.length == 0 {
581            return Ok(Vec::new());
582        }
583
584        self.file
585            .seek(SeekFrom::Start(section.offset))
586            .context("Failed to seek to section")?;
587
588        let mut buffer = vec![0u8; section.length as usize];
589        self.file
590            .read_exact(&mut buffer)
591            .context("Failed to read section data")?;
592
593        // Verify checksum
594        let computed = compute_checksum(&buffer);
595        if computed != section.checksum {
596            return Err(anyhow!(
597                "Checksum mismatch for section '{}': stored {}, computed {}",
598                name,
599                section.checksum,
600                computed
601            ));
602        }
603
604        Ok(buffer)
605    }
606
607    /// Get section metadata without reading data
608    pub fn get_section(&self, name: &str) -> Option<&Section> {
609        self.sections.get(name)
610    }
611
612    /// List all sections
613    pub fn list_sections(&self) -> Vec<Section> {
614        self.sections.values().cloned().collect()
615    }
616
617    /// Get the number of sections
618    pub fn section_count(&self) -> usize {
619        self.sections.len()
620    }
621
622    /// Get the file path
623    pub fn path(&self) -> &Path {
624        &self.path
625    }
626
627    /// Get a reference to the header
628    pub fn header(&self) -> &GeoFileHeader {
629        &self.header
630    }
631
632    /// Flush section table to EOF
633    ///
634    /// Phase A behavior:
635    /// - Appends a FRESH section table at current EOF
636    /// - Updates header.section_table_offset to point to new table
637    /// - Old section tables become dead bytes in file
638    /// - File size grows with each flush
639    ///
640    /// File layout after flush():
641    /// ```text
642    /// [0..128)                    - header
643    /// [128..next_data_offset)      - section payload area
644    /// [section_table_offset..EOF)  - current section table
645    /// ```
646    ///
647    /// Because `create_section()` guarantees EOF >= next_data_offset,
648    /// the section table is always placed after all section payloads.
649    ///
650    /// Compaction (reclaiming dead tables) is deferred to a later phase.
651    pub fn flush(&mut self) -> Result<()> {
652        let now = std::time::SystemTime::now()
653            .duration_since(std::time::UNIX_EPOCH)
654            .unwrap_or_default()
655            .as_secs();
656
657        // Seek to end of file for new table
658        let table_offset = self
659            .file
660            .seek(SeekFrom::End(0))
661            .context("Failed to seek to EOF for section table")?;
662
663        debug_print!(
664            "[FLUSH_DEBUG] Writing section table at offset {}",
665            table_offset
666        );
667        debug_print!("[FLUSH_DEBUG] Section count: {}", self.sections.len());
668
669        // Write all section entries
670        for section in self.sections.values() {
671            debug_print!(
672                "[FLUSH_DEBUG]   Section {}: offset={}, length={}",
673                section.name,
674                section.offset,
675                section.length
676            );
677            let entry = SectionEntry {
678                name: section.name.clone(),
679                offset: section.offset,
680                length: section.length,
681                capacity: section.capacity,
682                flags: section.flags,
683                checksum: section.checksum,
684            };
685            let entry_bytes = encode_section_entry(&entry);
686            self.file
687                .write_all(&entry_bytes)
688                .context("Failed to write section entry")?;
689        }
690
691        // Update header
692        self.header.section_table_offset = table_offset;
693        self.header.section_count = self.sections.len() as u64;
694        self.header.modified_at_epoch = now;
695
696        // Write header at offset 0
697        self.file
698            .seek(SeekFrom::Start(0))
699            .context("Failed to seek to header")?;
700        let header_bytes = encode_header(&self.header);
701        self.file
702            .write_all(&header_bytes)
703            .context("Failed to write header")?;
704
705        self.file.sync_all().context("Failed to sync file")?;
706
707        self.dirty = false;
708        Ok(())
709    }
710
711    /// Validate file structure integrity
712    ///
713    /// IMPORTANT: If `self.dirty == true`, returns an error.
714    /// Unflushed state cannot be validated against disk.
715    pub fn validate(&mut self) -> Result<()> {
716        // Cannot validate dirty/unflushed state
717        if self.dirty {
718            return Err(anyhow!(
719                "cannot validate dirty/unflushed state; flush first"
720            ));
721        }
722
723        // 1. File metadata basics
724        let metadata = self
725            .file
726            .metadata()
727            .context("Failed to get file metadata")?;
728        let file_len = metadata.len();
729
730        // 2. File must be at least header size
731        if file_len < HEADER_SIZE {
732            return Err(anyhow!(
733                "File too small: {} < header size {}",
734                file_len,
735                HEADER_SIZE
736            ));
737        }
738
739        // 3. Re-read header from disk to detect corruption
740        let mut header_buf = [0u8; HEADER_SIZE_USIZE];
741        self.file
742            .seek(SeekFrom::Start(0))
743            .context("Failed to seek to header for validation")?;
744        self.file
745            .read_exact(&mut header_buf)
746            .context("Failed to read header for validation")?;
747        let disk_header = decode_header(&header_buf)?;
748
749        // 4. Copy validated header to self
750        self.header = disk_header.clone();
751
752        // 5. Verify section_table_offset is in bounds
753        if disk_header.section_table_offset < HEADER_SIZE {
754            return Err(anyhow!(
755                "Section table offset {} before data area {}",
756                disk_header.section_table_offset,
757                HEADER_SIZE
758            ));
759        }
760        if disk_header.section_table_offset > file_len {
761            return Err(anyhow!(
762                "Section table offset {} beyond file length {}",
763                disk_header.section_table_offset,
764                file_len
765            ));
766        }
767
768        // 6. PHYSICAL RESERVATION CHECK
769        if file_len < disk_header.next_data_offset {
770            return Err(anyhow!(
771                "File truncated: length {} < next_data_offset {} (physical reservation missing)",
772                file_len,
773                disk_header.next_data_offset
774            ));
775        }
776
777        // 7. Verify next_data_offset <= section_table_offset
778        if disk_header.next_data_offset > disk_header.section_table_offset {
779            return Err(anyhow!(
780                "Data area overlaps table: next_data_offset {} > section_table_offset {}",
781                disk_header.next_data_offset,
782                disk_header.section_table_offset
783            ));
784        }
785
786        // 8. Verify section table fits in file
787        if disk_header.section_count > 0 {
788            let table_size = disk_header
789                .section_count
790                .checked_mul(SECTION_ENTRY_SIZE)
791                .ok_or_else(|| anyhow!("Section count overflow"))?;
792            let table_end = disk_header
793                .section_table_offset
794                .checked_add(table_size)
795                .ok_or_else(|| anyhow!("Section table end overflow"))?;
796            if table_end > file_len {
797                return Err(anyhow!(
798                    "Section table extends beyond file: offset {} + size {} = {} > length {}",
799                    disk_header.section_table_offset,
800                    table_size,
801                    table_end,
802                    file_len
803                ));
804            }
805        }
806
807        // 9. Verify section names are non-empty
808        for name in self.sections.keys() {
809            if name.is_empty() {
810                return Err(anyhow!("Section name is empty"));
811            }
812        }
813
814        // 10. Verify each section (sorted by offset for overlap detection)
815        let mut prev_end = HEADER_SIZE;
816        let mut sorted_sections: Vec<_> = self.sections.iter().collect();
817        sorted_sections.sort_by_key(|(_, section)| section.offset);
818        for (name, section) in sorted_sections {
819            // 10a. Offset in data area (before current table)
820            if section.offset < HEADER_SIZE {
821                return Err(anyhow!(
822                    "Section '{}' offset {} before data area {}",
823                    name,
824                    section.offset,
825                    HEADER_SIZE
826                ));
827            }
828            if section.offset >= disk_header.section_table_offset {
829                return Err(anyhow!(
830                    "Section '{}' offset {} at or after current table {}",
831                    name,
832                    section.offset,
833                    disk_header.section_table_offset
834                ));
835            }
836
837            // 10b. capacity >= length
838            if section.capacity < section.length {
839                return Err(anyhow!(
840                    "Section '{}' capacity {} < length {}",
841                    name,
842                    section.capacity,
843                    section.length
844                ));
845            }
846
847            // 10c. Section fits before current table
848            let section_end = section
849                .offset
850                .checked_add(section.capacity)
851                .ok_or_else(|| anyhow!("Section '{}' end overflow", name))?;
852            if section_end > disk_header.section_table_offset {
853                return Err(anyhow!(
854                    "Section '{}' (offset {} + capacity {} = {}) extends beyond current table start {}",
855                    name,
856                    section.offset,
857                    section.capacity,
858                    section_end,
859                    disk_header.section_table_offset
860                ));
861            }
862
863            // 10d. No overlap with previous section
864            if section.offset < prev_end {
865                return Err(anyhow!(
866                    "Section '{}' (offset {}) overlaps previous section (ends at {})",
867                    name,
868                    section.offset,
869                    prev_end
870                ));
871            }
872            prev_end = section_end;
873
874            // 10e. Verify checksum only if length > 0
875            if section.length > 0 {
876                self.file
877                    .seek(SeekFrom::Start(section.offset))
878                    .context("Failed to seek to section for checksum validation")?;
879                let mut buf = vec![0u8; section.length as usize];
880                self.file
881                    .read_exact(&mut buf)
882                    .context("Failed to read section for checksum validation")?;
883                let computed = compute_checksum(&buf);
884                if computed != section.checksum {
885                    return Err(anyhow!(
886                        "Checksum mismatch for section '{}': stored {}, computed {}",
887                        name,
888                        section.checksum,
889                        computed
890                    ));
891                }
892            }
893        }
894
895        // 11. Verify next_data_offset doesn't overlap last section
896        if disk_header.next_data_offset < prev_end {
897            return Err(anyhow!(
898                "next_data_offset {} overlaps last section (ends at {})",
899                disk_header.next_data_offset,
900                prev_end
901            ));
902        }
903
904        // 12. Special case: empty file
905        if disk_header.section_count == 0 && disk_header.section_table_offset != HEADER_SIZE {
906            return Err(anyhow!(
907                "Empty file: section_table_offset should be {}, got {}",
908                HEADER_SIZE,
909                disk_header.section_table_offset
910            ));
911        }
912
913        Ok(())
914    }
915
916    /// Validate that required sections exist
917    pub fn validate_required_sections(&self, required: &[&str]) -> Result<()> {
918        for name in required {
919            if !self.sections.contains_key(*name) {
920                return Err(anyhow!("Required section '{}' is missing", name));
921            }
922        }
923        Ok(())
924    }
925
926    /// Resize a section to a new (larger) capacity
927    ///
928    /// This method:
929    /// 1. Reads the current section data
930    /// 2. Creates a new section with the larger capacity at EOF
931    /// 3. Copies the data to the new location
932    /// 4. Removes the old section (becomes dead space)
933    /// 5. Flushes to disk
934    ///
935    /// Fails if:
936    /// - Section doesn't exist
937    /// - New capacity is smaller than current data length
938    /// - Read/write operations fail
939    ///
940    /// Note: The old section's space becomes dead space in the file.
941    /// A future compaction operation could reclaim this space.
942    pub fn resize_section(&mut self, name: &str, new_capacity: u64) -> Result<()> {
943        let section = self
944            .sections
945            .get(name)
946            .ok_or_else(|| anyhow!("Section '{}' not found", name))?;
947
948        if new_capacity < section.length {
949            return Err(anyhow!(
950                "Cannot resize section '{}' to {} bytes: current data length is {} bytes",
951                name,
952                new_capacity,
953                section.length
954            ));
955        }
956
957        if new_capacity != section.capacity {
958            debug_print!(
959                "[RESIZE_SECTION] Resizing '{}' from {} to {} bytes",
960                name,
961                section.capacity,
962                new_capacity
963            );
964
965            // Clone needed fields before any mutable operations
966            let (offset, length, _capacity, flags, checksum) = (
967                section.offset,
968                section.length,
969                section.capacity,
970                section.flags,
971                section.checksum,
972            );
973
974            // Read current data
975            let current_data = if length > 0 {
976                self.file
977                    .seek(SeekFrom::Start(offset))
978                    .context("Failed to seek to section for resize")?;
979                let mut buffer = vec![0u8; length as usize];
980                self.file
981                    .read_exact(&mut buffer)
982                    .context("Failed to read section data for resize")?;
983                buffer
984            } else {
985                Vec::new()
986            };
987
988            // Get current file length and next_data_offset
989            let file_len = self
990                .file
991                .metadata()
992                .context("Failed to get file metadata")?
993                .len();
994            let allocation_base = self.header.next_data_offset.max(file_len);
995
996            // Calculate new next_data_offset
997            let new_next_data_offset = allocation_base
998                .checked_add(new_capacity)
999                .ok_or_else(|| anyhow!("Data offset overflow during section resize"))?;
1000
1001            // Physically extend file to reserve new capacity
1002            self.file
1003                .set_len(new_next_data_offset)
1004                .context("Failed to extend file for section resize")?;
1005
1006            // Create new section at end of file
1007            let new_offset = allocation_base;
1008
1009            // Write data to new location
1010            if !current_data.is_empty() {
1011                self.file
1012                    .seek(SeekFrom::Start(new_offset))
1013                    .context("Failed to seek to new section location")?;
1014                self.file
1015                    .write_all(&current_data)
1016                    .context("Failed to write data to new section location")?;
1017            }
1018
1019            // Update section metadata (remove old, add new)
1020            self.sections.remove(name);
1021            self.sections.insert(
1022                name.to_string(),
1023                Section {
1024                    name: name.to_string(),
1025                    offset: new_offset,
1026                    length,
1027                    capacity: new_capacity,
1028                    flags,
1029                    checksum,
1030                },
1031            );
1032
1033            // Update header
1034            self.header.next_data_offset = new_next_data_offset;
1035            self.dirty = true;
1036
1037            debug_print!(
1038                "[RESIZE_SECTION] Section '{}' moved from offset {} to {}",
1039                name,
1040                offset,
1041                new_offset
1042            );
1043        }
1044
1045        Ok(())
1046    }
1047}
1048
1049// ============================================================================
1050// UNIT TESTS
1051// ============================================================================
1052
1053#[cfg(test)]
1054mod tests {
1055    use super::*;
1056
1057    // === HEADER ENCODING/DECODING TESTS ===
1058
1059    #[test]
1060    fn test_header_exactly_128_bytes() {
1061        let header = GeoFileHeader::default();
1062        let encoded = encode_header(&header);
1063        assert_eq!(encoded.len(), 128, "Header must be exactly 128 bytes");
1064    }
1065
1066    #[test]
1067    fn test_header_reserved_field_size() {
1068        let header = GeoFileHeader::default();
1069        assert_eq!(header.reserved.len(), 72);
1070    }
1071
1072    #[test]
1073    fn test_header_roundtrip_preserves_all_fields() {
1074        let header = GeoFileHeader {
1075            flags: 0x12345678,
1076            section_table_offset: 0x1000,
1077            section_count: 5,
1078            next_data_offset: 0x2000,
1079            created_at_epoch: 1234567890,
1080            modified_at_epoch: 1234567900,
1081            ..Default::default()
1082        };
1083
1084        let encoded = encode_header(&header);
1085        let decoded = decode_header(&encoded).unwrap();
1086
1087        assert_eq!(decoded, header);
1088    }
1089
1090    #[test]
1091    fn test_decode_header_reserved_slice_correct() {
1092        let mut buf = [0u8; 128];
1093        buf[0..8].copy_from_slice(&FILE_MAGIC);
1094        buf[8..12].copy_from_slice(&1u32.to_le_bytes());
1095        // Set some bytes in reserved area
1096        buf[100] = 0x42;
1097        buf[127] = 0xFF;
1098
1099        let decoded = decode_header(&buf).unwrap();
1100
1101        // Reserved should capture bytes 56..127
1102        assert_eq!(decoded.reserved[100 - 56], 0x42);
1103        assert_eq!(decoded.reserved[127 - 56], 0xFF);
1104    }
1105
1106    #[test]
1107    fn test_invalid_magic_rejected() {
1108        let mut buf = [0u8; 128];
1109        buf[0..8].copy_from_slice(b"BADMAGIC");
1110        assert!(decode_header(&buf).is_err());
1111    }
1112
1113    // === SECTION ENTRY ENCODING/DECODING TESTS ===
1114
1115    #[test]
1116    fn test_section_entry_exactly_64_bytes() {
1117        let entry = SectionEntry {
1118            name: "test".to_string(),
1119            offset: 0,
1120            length: 0,
1121            capacity: 0,
1122            flags: 0,
1123            checksum: 0,
1124        };
1125        let encoded = encode_section_entry(&entry);
1126        assert_eq!(encoded.len(), 64, "Section entry must be exactly 64 bytes");
1127    }
1128
1129    #[test]
1130    fn test_section_entry_roundtrip() {
1131        let entry = SectionEntry {
1132            name: "test_section".to_string(),
1133            offset: 1024,
1134            length: 512,
1135            capacity: 1024,
1136            flags: 0x12345678,
1137            checksum: 0xABCDEF01,
1138        };
1139        let encoded = encode_section_entry(&entry);
1140        assert_eq!(encoded.len(), 64);
1141        let decoded = decode_section_entry(&encoded).unwrap();
1142        assert_eq!(entry.name, decoded.name);
1143        assert_eq!(entry.offset, decoded.offset);
1144        assert_eq!(entry.length, decoded.length);
1145        assert_eq!(entry.capacity, decoded.capacity);
1146        assert_eq!(entry.flags, decoded.flags);
1147        assert_eq!(entry.checksum, decoded.checksum);
1148    }
1149
1150    #[test]
1151    fn test_section_name_encoding() {
1152        let name = "cfg_data";
1153        let encoded = encode_section_entry_name(name);
1154        let decoded = decode_section_entry_name(&encoded).unwrap();
1155        assert_eq!(name, decoded);
1156    }
1157
1158    // === CHECKSUM TESTS ===
1159
1160    #[test]
1161    fn test_checksum_deterministic() {
1162        let data = b"test data";
1163        let crc1 = compute_checksum(data);
1164        let crc2 = compute_checksum(data);
1165        assert_eq!(crc1, crc2);
1166    }
1167
1168    #[test]
1169    fn test_checksum_detects_corruption() {
1170        let data1 = b"test data";
1171        let data2 = b"test datb"; // One byte changed
1172        assert_ne!(compute_checksum(data1), compute_checksum(data2));
1173    }
1174
1175    #[test]
1176    fn test_checksum_empty() {
1177        let data = b"";
1178        let crc = compute_checksum(data);
1179        // CRC32 of empty string is a known value
1180        assert_eq!(crc, 0);
1181    }
1182
1183    // === NAME VALIDATION TESTS ===
1184
1185    #[test]
1186    fn test_section_name_32_bytes_accepted() {
1187        let name_32_bytes = "12345678901234567890123456789012"; // 32 ASCII bytes
1188        assert_eq!(name_32_bytes.len(), 32);
1189
1190        let encoded = encode_section_entry_name(name_32_bytes);
1191        let decoded = decode_section_entry_name(&encoded).unwrap();
1192        assert_eq!(name_32_bytes, decoded);
1193    }
1194
1195    #[test]
1196    fn test_section_name_encoding_truncates_at_32() {
1197        let name_33_bytes = "123456789012345678901234567890123"; // 33 ASCII bytes
1198        assert_eq!(name_33_bytes.len(), 33);
1199
1200        let encoded = encode_section_entry_name(name_33_bytes);
1201        let decoded = decode_section_entry_name(&encoded).unwrap();
1202        assert_eq!(decoded.len(), 32); // Truncated
1203        assert_eq!(decoded, "12345678901234567890123456789012");
1204    }
1205}
geographdb_core/storage/sectioned.rs

geographdb_core/storage/
sectioned.rs
