open_geocode/
records.rs

1//! Compact record store.
2//!
3//! Records are written in their final order (the builder sorts them along a
4//! Hilbert curve) and packed into blocks of [`BLOCK_RECORDS`]. A block starts
5//! with an absolute base coordinate; every record in it stores its display
6//! point as a small zigzag delta from that base, then its fields as varints and
7//! string-table ids, then any line geometry as a delta chain. Finding record
8//! `N` means jumping to block `N / 64` through the block index, then to the
9//! record through the block's table of body end offsets.
10//!
11//! ```text
12//! records/index    record_count u64, then (block_count + 1) u64 block offsets
13//! records/blocks   blocks: base_lon i32 | base_lat i32 | n x u32 body end | bodies
14//! records/strings  segment_count u64 | per segment: first record u64, first string u64
15//!                  | string_count u64 | string_count x u64 end offset | UTF-8 bytes
16//!
17//! body  tag u8        layer (4 bits) | centroid (1) | line geometry (1) | source kind (2)
18//!       dlon, dlat    zigzag varints from the block base
19//!       context       varint, 0 = none, else (tuple_id + 1) << 1 | ambiguous
20//!       source        zigzag OSM object id, or the address count for postcodes
21//!       fields        per layer, strings as varint ids local to the record's segment
22//!       geometry      line strings only: varint count, then zigzag deltas
23//! ```
24//!
25//! Strings are interned per segment: the build keeps one segment's dictionary
26//! in memory, not every distinct string of the input. A segment ends after
27//! [`SEGMENT_RECORDS`] records or once its dictionary uses its share of the
28//! build budget, whichever comes first. Records are in Hilbert order, so a
29//! segment covers one area and most of its strings are its own; only widely
30//! shared ones (countries, regions) repeat across segments.
31
32use std::{
33    collections::HashMap,
34    fs::File,
35    io::{BufWriter, Write},
36    path::PathBuf,
37    sync::Arc,
38};
39
40use anyhow::{Context, Result, bail};
41use geojson::{Geometry, GeometryValue};
42use serde::Serialize;
43use serde_json::{Value, json};
44
45use crate::{
46    container::{Bytes, Container, ContainerWriter},
47    extsort::Scratch,
48    memory::{MemoryBudget, Reservation},
49    pack::RecordId,
50    record::{
51        AddressComponents, AddressRecord, DERIVED_FROM_ADDRESS_RECORDS, DerivedSourceProvenance,
52        InterpolationAddressComponents, InterpolationRange, InterpolationRecord, Layer,
53        LocationPrecision, OsmObjectType, PlaceRecord, PostcodeRecord, Record, SourceProvenance,
54        StreetRecord, point_geometry,
55    },
56    util::codec::{get_i64, get_u8, get_u32, get_u64, put_i64, put_u64, read_u32_le, read_u64_le},
57};
58
59pub const SECTION_INDEX: &str = "records/index";
60pub const SECTION_BLOCKS: &str = "records/blocks";
61pub const SECTION_STRINGS: &str = "records/strings";
62pub const RECORDS_VERSION: u32 = 4;
63
64pub const BLOCK_RECORDS: u64 = 64;
65/// Approximate bytes a dictionary entry costs beyond its text: the boxed
66/// key, the id and the hash table slot.
67const DICTIONARY_ENTRY_OVERHEAD: usize = 48;
68/// Records per string segment.
69pub const SEGMENT_RECORDS: u64 = 1 << 20;
70/// Share of the build budget the string dictionary may use.
71const DICTIONARY_BUDGET_FRACTION: usize = 8;
72const COORDINATE_SCALE: f64 = 10_000_000.0;
73const BLOCK_HEADER_BYTES: usize = 8;
74
75const TAG_LAYER_MASK: u8 = 0x0f;
76const TAG_CENTROID: u8 = 0x10;
77const TAG_LINE: u8 = 0x20;
78const TAG_SOURCE_SHIFT: u8 = 6;
79
80#[derive(Debug, Clone, Copy, PartialEq, Eq)]
81enum SourceKind {
82    Node = 0,
83    Way = 1,
84    Relation = 2,
85    Derived = 3,
86}
87
88/// Admin context reference stored in a record body.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub struct ContextRef {
91    pub tuple_id: u32,
92    pub ambiguous: bool,
93}
94
95#[derive(Debug, Clone, Serialize, PartialEq)]
96pub struct RecordSummary {
97    pub id: String,
98    pub layer: String,
99    pub label: String,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub point: Option<RecordPoint>,
102    pub source: RecordSource,
103}
104
105#[derive(Debug, Clone, Copy, Serialize, PartialEq)]
106pub struct RecordPoint {
107    pub lon: f64,
108    pub lat: f64,
109    pub precision: RecordPointPrecision,
110}
111
112#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
113#[serde(rename_all = "snake_case")]
114pub enum RecordPointPrecision {
115    Point,
116    Centroid,
117    RepresentativePoint,
118}
119
120#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
121pub struct RecordSource {
122    pub dataset: String,
123    #[serde(skip_serializing_if = "Option::is_none")]
124    pub object_type: Option<OsmObjectType>,
125    #[serde(skip_serializing_if = "Option::is_none")]
126    pub object_id: Option<i64>,
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub derived_from: Option<String>,
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub record_count: Option<u64>,
131}
132
133#[derive(Debug, Clone, PartialEq)]
134pub struct ContextRecord {
135    pub id: String,
136    pub layer: String,
137    pub label: String,
138    pub name: String,
139    pub postcode: Option<String>,
140    pub point: Option<RecordPoint>,
141}
142
143/// Global string ids `first..end` belong to one segment.
144#[derive(Debug, Clone, Copy, PartialEq, Eq)]
145struct StringSegment {
146    first: u64,
147    end: u64,
148}
149
150/// The fixed part of a record: enough for spatial filtering without touching
151/// strings or geometry.
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153pub struct RecordHeader {
154    pub layer: Layer,
155    strings: StringSegment,
156    pub lon_e7: i32,
157    pub lat_e7: i32,
158    pub context: Option<ContextRef>,
159    centroid: bool,
160    line: bool,
161    source_kind: SourceKind,
162    source_value: u64,
163}
164
165impl RecordHeader {
166    pub fn lon(&self) -> f64 {
167        dequantize(self.lon_e7)
168    }
169
170    pub fn lat(&self) -> f64 {
171        dequantize(self.lat_e7)
172    }
173
174    pub fn has_line(&self) -> bool {
175        self.line
176    }
177
178    fn point(&self) -> RecordPoint {
179        let precision = if self.line {
180            RecordPointPrecision::RepresentativePoint
181        } else if self.layer == Layer::Address && !self.centroid {
182            RecordPointPrecision::Point
183        } else {
184            RecordPointPrecision::Centroid
185        };
186        RecordPoint {
187            lon: self.lon(),
188            lat: self.lat(),
189            precision,
190        }
191    }
192
193    fn osm_source(&self) -> Result<(OsmObjectType, i64)> {
194        let object_type = match self.source_kind {
195            SourceKind::Node => OsmObjectType::Node,
196            SourceKind::Way => OsmObjectType::Way,
197            SourceKind::Relation => OsmObjectType::Relation,
198            SourceKind::Derived => bail!("derived record has no OSM source"),
199        };
200        Ok((object_type, crate::util::codec::unzigzag(self.source_value)))
201    }
202
203    fn source(&self) -> Result<RecordSource> {
204        if self.source_kind == SourceKind::Derived {
205            return Ok(RecordSource {
206                dataset: "osm".into(),
207                object_type: None,
208                object_id: None,
209                derived_from: Some(DERIVED_FROM_ADDRESS_RECORDS.into()),
210                record_count: Some(self.source_value),
211            });
212        }
213        let (object_type, object_id) = self.osm_source()?;
214        Ok(RecordSource {
215            dataset: "osm".into(),
216            object_type: Some(object_type),
217            object_id: Some(object_id),
218            derived_from: None,
219            record_count: None,
220        })
221    }
222
223    fn provenance(&self) -> Result<SourceProvenance> {
224        let (object_type, object_id) = self.osm_source()?;
225        Ok(SourceProvenance::osm(object_type, object_id))
226    }
227}
228
229pub struct RecordsWriter {
230    blocks: BufWriter<File>,
231    blocks_path: PathBuf,
232    blocks_len: u64,
233    /// Start of every block, streamed to scratch.
234    block_offsets: BufWriter<File>,
235    block_offsets_path: PathBuf,
236    /// Bodies of the open block and where each one ends.
237    block: Vec<u8>,
238    block_ends: Vec<u32>,
239    block_base: (i32, i32),
240    record_count: u64,
241    strings: StringTable,
242    body: Vec<u8>,
243}
244
245/// Strings of the current segment. Bytes and end offsets stream to scratch
246/// files; only the current segment's dictionary is held in memory.
247struct StringTable {
248    ids: HashMap<Box<str>, u32>,
249    /// The dictionary's memory, held against the build budget. It cannot
250    /// spill, but it is bounded by one segment and released at the next.
251    memory: Reservation,
252    /// Dictionary bytes after which the next record starts a new segment.
253    memory_share: usize,
254    segment_records: u64,
255    /// First record and first string of every segment.
256    segments: Vec<(u64, u64)>,
257    count: u64,
258    bytes_len: u64,
259    bytes: BufWriter<File>,
260    bytes_path: PathBuf,
261    ends: BufWriter<File>,
262    ends_path: PathBuf,
263}
264
265impl StringTable {
266    /// Segment-local id of `value`.
267    fn id(&mut self, value: &str) -> Result<u32> {
268        if let Some(&id) = self.ids.get(value) {
269            return Ok(id);
270        }
271        let id = u32::try_from(self.ids.len()).context("string segment is full")?;
272        self.bytes.write_all(value.as_bytes())?;
273        self.bytes_len += value.len() as u64;
274        self.ends.write_all(&self.bytes_len.to_le_bytes())?;
275        self.count += 1;
276        self.ids.insert(value.into(), id);
277        let bytes = value.len() + DICTIONARY_ENTRY_OVERHEAD;
278        if !self.memory.try_grow(bytes) {
279            self.memory.force_grow(bytes);
280        }
281        Ok(id)
282    }
283
284    fn start_segment(&mut self, first_record: u64) {
285        self.ids = HashMap::new();
286        self.memory.release_all();
287        self.segments.push((first_record, self.count));
288    }
289
290    fn segment_full(&self, record_id: u64) -> bool {
291        let Some((first_record, _)) = self.segments.last() else {
292            return true;
293        };
294        record_id - first_record >= self.segment_records || self.memory.bytes() >= self.memory_share
295    }
296}
297
298impl RecordsWriter {
299    pub fn create(scratch: &Arc<Scratch>, budget: &Arc<MemoryBudget>) -> Result<Self> {
300        let blocks_path = scratch.path().join("records-blocks.bin");
301        let block_offsets_path = scratch.path().join("records-block-offsets.bin");
302        let bytes_path = scratch.path().join("records-strings.bin");
303        let ends_path = scratch.path().join("records-string-ends.bin");
304        let create = |path: &PathBuf| -> Result<BufWriter<File>> {
305            Ok(BufWriter::with_capacity(
306                1 << 20,
307                File::create(path)
308                    .with_context(|| format!("failed to create {}", path.display()))?,
309            ))
310        };
311        Ok(Self {
312            blocks: create(&blocks_path)?,
313            blocks_path,
314            blocks_len: 0,
315            block_offsets: create(&block_offsets_path)?,
316            block_offsets_path,
317            block: Vec::new(),
318            block_ends: Vec::new(),
319            block_base: (0, 0),
320            record_count: 0,
321            strings: StringTable {
322                ids: HashMap::new(),
323                memory: Reservation::empty(budget),
324                memory_share: budget.limit() / DICTIONARY_BUDGET_FRACTION,
325                segment_records: SEGMENT_RECORDS,
326                segments: Vec::new(),
327                count: 0,
328                bytes_len: 0,
329                bytes: create(&bytes_path)?,
330                bytes_path,
331                ends: create(&ends_path)?,
332                ends_path,
333            },
334            body: Vec::new(),
335        })
336    }
337
338    pub fn record_count(&self) -> u64 {
339        self.record_count
340    }
341
342    pub fn string_count(&self) -> u64 {
343        self.strings.count
344    }
345
346    #[cfg(test)]
347    fn with_segment_records(mut self, segment_records: u64) -> Self {
348        // Applies from the next segment; the writer has not started one yet.
349        self.strings.segment_records = segment_records;
350        self
351    }
352
353    /// Strings in the current segment's in-memory dictionary.
354    pub fn resident_strings(&self) -> usize {
355        self.strings.ids.len()
356    }
357
358    pub fn write(&mut self, record: &Record, context: Option<ContextRef>) -> Result<RecordId> {
359        let [lon, lat] = record
360            .display_point()
361            .with_context(|| format!("record {} has no finite display point", record.id()))?;
362        let point = (quantize(lon)?, quantize(lat)?);
363        if self.strings.segment_full(self.record_count) {
364            self.strings.start_segment(self.record_count);
365        }
366        if self.record_count % BLOCK_RECORDS == 0 {
367            self.flush_block()?;
368            self.block_base = point;
369        }
370
371        let mut body = std::mem::take(&mut self.body);
372        body.clear();
373        self.encode_body(&mut body, record, point, context)?;
374        self.block.extend_from_slice(&body);
375        self.block_ends
376            .push(u32::try_from(self.block.len()).context("record block exceeds 4 GiB")?);
377        self.body = body;
378
379        let id = self.record_count;
380        self.record_count += 1;
381        Ok(id)
382    }
383
384    fn encode_body(
385        &mut self,
386        out: &mut Vec<u8>,
387        record: &Record,
388        point: (i32, i32),
389        context: Option<ContextRef>,
390    ) -> Result<()> {
391        let line = line_coordinates(record.geometry())?;
392        let (source_kind, source_value) = match record {
393            Record::Address(record) => osm_source(&record.source),
394            Record::Interpolation(record) => osm_source(&record.source),
395            Record::Street(record) => osm_source(&record.source),
396            Record::Place(_, record) => osm_source(&record.source),
397            Record::Postcode(record) => (SourceKind::Derived, record.source.record_count),
398        };
399        let centroid = matches!(record, Record::Address(address) if address.location_precision == LocationPrecision::Centroid);
400        let mut tag = layer_code(record.layer()) | (source_kind as u8) << TAG_SOURCE_SHIFT;
401        if centroid {
402            tag |= TAG_CENTROID;
403        }
404        if line.is_some() {
405            tag |= TAG_LINE;
406        }
407        out.push(tag);
408        put_i64(out, i64::from(point.0) - i64::from(self.block_base.0));
409        put_i64(out, i64::from(point.1) - i64::from(self.block_base.1));
410        put_u64(
411            out,
412            context.map_or(0, |context| {
413                (u64::from(context.tuple_id) + 1) << 1 | u64::from(context.ambiguous)
414            }),
415        );
416        put_u64(out, source_value);
417
418        match record {
419            Record::Address(record) => {
420                let address = &record.address;
421                put_u64(out, u64::from(self.strings.id(&address.number)?));
422                self.put_optional_strings(
423                    out,
424                    &[
425                        address.street.as_deref(),
426                        address.place.as_deref(),
427                        address.unit.as_deref(),
428                        address.locality.as_deref(),
429                        address.region.as_deref(),
430                        address.postcode.as_deref(),
431                        address.country.as_deref(),
432                    ],
433                )?;
434            }
435            Record::Interpolation(record) => {
436                let address = &record.address;
437                self.put_optional_strings(
438                    out,
439                    &[
440                        address.street.as_deref(),
441                        address.place.as_deref(),
442                        address.locality.as_deref(),
443                        address.region.as_deref(),
444                        address.postcode.as_deref(),
445                        address.country.as_deref(),
446                    ],
447                )?;
448                let range = &record.interpolation;
449                put_u64(out, u64::from(self.strings.id(&range.kind)?));
450                put_u64(out, u64::from(range.start));
451                put_u64(out, u64::from(range.end.wrapping_sub(range.start)));
452                put_u64(out, u64::from(range.step));
453                put_i64(out, record.anchor_node_ids[0]);
454                put_i64(out, record.anchor_node_ids[1]);
455            }
456            Record::Street(record) => {
457                put_u64(out, u64::from(self.strings.id(&record.name)?));
458            }
459            Record::Postcode(record) => {
460                if record.source.derived_from != DERIVED_FROM_ADDRESS_RECORDS {
461                    bail!("unsupported postcode source {}", record.source.derived_from);
462                }
463                put_u64(out, u64::from(self.strings.id(&record.postcode)?));
464            }
465            Record::Place(_, record) => {
466                put_u64(out, u64::from(self.strings.id(&record.name)?));
467                put_u64(out, u64::from(self.strings.id(&record.place_type)?));
468            }
469        }
470
471        if let Some(line) = line {
472            put_u64(out, line.len() as u64);
473            let mut previous = point;
474            for position in line {
475                put_i64(out, i64::from(position.0) - i64::from(previous.0));
476                put_i64(out, i64::from(position.1) - i64::from(previous.1));
477                previous = position;
478            }
479        }
480        Ok(())
481    }
482
483    fn put_optional_strings(&mut self, out: &mut Vec<u8>, values: &[Option<&str>]) -> Result<()> {
484        let mut present = 0u8;
485        for (bit, value) in values.iter().enumerate() {
486            if value.is_some() {
487                present |= 1 << bit;
488            }
489        }
490        out.push(present);
491        for value in values.iter().flatten() {
492            put_u64(out, u64::from(self.strings.id(value)?));
493        }
494        Ok(())
495    }
496
497    fn flush_block(&mut self) -> Result<()> {
498        if self.block_ends.is_empty() {
499            return Ok(());
500        }
501        self.block_offsets
502            .write_all(&self.blocks_len.to_le_bytes())?;
503        self.blocks.write_all(&self.block_base.0.to_le_bytes())?;
504        self.blocks.write_all(&self.block_base.1.to_le_bytes())?;
505        for end in &self.block_ends {
506            self.blocks.write_all(&end.to_le_bytes())?;
507        }
508        self.blocks.write_all(&self.block)?;
509        self.blocks_len +=
510            (BLOCK_HEADER_BYTES + self.block_ends.len() * 4 + self.block.len()) as u64;
511        self.block.clear();
512        self.block_ends.clear();
513        Ok(())
514    }
515
516    /// Copy the store into the Pack.
517    pub fn finish(mut self, pack: &mut ContainerWriter) -> Result<()> {
518        self.flush_block()?;
519        self.blocks.flush()?;
520        self.block_offsets.flush()?;
521        self.strings.bytes.flush()?;
522        self.strings.ends.flush()?;
523
524        pack.begin(SECTION_INDEX, RECORDS_VERSION)?;
525        pack.write_all(&self.record_count.to_le_bytes())?;
526        std::io::copy(&mut File::open(&self.block_offsets_path)?, pack)?;
527        pack.write_all(&self.blocks_len.to_le_bytes())?;
528        pack.end()?;
529
530        pack.add_file(SECTION_BLOCKS, RECORDS_VERSION, &self.blocks_path)?;
531
532        let strings = &self.strings;
533        pack.begin(SECTION_STRINGS, RECORDS_VERSION)?;
534        pack.write_all(&(strings.segments.len() as u64).to_le_bytes())?;
535        for (first_record, first_string) in &strings.segments {
536            pack.write_all(&first_record.to_le_bytes())?;
537            pack.write_all(&first_string.to_le_bytes())?;
538        }
539        pack.write_all(&strings.count.to_le_bytes())?;
540        std::io::copy(&mut File::open(&strings.ends_path)?, pack)?;
541        std::io::copy(&mut File::open(&strings.bytes_path)?, pack)?;
542        pack.end()
543    }
544}
545
546#[derive(Clone)]
547pub struct RecordsReader {
548    record_count: u64,
549    index: Bytes,
550    blocks: Bytes,
551    strings: Bytes,
552    segment_count: u64,
553    string_count: u64,
554    /// Byte offsets within `strings` of the end-offset table and the text.
555    ends_start: usize,
556    text_start: usize,
557}
558
559impl RecordsReader {
560    pub fn open(container: &Container) -> Result<Self> {
561        let index = container.section(SECTION_INDEX, RECORDS_VERSION)?;
562        let blocks = container.section(SECTION_BLOCKS, RECORDS_VERSION)?;
563        let strings = container.section(SECTION_STRINGS, RECORDS_VERSION)?;
564
565        let record_count = read_u64_le(&index, 0).context("records index is truncated")?;
566        let block_count = record_count.div_ceil(BLOCK_RECORDS);
567        if index.len() as u64 != 8 + (block_count + 1) * 8 {
568            bail!("records index does not match its record count");
569        }
570        if read_u64_le(&index, index.len() - 8) != Some(blocks.len() as u64) {
571            bail!("records index does not match the blocks section");
572        }
573        let truncated = || anyhow::anyhow!("string table is truncated");
574        let segment_count = read_u64_le(&strings, 0).ok_or_else(truncated)?;
575        let count_at = usize::try_from(8 + segment_count * 16)?;
576        let mut previous: Option<(u64, u64)> = None;
577        for segment in 0..segment_count {
578            let at = (8 + segment * 16) as usize;
579            let first_record = read_u64_le(&strings, at).ok_or_else(truncated)?;
580            let first_string = read_u64_le(&strings, at + 8).ok_or_else(truncated)?;
581            let ordered = match previous {
582                None => first_record == 0,
583                Some((record, string)) => first_record > record && first_string >= string,
584            };
585            if !ordered || first_record >= record_count {
586                bail!("string table segments do not match the records");
587            }
588            previous = Some((first_record, first_string));
589        }
590        if (segment_count == 0) != (record_count == 0) {
591            bail!("string table segments do not match the records");
592        }
593        let string_count = read_u64_le(&strings, count_at).ok_or_else(truncated)?;
594        let ends_start = count_at + 8;
595        let text_start = ends_start + usize::try_from(string_count * 8)?;
596        let text_len = if string_count == 0 {
597            0
598        } else {
599            read_u64_le(&strings, text_start - 8).ok_or_else(truncated)?
600        };
601        if strings.len() as u64 != text_start as u64 + text_len {
602            bail!("string table does not match its offsets");
603        }
604        Ok(Self {
605            record_count,
606            index,
607            blocks,
608            strings,
609            segment_count,
610            string_count,
611            ends_start,
612            text_start,
613        })
614    }
615
616    pub fn record_count(&self) -> u64 {
617        self.record_count
618    }
619
620    pub fn header(&self, id: RecordId) -> Result<RecordHeader> {
621        Ok(self.body(id)?.0)
622    }
623
624    pub fn record(&self, id: RecordId) -> Result<Record> {
625        let (header, mut fields) = self.body(id)?;
626        let geometry = |fields: &mut &[u8]| self.geometry(&header, fields);
627        Ok(match header.layer {
628            Layer::Address => {
629                let address = self.address_components(header.strings, &mut fields)?;
630                Record::Address(AddressRecord {
631                    address,
632                    geometry: geometry(&mut fields)?,
633                    location_precision: if header.centroid {
634                        LocationPrecision::Centroid
635                    } else {
636                        LocationPrecision::Point
637                    },
638                    source: header.provenance()?,
639                })
640            }
641            Layer::Interpolation => {
642                let (address, interpolation, anchor_node_ids) =
643                    self.interpolation_fields(header.strings, &mut fields)?;
644                Record::Interpolation(InterpolationRecord {
645                    address,
646                    interpolation,
647                    anchor_node_ids,
648                    geometry: geometry(&mut fields)?,
649                    representative_point: [header.lon(), header.lat()],
650                    source: header.provenance()?,
651                })
652            }
653            Layer::Street => Record::Street(StreetRecord {
654                name: self.string_field(header.strings, &mut fields)?.to_string(),
655                geometry: geometry(&mut fields)?,
656                representative_point: [header.lon(), header.lat()],
657                source: header.provenance()?,
658            }),
659            Layer::Postcode => Record::Postcode(PostcodeRecord {
660                postcode: self.string_field(header.strings, &mut fields)?.to_string(),
661                geometry: geometry(&mut fields)?,
662                source: DerivedSourceProvenance::osm_address_records(header.source_value),
663            }),
664            layer => {
665                let place_layer = layer.place_layer().expect("remaining layers are places");
666                Record::Place(
667                    place_layer,
668                    PlaceRecord {
669                        name: self.string_field(header.strings, &mut fields)?.to_string(),
670                        place_type: self.string_field(header.strings, &mut fields)?.to_string(),
671                        geometry: geometry(&mut fields)?,
672                        source: header.provenance()?,
673                    },
674                )
675            }
676        })
677    }
678
679    /// Id, label, layer, point and source without decoding geometry.
680    pub fn summary(&self, id: RecordId) -> Result<RecordSummary> {
681        let (header, mut fields) = self.body(id)?;
682        let source = header.source()?;
683        let (record_id, label) = match header.layer {
684            Layer::Address => {
685                let address = self.address_components(header.strings, &mut fields)?;
686                let (object_type, object_id) = header.osm_source()?;
687                (
688                    crate::labels::osm_record_id(object_type, object_id),
689                    crate::labels::address_label(&address),
690                )
691            }
692            Layer::Interpolation => {
693                let (address, range, anchors) =
694                    self.interpolation_fields(header.strings, &mut fields)?;
695                let (_, way_id) = header.osm_source()?;
696                (
697                    crate::labels::interpolation_record_id(way_id, anchors[0], anchors[1]),
698                    crate::labels::interpolation_label(
699                        &crate::labels::interpolation_name(&address),
700                        &range,
701                        &address,
702                    ),
703                )
704            }
705            Layer::Street => {
706                let (object_type, object_id) = header.osm_source()?;
707                (
708                    crate::labels::osm_record_id(object_type, object_id),
709                    self.string_field(header.strings, &mut fields)?.to_string(),
710                )
711            }
712            Layer::Postcode => {
713                let postcode = self.string_field(header.strings, &mut fields)?;
714                (
715                    crate::labels::derived_postcode_id(postcode),
716                    postcode.to_string(),
717                )
718            }
719            _ => {
720                let name = self.string_field(header.strings, &mut fields)?;
721                let place_type = self.string_field(header.strings, &mut fields)?;
722                let (object_type, object_id) = header.osm_source()?;
723                (
724                    crate::labels::place_record_id(object_type, object_id, place_type),
725                    name.to_string(),
726                )
727            }
728        };
729        Ok(RecordSummary {
730            id: record_id,
731            layer: header.layer.as_str().to_string(),
732            label,
733            point: Some(header.point()),
734            source,
735        })
736    }
737
738    /// The postcode an address, interpolation or postcode record states,
739    /// without decoding its geometry.
740    pub fn postcode(&self, id: RecordId) -> Result<Option<String>> {
741        let (header, mut fields) = self.body(id)?;
742        Ok(match header.layer {
743            Layer::Address => {
744                self.address_components(header.strings, &mut fields)?
745                    .postcode
746            }
747            Layer::Interpolation => {
748                self.interpolation_fields(header.strings, &mut fields)?
749                    .0
750                    .postcode
751            }
752            Layer::Postcode => Some(self.string_field(header.strings, &mut fields)?.to_string()),
753            _ => None,
754        })
755    }
756
757    /// Context (place or postcode) record view, `None` for other layers.
758    pub fn context_record(&self, id: RecordId) -> Result<Option<ContextRecord>> {
759        let header = self.header(id)?;
760        if !header.layer.is_context() {
761            return Ok(None);
762        }
763        let summary = self.summary(id)?;
764        Ok(Some(ContextRecord {
765            id: summary.id,
766            layer: summary.layer,
767            name: summary.label.clone(),
768            postcode: (header.layer == Layer::Postcode).then(|| summary.label.clone()),
769            label: summary.label,
770            point: summary.point,
771        }))
772    }
773
774    /// Line geometry as 1e-7 degree coordinates, `None` for point records.
775    pub fn line(&self, id: RecordId) -> Result<Option<Vec<[i32; 2]>>> {
776        let (header, mut fields) = self.body(id)?;
777        if !header.line {
778            return Ok(None);
779        }
780        self.skip_fields(&header, &mut fields)?;
781        self.line_points(&header, &mut fields).map(Some)
782    }
783
784    pub fn record_json(&self, id: RecordId) -> Result<Value> {
785        let record = self.record(id)?;
786        let mut value = serde_json::to_value(&record)?;
787        let object = value
788            .as_object_mut()
789            .context("record JSON must be an object")?;
790        object.insert("layer".to_string(), json!(record.layer().as_str()));
791        Ok(value)
792    }
793
794    fn segment(&self, record_id: RecordId) -> StringSegment {
795        let field = |segment: u64, offset: u64| {
796            read_u64_le(&self.strings, (8 + segment * 16 + offset) as usize)
797                .expect("validated segments")
798        };
799        // Last segment whose first record is at or before this one.
800        let (mut low, mut high) = (0, self.segment_count);
801        while low < high {
802            let mid = low + (high - low) / 2;
803            if field(mid, 0) <= record_id {
804                low = mid + 1;
805            } else {
806                high = mid;
807            }
808        }
809        let segment = low.saturating_sub(1);
810        StringSegment {
811            first: field(segment, 8),
812            end: if segment + 1 < self.segment_count {
813                field(segment + 1, 8)
814            } else {
815                self.string_count
816            },
817        }
818    }
819
820    fn string(&self, segment: StringSegment, local: u32) -> Result<&str> {
821        let id = segment.first + u64::from(local);
822        if id >= segment.end {
823            bail!("string id {local} is out of range for its segment");
824        }
825        let end_of = |id: u64| {
826            read_u64_le(&self.strings, self.ends_start + id as usize * 8).expect("validated ends")
827        };
828        let start = if id == 0 { 0 } else { end_of(id - 1) };
829        let bytes = self
830            .strings
831            .get(self.text_start + start as usize..self.text_start + end_of(id) as usize)
832            .context("string offsets are out of range")?;
833        std::str::from_utf8(bytes).context("string table entry is not valid UTF-8")
834    }
835
836    /// The block base point and the body bytes of a record.
837    fn locate(&self, id: RecordId) -> Result<(i32, i32, &[u8])> {
838        if id >= self.record_count {
839            bail!("record {id} is out of range");
840        }
841        let block = id / BLOCK_RECORDS;
842        let start = read_u64_le(&self.index, (8 + block * 8) as usize).expect("validated index");
843        let end = read_u64_le(&self.index, (16 + block * 8) as usize).expect("validated index");
844        let block_bytes = self
845            .blocks
846            .get(start as usize..end as usize)
847            .context("record block is out of range")?;
848        let records_in_block =
849            (self.record_count - block * BLOCK_RECORDS).min(BLOCK_RECORDS) as usize;
850        let bodies_start = BLOCK_HEADER_BYTES + records_in_block * 4;
851        if block_bytes.len() < bodies_start {
852            bail!("record block {block} is truncated");
853        }
854        let base_lon = i32::from_le_bytes(block_bytes[0..4].try_into().expect("block header"));
855        let base_lat = i32::from_le_bytes(block_bytes[4..8].try_into().expect("block header"));
856        let slot = (id % BLOCK_RECORDS) as usize;
857        let body_end = |slot: usize| {
858            read_u32_le(block_bytes, BLOCK_HEADER_BYTES + slot * 4).expect("validated header")
859                as usize
860        };
861        let body_start = if slot == 0 { 0 } else { body_end(slot - 1) };
862        let body = block_bytes
863            .get(bodies_start + body_start..bodies_start + body_end(slot))
864            .context("record body is out of range")?;
865        Ok((base_lon, base_lat, body))
866    }
867
868    /// Layer and display point only: the hot path of spatial queries.
869    pub fn point(&self, id: RecordId) -> Result<(Layer, f64, f64)> {
870        let (base_lon, base_lat, mut body) = self.locate(id)?;
871        let tag = get_u8(&mut body)?;
872        let layer = *Layer::ALL
873            .get(usize::from(tag & TAG_LAYER_MASK))
874            .with_context(|| format!("record {id} has an unknown layer code"))?;
875        let lon = add_delta(base_lon, get_i64(&mut body)?)?;
876        let lat = add_delta(base_lat, get_i64(&mut body)?)?;
877        Ok((layer, dequantize(lon), dequantize(lat)))
878    }
879
880    fn body(&self, id: RecordId) -> Result<(RecordHeader, &[u8])> {
881        let (base_lon, base_lat, mut body) = self.locate(id)?;
882        let tag = get_u8(&mut body)?;
883        let layer = *Layer::ALL
884            .get(usize::from(tag & TAG_LAYER_MASK))
885            .with_context(|| format!("record {id} has an unknown layer code"))?;
886        let source_kind = match tag >> TAG_SOURCE_SHIFT {
887            0 => SourceKind::Node,
888            1 => SourceKind::Way,
889            2 => SourceKind::Relation,
890            _ => SourceKind::Derived,
891        };
892        let lon_e7 = add_delta(base_lon, get_i64(&mut body)?)?;
893        let lat_e7 = add_delta(base_lat, get_i64(&mut body)?)?;
894        let context = match get_u64(&mut body)? {
895            0 => None,
896            value => Some(ContextRef {
897                tuple_id: u32::try_from((value >> 1) - 1).context("context id out of range")?,
898                ambiguous: value & 1 == 1,
899            }),
900        };
901        let source_value = get_u64(&mut body)?;
902        Ok((
903            RecordHeader {
904                layer,
905                strings: self.segment(id),
906                lon_e7,
907                lat_e7,
908                context,
909                centroid: tag & TAG_CENTROID != 0,
910                line: tag & TAG_LINE != 0,
911                source_kind,
912                source_value,
913            },
914            body,
915        ))
916    }
917
918    fn string_field(&self, strings: StringSegment, fields: &mut &[u8]) -> Result<&str> {
919        self.string(strings, get_u32(fields)?)
920    }
921
922    fn optional_strings<const N: usize>(
923        &self,
924        strings: StringSegment,
925        fields: &mut &[u8],
926    ) -> Result<[Option<String>; N]> {
927        let present = get_u8(fields)?;
928        let mut values: [Option<String>; N] = std::array::from_fn(|_| None);
929        for (bit, value) in values.iter_mut().enumerate() {
930            if present & (1 << bit) != 0 {
931                *value = Some(self.string_field(strings, fields)?.to_string());
932            }
933        }
934        Ok(values)
935    }
936
937    fn address_components(
938        &self,
939        strings: StringSegment,
940        fields: &mut &[u8],
941    ) -> Result<AddressComponents> {
942        let number = self.string_field(strings, fields)?.to_string();
943        let [street, place, unit, locality, region, postcode, country] =
944            self.optional_strings(strings, fields)?;
945        Ok(AddressComponents {
946            number,
947            street,
948            place,
949            unit,
950            locality,
951            region,
952            postcode,
953            country,
954        })
955    }
956
957    fn interpolation_fields(
958        &self,
959        strings: StringSegment,
960        fields: &mut &[u8],
961    ) -> Result<(InterpolationAddressComponents, InterpolationRange, [i64; 2])> {
962        let [street, place, locality, region, postcode, country] =
963            self.optional_strings(strings, fields)?;
964        let kind = self.string_field(strings, fields)?.to_string();
965        let start = get_u32(fields)?;
966        let end = start.wrapping_add(get_u32(fields)?);
967        let step = get_u32(fields)?;
968        let anchors = [get_i64(fields)?, get_i64(fields)?];
969        Ok((
970            InterpolationAddressComponents {
971                street,
972                place,
973                locality,
974                region,
975                postcode,
976                country,
977            },
978            InterpolationRange {
979                kind,
980                start,
981                end,
982                step,
983            },
984            anchors,
985        ))
986    }
987
988    fn skip_fields(&self, header: &RecordHeader, fields: &mut &[u8]) -> Result<()> {
989        match header.layer {
990            Layer::Address => {
991                self.address_components(header.strings, fields)?;
992            }
993            Layer::Interpolation => {
994                self.interpolation_fields(header.strings, fields)?;
995            }
996            Layer::Street | Layer::Postcode => {
997                get_u32(fields)?;
998            }
999            _ => {
1000                get_u32(fields)?;
1001                get_u32(fields)?;
1002            }
1003        }
1004        Ok(())
1005    }
1006
1007    fn geometry(&self, header: &RecordHeader, fields: &mut &[u8]) -> Result<Geometry> {
1008        if !header.line {
1009            return Ok(point_geometry(header.lon(), header.lat()));
1010        }
1011        let coordinates = self
1012            .line_points(header, fields)?
1013            .into_iter()
1014            .map(|[lon, lat]| vec![dequantize(lon), dequantize(lat)].into())
1015            .collect();
1016        Ok(Geometry::new(GeometryValue::LineString { coordinates }))
1017    }
1018
1019    fn line_points(&self, header: &RecordHeader, fields: &mut &[u8]) -> Result<Vec<[i32; 2]>> {
1020        let count = usize::try_from(get_u64(fields)?)?;
1021        if count < 2 || count > fields.len() {
1022            bail!("stored line geometry has an invalid point count {count}");
1023        }
1024        let mut points = Vec::with_capacity(count);
1025        let mut previous = [header.lon_e7, header.lat_e7];
1026        for _ in 0..count {
1027            let lon = add_delta(previous[0], get_i64(fields)?)?;
1028            let lat = add_delta(previous[1], get_i64(fields)?)?;
1029            previous = [lon, lat];
1030            points.push(previous);
1031        }
1032        Ok(points)
1033    }
1034}
1035
1036fn osm_source(source: &SourceProvenance) -> (SourceKind, u64) {
1037    let kind = match source.object_type {
1038        OsmObjectType::Node => SourceKind::Node,
1039        OsmObjectType::Way => SourceKind::Way,
1040        OsmObjectType::Relation => SourceKind::Relation,
1041    };
1042    (kind, crate::util::codec::zigzag(source.object_id))
1043}
1044
1045fn layer_code(layer: Layer) -> u8 {
1046    Layer::ALL
1047        .iter()
1048        .position(|candidate| *candidate == layer)
1049        .expect("every layer has a code") as u8
1050}
1051
1052/// Quantized vertices of a line geometry, `None` for points.
1053fn line_coordinates(geometry: &Geometry) -> Result<Option<Vec<(i32, i32)>>> {
1054    match &geometry.value {
1055        GeometryValue::Point { .. } => Ok(None),
1056        GeometryValue::LineString { coordinates } => {
1057            if coordinates.len() < 2 {
1058                bail!("line geometry needs at least two positions");
1059            }
1060            coordinates
1061                .iter()
1062                .map(|position| {
1063                    let [lon, lat, ..] = position.as_slice() else {
1064                        bail!("line position is missing lon/lat");
1065                    };
1066                    Ok((quantize(*lon)?, quantize(*lat)?))
1067                })
1068                .collect::<Result<Vec<_>>>()
1069                .map(Some)
1070        }
1071        other => bail!("unsupported stored record geometry: {}", other.type_name()),
1072    }
1073}
1074
1075pub(crate) fn quantize(value: f64) -> Result<i32> {
1076    if !value.is_finite() {
1077        bail!("coordinate must be finite");
1078    }
1079    let scaled = (value * COORDINATE_SCALE).round();
1080    if scaled < i32::MIN as f64 || scaled > i32::MAX as f64 {
1081        bail!("coordinate {value} is out of range");
1082    }
1083    Ok(scaled as i32)
1084}
1085
1086pub(crate) fn dequantize(value: i32) -> f64 {
1087    value as f64 / COORDINATE_SCALE
1088}
1089
1090fn add_delta(base: i32, delta: i64) -> Result<i32> {
1091    i32::try_from(i64::from(base) + delta).context("coordinate delta is out of range")
1092}
1093
1094#[cfg(test)]
1095mod tests {
1096    use super::*;
1097    use crate::record::PlaceLayer;
1098
1099    fn line(points: &[[f64; 2]]) -> Geometry {
1100        Geometry::new(GeometryValue::LineString {
1101            coordinates: points.iter().map(|point| point.to_vec().into()).collect(),
1102        })
1103    }
1104
1105    fn sample_records() -> Vec<Record> {
1106        let mut records = vec![
1107            Record::Address(AddressRecord {
1108                address: AddressComponents {
1109                    number: "10".into(),
1110                    street: None,
1111                    place: Some("Market Square".into()),
1112                    unit: Some("2".into()),
1113                    locality: Some("Toronto".into()),
1114                    region: Some("ON".into()),
1115                    postcode: Some("M5V 1A1".into()),
1116                    country: Some("CA".into()),
1117                },
1118                geometry: point_geometry(-79.0, 43.0),
1119                location_precision: LocationPrecision::Centroid,
1120                source: SourceProvenance::osm(OsmObjectType::Node, 1),
1121            }),
1122            Record::Street(StreetRecord {
1123                name: "King Street".into(),
1124                geometry: line(&[[-79.0, 43.0], [-79.001, 43.001], [-79.0015, 43.0]]),
1125                representative_point: [-79.0, 43.0],
1126                source: SourceProvenance::osm(OsmObjectType::Way, 2),
1127            }),
1128            Record::Postcode(PostcodeRecord {
1129                postcode: "M5V 1A1".into(),
1130                geometry: point_geometry(-79.0, 43.0),
1131                source: DerivedSourceProvenance::osm_address_records(12),
1132            }),
1133            Record::Interpolation(InterpolationRecord {
1134                address: InterpolationAddressComponents {
1135                    street: Some("King Street".into()),
1136                    place: None,
1137                    locality: Some("Toronto".into()),
1138                    region: None,
1139                    postcode: Some("M5V".into()),
1140                    country: Some("CA".into()),
1141                },
1142                interpolation: InterpolationRange {
1143                    kind: "even".into(),
1144                    start: 10,
1145                    end: 20,
1146                    step: 2,
1147                },
1148                anchor_node_ids: [10, 20],
1149                geometry: line(&[[-79.0, 43.0], [-79.001, 43.001]]),
1150                representative_point: [-79.0, 43.0],
1151                source: SourceProvenance::osm(OsmObjectType::Way, 5),
1152            }),
1153        ];
1154        for layer in [
1155            PlaceLayer::Country,
1156            PlaceLayer::Region,
1157            PlaceLayer::District,
1158            PlaceLayer::Place,
1159            PlaceLayer::Locality,
1160            PlaceLayer::Neighbourhood,
1161        ] {
1162            records.push(Record::Place(
1163                layer,
1164                PlaceRecord {
1165                    name: "Example place".into(),
1166                    place_type: "city".into(),
1167                    geometry: point_geometry(-79.0, 43.0),
1168                    source: SourceProvenance::osm(OsmObjectType::Relation, 3),
1169                },
1170            ));
1171        }
1172        records.push(Record::Place(
1173            PlaceLayer::Country,
1174            PlaceRecord {
1175                name: "Canada".into(),
1176                place_type: "derived_country:CA".into(),
1177                geometry: point_geometry(-79.0, 43.0),
1178                source: SourceProvenance::osm(OsmObjectType::Relation, 4),
1179            },
1180        ));
1181        records
1182    }
1183
1184    fn write_store(records: &[Record], contexts: &[Option<ContextRef>]) -> RecordsReader {
1185        let root =
1186            std::env::temp_dir().join(format!("open-geocode-records-{}", uuid::Uuid::new_v4()));
1187        let scratch = Scratch::create(root.join("scratch")).expect("scratch");
1188        let mut writer =
1189            RecordsWriter::create(&scratch, &MemoryBudget::unlimited()).expect("writer");
1190        for (index, record) in records.iter().enumerate() {
1191            let id = writer
1192                .write(record, contexts.get(index).copied().flatten())
1193                .expect("write");
1194            assert_eq!(id, index as u64);
1195        }
1196        let path = root.join("pack.ogp");
1197        let mut pack = ContainerWriter::create(&path).expect("pack");
1198        writer.finish(&mut pack).expect("finish");
1199        pack.finish().expect("pack finish");
1200        RecordsReader::open(&Container::open(&path).expect("open")).expect("reader")
1201    }
1202
1203    #[test]
1204    fn round_trips_every_layer_and_matches_summaries() {
1205        let records = sample_records();
1206        let reader = write_store(&records, &[]);
1207        assert_eq!(reader.record_count(), records.len() as u64);
1208        for (id, record) in records.iter().enumerate() {
1209            let id = id as u64;
1210            assert_eq!(&reader.record(id).expect("record"), record);
1211            let summary = reader.summary(id).expect("summary");
1212            let full = reader.record_json(id).expect("json");
1213            assert_eq!(summary.id, full["id"]);
1214            assert_eq!(summary.label, full["label"]);
1215            assert_eq!(summary.layer, full["layer"]);
1216            assert_eq!(
1217                serde_json::to_value(&summary.source).expect("source"),
1218                full["source"]
1219            );
1220            let point = summary.point.expect("point");
1221            assert_eq!((point.lon, point.lat), (-79.0, 43.0));
1222        }
1223        assert_eq!(
1224            reader.record_json(3).expect("interpolation")["anchor_ids"],
1225            json!(["osm:node:10", "osm:node:20"])
1226        );
1227        assert_eq!(reader.line(0).expect("address line"), None);
1228        assert_eq!(
1229            reader.line(1).expect("street line"),
1230            Some(vec![
1231                [-790_000_000, 430_000_000],
1232                [-790_010_000, 430_010_000],
1233                [-790_015_000, 430_000_000]
1234            ])
1235        );
1236    }
1237
1238    #[test]
1239    fn string_dictionary_is_bounded_by_the_segment() {
1240        let root =
1241            std::env::temp_dir().join(format!("open-geocode-segments-{}", uuid::Uuid::new_v4()));
1242        let scratch = Scratch::create(root.join("scratch")).expect("scratch");
1243        let budget = MemoryBudget::unlimited();
1244        let mut writer = RecordsWriter::create(&scratch, &budget)
1245            .expect("writer")
1246            .with_segment_records(100);
1247        let base = sample_records().remove(0);
1248        let mut records = Vec::new();
1249        let mut most_resident = 0;
1250        for index in 0..1_050 {
1251            let mut record = base.clone();
1252            if let Record::Address(address) = &mut record {
1253                // Every record has its own street; locality is shared by all.
1254                address.address.street = Some(format!("Street {index}"));
1255                address.address.number = (index % 7).to_string();
1256            }
1257            writer.write(&record, None).expect("write");
1258            most_resident = most_resident.max(writer.resident_strings());
1259            assert!(budget.used() >= writer.resident_strings() * DICTIONARY_ENTRY_OVERHEAD);
1260            records.push(record);
1261        }
1262        assert!(
1263            most_resident <= 100 + 12,
1264            "dictionary held {most_resident} strings"
1265        );
1266        assert!(budget.peak() < 200 * (DICTIONARY_ENTRY_OVERHEAD + 16));
1267        let path = root.join("pack.ogp");
1268        let mut pack = ContainerWriter::create(&path).expect("pack");
1269        writer.finish(&mut pack).expect("finish");
1270        pack.finish().expect("pack finish");
1271        let reader = RecordsReader::open(&Container::open(&path).expect("open")).expect("reader");
1272        for (id, record) in records.iter().enumerate() {
1273            assert_eq!(&reader.record(id as u64).expect("record"), record);
1274        }
1275    }
1276
1277    #[test]
1278    fn string_dictionary_stays_within_its_share_of_a_small_budget() {
1279        let root =
1280            std::env::temp_dir().join(format!("open-geocode-dictionary-{}", uuid::Uuid::new_v4()));
1281        let scratch = Scratch::create(root.join("scratch")).expect("scratch");
1282        let budget = MemoryBudget::new(64 << 20);
1283        let mut writer = RecordsWriter::create(&scratch, &budget).expect("writer");
1284        let base = sample_records().remove(0);
1285        // Every record brings a new street name: the worst case.
1286        for index in 0..200_000 {
1287            let mut record = base.clone();
1288            if let Record::Address(address) = &mut record {
1289                address.address.street = Some(format!("Unique Street Number {index}"));
1290            }
1291            writer.write(&record, None).expect("write");
1292        }
1293        // One entry can land after the share is reached; the next record
1294        // starts a new segment.
1295        let share = budget.limit() / DICTIONARY_BUDGET_FRACTION;
1296        let slack = 8 * (64 + DICTIONARY_ENTRY_OVERHEAD);
1297        assert!(
1298            budget.peak() <= share + slack,
1299            "dictionary peaked at {} of {share}",
1300            budget.peak()
1301        );
1302        assert!(writer.strings.segments.len() > 1);
1303        let path = root.join("pack.ogp");
1304        let mut pack = ContainerWriter::create(&path).expect("pack");
1305        writer.finish(&mut pack).expect("finish");
1306        pack.finish().expect("pack finish");
1307        let reader = RecordsReader::open(&Container::open(&path).expect("open")).expect("reader");
1308        for id in [0, 99_999, 199_999] {
1309            let Record::Address(address) = reader.record(id).expect("record") else {
1310                panic!("address");
1311            };
1312            assert_eq!(
1313                address.address.street.as_deref(),
1314                Some(format!("Unique Street Number {id}").as_str())
1315            );
1316        }
1317    }
1318
1319    #[test]
1320    fn spans_blocks_and_keeps_contexts() {
1321        let base = sample_records().remove(0);
1322        let mut records = Vec::new();
1323        let mut contexts = Vec::new();
1324        for index in 0..(BLOCK_RECORDS * 3 + 5) {
1325            let mut record = base.clone();
1326            if let Record::Address(address) = &mut record {
1327                address.address.number = index.to_string();
1328                address.geometry = point_geometry(-79.0 + index as f64 * 0.001, 43.0);
1329                address.source.object_id = index as i64;
1330            }
1331            records.push(record);
1332            contexts.push((index % 3 != 0).then_some(ContextRef {
1333                tuple_id: index as u32,
1334                ambiguous: index % 2 == 0,
1335            }));
1336        }
1337        let reader = write_store(&records, &contexts);
1338        for (index, record) in records.iter().enumerate() {
1339            assert_eq!(&reader.record(index as u64).expect("record"), record);
1340            assert_eq!(
1341                reader.header(index as u64).expect("header").context,
1342                contexts[index]
1343            );
1344        }
1345        assert!(reader.record(records.len() as u64).is_err());
1346    }
1347
1348    #[test]
1349    fn line_geometry_follows_the_summary_fields() {
1350        let mut records = sample_records();
1351        records.truncate(2);
1352        let reader = write_store(&records, &[]);
1353        let (header, fields) = reader.body(1).expect("body");
1354        let mut cursor = fields;
1355        reader.skip_fields(&header, &mut cursor).expect("fields");
1356        let geometry_offset = fields.len() - cursor.len();
1357
1358        // A damaged point count breaks full decoding but not the summary, which
1359        // stops before the geometry bytes.
1360        let mut damaged = fields.to_vec();
1361        damaged[geometry_offset] = 1;
1362        let mut cursor = damaged.as_slice();
1363        assert_eq!(
1364            reader
1365                .string_field(header.strings, &mut cursor)
1366                .expect("name"),
1367            "King Street"
1368        );
1369        assert!(reader.geometry(&header, &mut cursor).is_err());
1370    }
1371}
open_geocode/records.rs

open_geocode/
records.rs
