open_geocode/
text_index.rs

1//! Tantivy text index.
2//!
3//! The index is built in the scratch directory, merged to one segment, and its
4//! files are copied into the Pack as `text/<file>` sections. At runtime a
5//! read-only [`Directory`] serves those sections straight from the Pack
6//! mapping, so the text index needs no files of its own.
7
8use std::{
9    collections::{BTreeSet, HashMap},
10    fmt, fs, io, mem,
11    path::{Path, PathBuf},
12    sync::Arc,
13};
14
15use anyhow::{Context, Result, bail};
16use tantivy::{
17    Index, IndexWriter, TantivyDocument,
18    directory::{
19        Directory, FileHandle, RamDirectory, WatchCallback, WatchHandle, WritePtr,
20        error::{DeleteError, OpenReadError, OpenWriteError},
21    },
22    indexer::UserOperation,
23    merge_policy::NoMergePolicy,
24    schema::{FAST, Field, IndexRecordOption, STRING, Schema, TextFieldIndexing, TextOptions},
25};
26
27use crate::{
28    container::{Bytes, Container, ContainerWriter},
29    extsort::Scratch,
30    memory::{MemoryBudget, Reservation},
31    pack::RecordId,
32    record::{
33        AddressComponents, AddressRecord, InterpolationAddressComponents, InterpolationRecord,
34        PlaceRecord, PostcodeRecord, Record, StreetRecord,
35    },
36    util::text::collapse_whitespace,
37};
38
39pub const TEXT_SECTION_PREFIX: &str = "text/";
40pub const TEXT_INDEX_SCHEMA_VERSION: u32 = 5;
41
42/// Tantivy needs at least 15 MB per indexing thread (it uses fewer threads
43/// when given less) and gains little beyond 1.6 GB.
44const MIN_INDEX_MEMORY_BYTES: usize = 16 << 20;
45const MAX_INDEX_MEMORY_BYTES: usize = 1_600_000_000;
46const TEXT_INDEX_BATCH_SIZE: usize = 10_000;
47const AUTOCOMPLETE_SUBJECT_FIELD: &str = "autocomplete_subject_text";
48
49pub struct TextIndexWriter {
50    /// Created with the first document, so its buffers are reserved only for
51    /// the phase that writes records.
52    writer: Option<(IndexWriter, Reservation)>,
53    budget: Arc<MemoryBudget>,
54    memory_bytes: usize,
55    index: Index,
56    path: PathBuf,
57    fields: TextIndexFields,
58    pending_documents: Vec<TantivyDocument>,
59    document_count: u64,
60}
61
62#[derive(Debug, Clone, Copy)]
63pub struct TextIndexFields {
64    pub record_id: Field,
65    pub layer: Field,
66    pub content_text: Field,
67    pub label_text: Field,
68    pub name_text: Field,
69    pub address_number: Field,
70    pub postcode_exact: Field,
71    pub autocomplete_subject_text: Field,
72}
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75pub struct TextIndexCommit {
76    pub document_count: u64,
77    pub bytes: u64,
78}
79
80#[derive(Debug, Clone, PartialEq, Eq)]
81pub struct TextIndexDocument {
82    pub record_id: RecordId,
83    pub layer: String,
84    pub content_text: String,
85    pub label: Option<String>,
86    pub name: Option<String>,
87    pub address_number: Option<String>,
88    pub postcode: Option<String>,
89    pub autocomplete_subject_text: String,
90}
91
92#[derive(Debug, Clone, PartialEq, Eq)]
93struct TextIndexProjection {
94    document: TextIndexDocument,
95}
96
97impl fmt::Debug for TextIndexWriter {
98    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
99        formatter
100            .debug_struct("TextIndexWriter")
101            .field("fields", &self.fields)
102            .field("pending_document_count", &self.pending_documents.len())
103            .field("document_count", &self.document_count)
104            .finish_non_exhaustive()
105    }
106}
107
108impl TextIndexWriter {
109    pub fn create(
110        scratch: &Arc<Scratch>,
111        budget: &Arc<MemoryBudget>,
112        memory_bytes: usize,
113    ) -> Result<Self> {
114        let (schema, fields) = build_schema();
115        let path = scratch.path().join("text");
116        fs::create_dir_all(&path)
117            .with_context(|| format!("failed to create {}", path.display()))?;
118        let index = Index::create_in_dir(&path, schema)
119            .with_context(|| format!("failed to create Tantivy index {}", path.display()))?;
120        Ok(Self {
121            writer: None,
122            budget: Arc::clone(budget),
123            memory_bytes: memory_bytes.clamp(MIN_INDEX_MEMORY_BYTES, MAX_INDEX_MEMORY_BYTES),
124            index,
125            path,
126            fields,
127            pending_documents: Vec::with_capacity(TEXT_INDEX_BATCH_SIZE),
128            document_count: 0,
129        })
130    }
131
132    pub fn fields(&self) -> TextIndexFields {
133        self.fields
134    }
135
136    fn writer(&mut self) -> Result<&mut IndexWriter> {
137        if self.writer.is_none() {
138            // Reserve before Tantivy allocates; the share was sized within the
139            // budget, so this only overshoots if a caller asked for more.
140            let mut memory = Reservation::empty(&self.budget);
141            if !memory.try_grow(self.memory_bytes) {
142                memory.force_grow(self.memory_bytes);
143            }
144            let writer = self
145                .index
146                .writer(self.memory_bytes)
147                .context("failed to create Tantivy index writer")?;
148            // Disable background auto-merges during the build so the only
149            // merge is the single deterministic one forced in `finish()`.
150            writer.set_merge_policy(Box::new(NoMergePolicy));
151            self.writer = Some((writer, memory));
152        }
153        Ok(&mut self.writer.as_mut().expect("writer created").0)
154    }
155
156    pub fn add(&mut self, document: TantivyDocument) -> Result<()> {
157        self.pending_documents.push(document);
158        self.document_count += 1;
159        if self.pending_documents.len() >= TEXT_INDEX_BATCH_SIZE {
160            self.flush()?;
161        }
162        Ok(())
163    }
164
165    fn flush(&mut self) -> Result<()> {
166        if self.pending_documents.is_empty() {
167            return Ok(());
168        }
169        let documents = mem::replace(
170            &mut self.pending_documents,
171            Vec::with_capacity(TEXT_INDEX_BATCH_SIZE),
172        );
173        self.writer()?
174            .run(documents.into_iter().map(UserOperation::Add))
175            .context("failed to batch index records")?;
176        Ok(())
177    }
178
179    /// Commit, merge to a single segment and copy the index into the Pack.
180    pub fn finish(mut self, pack: &mut ContainerWriter) -> Result<TextIndexCommit> {
181        self.flush()?;
182        self.writer()?;
183        let (mut writer, memory) = self.writer.take().expect("writer created");
184        writer
185            .commit()
186            .context("failed to commit Tantivy text index")?;
187        let segment_ids = self
188            .index
189            .searchable_segment_ids()
190            .context("failed to list text index segments")?;
191        if segment_ids.len() > 1 {
192            writer
193                .merge(&segment_ids)
194                .wait()
195                .context("failed to merge text index segments")?;
196            writer
197                .commit()
198                .context("failed to commit merged text index")?;
199        }
200        writer
201            .garbage_collect_files()
202            .wait()
203            .context("failed to garbage collect text index files")?;
204        writer
205            .wait_merging_threads()
206            .context("failed to stop Tantivy indexing threads")?;
207        drop(memory);
208
209        let mut files = fs::read_dir(&self.path)?
210            .map(|entry| entry.map(|entry| entry.path()))
211            .collect::<io::Result<Vec<_>>>()?;
212        files.sort();
213        let mut bytes = 0;
214        for file in files {
215            let name = file
216                .file_name()
217                .and_then(|name| name.to_str())
218                .context("text index file name is not UTF-8")?;
219            if name.ends_with(".lock") {
220                continue;
221            }
222            bytes += fs::metadata(&file)?.len();
223            pack.add_file(
224                &format!("{TEXT_SECTION_PREFIX}{name}"),
225                TEXT_INDEX_SCHEMA_VERSION,
226                &file,
227            )?;
228        }
229        Ok(TextIndexCommit {
230            document_count: self.document_count,
231            bytes,
232        })
233    }
234}
235
236impl TextIndexFields {
237    pub fn from_schema(schema: &Schema) -> Result<Self> {
238        Ok(Self {
239            record_id: schema.get_field("record_id")?,
240            layer: schema.get_field("layer")?,
241            content_text: schema.get_field("content_text")?,
242            label_text: schema.get_field("label_text")?,
243            name_text: schema.get_field("name_text")?,
244            address_number: schema.get_field("address_number")?,
245            postcode_exact: schema.get_field("postcode_exact")?,
246            autocomplete_subject_text: schema.get_field(AUTOCOMPLETE_SUBJECT_FIELD)?,
247        })
248    }
249
250    /// Tantivy document for a record. Pure, so callers can build documents in
251    /// parallel before handing them to the writer.
252    pub fn document(self, record_id: RecordId, record: &Record) -> TantivyDocument {
253        self.to_tantivy_document(&TextIndexDocument::from_record(record_id, record))
254    }
255
256    fn to_tantivy_document(self, projected: &TextIndexDocument) -> TantivyDocument {
257        let mut document = TantivyDocument::default();
258        document.add_u64(self.record_id, projected.record_id);
259        document.add_text(self.layer, &projected.layer);
260        if !projected.content_text.is_empty() {
261            document.add_text(self.content_text, &projected.content_text);
262        }
263        add_text_if_present(&mut document, self.label_text, projected.label.as_deref());
264        add_text_if_present(&mut document, self.name_text, projected.name.as_deref());
265        add_normalized_text_if_present(
266            &mut document,
267            self.address_number,
268            projected.address_number.as_deref(),
269        );
270        add_normalized_text_if_present(
271            &mut document,
272            self.postcode_exact,
273            projected.postcode.as_deref(),
274        );
275        if !projected.autocomplete_subject_text.is_empty() {
276            document.add_text(
277                self.autocomplete_subject_text,
278                &projected.autocomplete_subject_text,
279            );
280        }
281        document
282    }
283}
284
285impl TextIndexDocument {
286    pub fn from_record(record_id: RecordId, record: &Record) -> Self {
287        match record {
288            Record::Address(record) => Self::project_address(record_id, record),
289            Record::Interpolation(record) => Self::project_interpolation(record_id, record),
290            Record::Street(record) => Self::project_street(record_id, record),
291            Record::Postcode(record) => Self::project_postcode(record_id, record),
292            Record::Place(layer, record) => Self::project_place(record_id, layer.as_str(), record),
293        }
294        .document
295    }
296
297    fn project_address(record_id: RecordId, address: &AddressRecord) -> TextIndexProjection {
298        let mut builder = ProjectionBuilder::new(record_id, "address");
299        builder.label_for_search(&address.label());
300        builder.name_for_search(&address.name());
301        builder.address(&address.address);
302        builder.build()
303    }
304
305    fn project_interpolation(
306        record_id: RecordId,
307        interpolation: &InterpolationRecord,
308    ) -> TextIndexProjection {
309        let mut builder = ProjectionBuilder::new(record_id, "interpolation");
310        builder.label_for_search(&interpolation.label());
311        builder.name_for_search(&interpolation.name());
312        builder.interpolation_address(&interpolation.address);
313        builder.build()
314    }
315
316    fn project_street(record_id: RecordId, street: &StreetRecord) -> TextIndexProjection {
317        let mut builder = ProjectionBuilder::new(record_id, "street");
318        builder.label(&street.label());
319        builder.name(&street.name);
320        builder.build()
321    }
322
323    fn project_postcode(record_id: RecordId, postcode: &PostcodeRecord) -> TextIndexProjection {
324        let mut builder = ProjectionBuilder::new(record_id, "postcode");
325        builder.label_for_search(&postcode.label());
326        builder.name_for_search(&postcode.name());
327        builder.postcode(&postcode.postcode);
328        builder.build()
329    }
330
331    fn project_place(record_id: RecordId, layer: &str, place: &PlaceRecord) -> TextIndexProjection {
332        let mut builder = ProjectionBuilder::new(record_id, layer);
333        builder.label(&place.label());
334        builder.name(&place.name);
335        builder.build()
336    }
337}
338
339pub fn open_text_index(container: &Container) -> Result<Index> {
340    Index::open(PackDirectory::new(container)?).context("failed to open the Pack text index")
341}
342
343/// Read-only Tantivy directory over the `text/` sections of a Pack. Writes the
344/// reader needs (lock files) go to a private in-memory directory.
345#[derive(Clone)]
346struct PackDirectory {
347    files: Arc<HashMap<PathBuf, Bytes>>,
348    scratch: RamDirectory,
349}
350
351impl fmt::Debug for PackDirectory {
352    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
353        formatter
354            .debug_struct("PackDirectory")
355            .field("files", &self.files.len())
356            .finish()
357    }
358}
359
360impl PackDirectory {
361    fn new(container: &Container) -> Result<Self> {
362        let mut files = HashMap::new();
363        for (name, info) in container.sections() {
364            let Some(file) = name.strip_prefix(TEXT_SECTION_PREFIX) else {
365                continue;
366            };
367            if info.version != TEXT_INDEX_SCHEMA_VERSION {
368                bail!(
369                    "text index version {} is unsupported; rebuild the Pack for version {TEXT_INDEX_SCHEMA_VERSION}",
370                    info.version
371                );
372            }
373            files.insert(PathBuf::from(file), container.raw_section(name)?);
374        }
375        if files.is_empty() {
376            bail!("Pack has no text index");
377        }
378        Ok(Self {
379            files: Arc::new(files),
380            scratch: RamDirectory::create(),
381        })
382    }
383}
384
385impl Directory for PackDirectory {
386    fn get_file_handle(&self, path: &Path) -> Result<Arc<dyn FileHandle>, OpenReadError> {
387        match self.files.get(path) {
388            Some(bytes) => Ok(Arc::new(bytes.clone())),
389            None => self.scratch.get_file_handle(path),
390        }
391    }
392
393    fn delete(&self, path: &Path) -> Result<(), DeleteError> {
394        self.scratch.delete(path)
395    }
396
397    fn exists(&self, path: &Path) -> Result<bool, OpenReadError> {
398        Ok(self.files.contains_key(path) || self.scratch.exists(path)?)
399    }
400
401    fn open_write(&self, path: &Path) -> Result<WritePtr, OpenWriteError> {
402        self.scratch.open_write(path)
403    }
404
405    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError> {
406        match self.files.get(path) {
407            Some(bytes) => Ok(bytes.as_slice().to_vec()),
408            None => self.scratch.atomic_read(path),
409        }
410    }
411
412    fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
413        self.scratch.atomic_write(path, data)
414    }
415
416    fn sync_directory(&self) -> io::Result<()> {
417        Ok(())
418    }
419
420    fn watch(&self, _watch_callback: WatchCallback) -> tantivy::Result<WatchHandle> {
421        Ok(WatchHandle::empty())
422    }
423}
424
425fn build_schema() -> (Schema, TextIndexFields) {
426    let mut builder = Schema::builder();
427    let record_id = builder.add_u64_field("record_id", FAST);
428    let exact_unstored = exact_string_options();
429    let layer = builder.add_text_field("layer", exact_unstored.clone());
430    let content_text = builder.add_text_field("content_text", searchable_text_options());
431    let label_text = builder.add_text_field("label_text", searchable_text_options());
432    let name_text = builder.add_text_field("name_text", searchable_text_options());
433    let address_number = builder.add_text_field("address_number", exact_unstored.clone());
434    let postcode_exact = builder.add_text_field("postcode_exact", exact_unstored);
435    let autocomplete_subject_text = builder.add_text_field(
436        AUTOCOMPLETE_SUBJECT_FIELD,
437        autocomplete_subject_text_options(),
438    );
439    let schema = builder.build();
440    let fields = TextIndexFields {
441        record_id,
442        layer,
443        content_text,
444        label_text,
445        name_text,
446        address_number,
447        postcode_exact,
448        autocomplete_subject_text,
449    };
450    (schema, fields)
451}
452
453fn searchable_text_options() -> TextOptions {
454    TextOptions::default().set_indexing_options(
455        TextFieldIndexing::default()
456            .set_tokenizer("default")
457            .set_index_option(IndexRecordOption::WithFreqs)
458            .set_fieldnorms(false),
459    )
460}
461
462fn autocomplete_subject_text_options() -> TextOptions {
463    TextOptions::default().set_indexing_options(
464        TextFieldIndexing::default()
465            .set_tokenizer("default")
466            .set_index_option(IndexRecordOption::WithFreqsAndPositions)
467            .set_fieldnorms(false),
468    )
469}
470
471fn exact_string_options() -> TextOptions {
472    STRING.set_indexing_options(
473        TextFieldIndexing::default()
474            .set_tokenizer("raw")
475            .set_index_option(IndexRecordOption::Basic)
476            .set_fieldnorms(false),
477    )
478}
479
480#[derive(Debug)]
481struct ProjectionBuilder {
482    projected: TextIndexDocument,
483    content_parts: Vec<String>,
484    autocomplete_subject_parts: Vec<String>,
485}
486
487impl ProjectionBuilder {
488    fn new(record_id: RecordId, layer: &str) -> Self {
489        Self {
490            projected: TextIndexDocument {
491                record_id,
492                layer: layer.to_string(),
493                content_text: String::new(),
494                label: None,
495                name: None,
496                address_number: None,
497                postcode: None,
498                autocomplete_subject_text: String::new(),
499            },
500            content_parts: Vec::new(),
501            autocomplete_subject_parts: Vec::new(),
502        }
503    }
504
505    fn label(&mut self, value: &str) {
506        self.projected.label = collapse_whitespace(value);
507        self.add_content_text(value);
508        self.add_autocomplete_subject_text(value);
509    }
510
511    fn name(&mut self, value: &str) {
512        self.projected.name = collapse_whitespace(value);
513        self.add_content_text(value);
514        self.add_autocomplete_subject_text(value);
515    }
516
517    fn label_for_search(&mut self, value: &str) {
518        self.projected.label = collapse_whitespace(value);
519        self.add_content_text(value);
520    }
521
522    fn name_for_search(&mut self, value: &str) {
523        self.projected.name = collapse_whitespace(value);
524        self.add_content_text(value);
525    }
526
527    fn address(&mut self, address: &AddressComponents) {
528        self.projected.address_number = collapse_whitespace(&address.number);
529        self.add_content_text(&address.number);
530        self.add_optional_content_text(address.street.as_deref());
531        self.add_optional_content_text(address.place.as_deref());
532        self.add_optional_content_text(address.unit.as_deref());
533        self.add_optional_content_text(address.locality.as_deref());
534        self.add_optional_content_text(address.region.as_deref());
535        let address_subject = address.street.as_deref().or(address.place.as_deref());
536        self.add_optional_autocomplete_text(address_subject);
537        self.projected.postcode = self.add_optional_postcode_text(address.postcode.as_deref());
538        self.add_optional_content_text(address.country.as_deref());
539    }
540
541    fn interpolation_address(&mut self, address: &InterpolationAddressComponents) {
542        self.add_optional_content_text(address.street.as_deref());
543        self.add_optional_content_text(address.place.as_deref());
544        self.add_optional_content_text(address.locality.as_deref());
545        self.add_optional_content_text(address.region.as_deref());
546        self.projected.postcode =
547            self.add_optional_postcode_for_search(address.postcode.as_deref());
548        self.add_optional_content_text(address.country.as_deref());
549    }
550
551    fn postcode(&mut self, value: &str) {
552        self.projected.postcode = collapse_whitespace(value);
553        self.add_content_text(value);
554        self.add_postcode_subject_text(value);
555    }
556
557    fn add_optional_autocomplete_text(&mut self, value: Option<&str>) {
558        let Some(cleaned) = value.and_then(collapse_whitespace) else {
559            return;
560        };
561        self.add_autocomplete_subject_text(&cleaned);
562    }
563
564    fn add_optional_content_text(&mut self, value: Option<&str>) {
565        if let Some(cleaned) = value.and_then(collapse_whitespace) {
566            self.add_content_text(&cleaned);
567        }
568    }
569
570    fn add_optional_postcode_text(&mut self, value: Option<&str>) -> Option<String> {
571        let cleaned = value.and_then(collapse_whitespace)?;
572        self.add_content_text(&cleaned);
573        self.add_postcode_subject_text(&cleaned);
574        Some(cleaned)
575    }
576
577    fn add_optional_postcode_for_search(&mut self, value: Option<&str>) -> Option<String> {
578        let cleaned = value.and_then(collapse_whitespace)?;
579        self.add_content_text(&cleaned);
580        Some(cleaned)
581    }
582
583    fn add_content_text(&mut self, value: &str) {
584        if let Some(normalized) = normalize_index_text(value) {
585            self.content_parts.push(normalized);
586        }
587    }
588
589    fn add_autocomplete_subject_text(&mut self, value: &str) {
590        if let Some(normalized) = normalize_index_text(value) {
591            self.autocomplete_subject_parts.push(normalized);
592        }
593    }
594
595    fn add_postcode_subject_text(&mut self, value: &str) {
596        if let Some(normalized) = normalize_index_text(value) {
597            self.content_parts.push(normalized.clone());
598            self.autocomplete_subject_parts.push(normalized.clone());
599            let compact = normalized.split_whitespace().collect::<String>();
600            if compact != normalized {
601                self.content_parts.push(compact.clone());
602                self.autocomplete_subject_parts.push(compact);
603            }
604        }
605    }
606
607    fn build(mut self) -> TextIndexProjection {
608        let content_parts = unique_parts(self.content_parts);
609        self.projected.content_text = content_parts.join(" ");
610        let autocomplete_subject_parts = unique_parts(self.autocomplete_subject_parts);
611        self.projected.autocomplete_subject_text = autocomplete_subject_parts.join(" ");
612
613        TextIndexProjection {
614            document: self.projected,
615        }
616    }
617}
618
619fn add_text_if_present(document: &mut TantivyDocument, field: Field, value: Option<&str>) {
620    if let Some(value) = value.and_then(collapse_whitespace) {
621        document.add_text(field, value);
622    }
623}
624
625fn add_normalized_text_if_present(
626    document: &mut TantivyDocument,
627    field: Field,
628    value: Option<&str>,
629) {
630    if let Some(value) = value.and_then(normalize_index_text) {
631        document.add_text(field, value);
632    }
633}
634
635pub(crate) fn normalize_index_text(value: &str) -> Option<String> {
636    let mut normalized = String::with_capacity(value.len());
637    let mut previous_was_space = true;
638    for character in value.chars() {
639        if character.is_alphanumeric() {
640            for folded in character.to_lowercase() {
641                normalized.push(folded);
642            }
643            previous_was_space = false;
644        } else if !previous_was_space {
645            normalized.push(' ');
646            previous_was_space = true;
647        }
648    }
649    let normalized = normalized.trim().to_string();
650    if normalized.is_empty() {
651        None
652    } else {
653        Some(normalized)
654    }
655}
656
657fn unique_parts(parts: Vec<String>) -> Vec<String> {
658    let mut seen = BTreeSet::new();
659    parts
660        .into_iter()
661        .filter(|part| seen.insert(part.clone()))
662        .collect()
663}
664
665#[cfg(test)]
666mod tests {
667    use std::collections::BTreeMap;
668
669    use crate::record::{
670        AddressRecord, DerivedSourceProvenance, InterpolationRange, LocationPrecision,
671        OsmObjectType, PlaceLayer, PostcodeRecord, SourceProvenance, point_geometry,
672    };
673
674    use super::*;
675
676    #[test]
677    fn projects_address_fields_for_search() {
678        let record = AddressRecord {
679            address: AddressComponents {
680                number: "221B".to_string(),
681                street: Some("Baker Street".to_string()),
682                place: None,
683                unit: None,
684                locality: Some("London".to_string()),
685                region: None,
686                postcode: Some("NW1".to_string()),
687                country: Some("GB".to_string()),
688            },
689            geometry: point_geometry(-0.1586, 51.5237),
690            location_precision: LocationPrecision::Point,
691            source: SourceProvenance::osm(OsmObjectType::Node, 123),
692        };
693
694        let projected = TextIndexDocument::from_record(42, &record.into());
695
696        assert_eq!(projected.record_id, 42);
697        assert_eq!(projected.layer, "address");
698        assert_eq!(projected.address_number.as_deref(), Some("221B"));
699        assert!(projected.content_text.contains("221b baker street"));
700        assert_eq!(projected.autocomplete_subject_text, "baker street nw1");
701        assert!(!projected.autocomplete_subject_text.contains("london"));
702    }
703
704    #[test]
705    fn projects_interpolation_without_indexing_range_fields() {
706        let record = crate::record::InterpolationRecord {
707            address: InterpolationAddressComponents {
708                street: Some("Baker Street".to_string()),
709                place: None,
710                locality: Some("London".to_string()),
711                region: None,
712                postcode: Some("NW1".to_string()),
713                country: Some("GB".to_string()),
714            },
715            interpolation: InterpolationRange {
716                kind: "odd".to_string(),
717                start: 1,
718                end: 99,
719                step: 2,
720            },
721            anchor_node_ids: [1, 2],
722            geometry: point_geometry(-0.1586, 51.5237),
723            representative_point: [-0.1586, 51.5237],
724            source: SourceProvenance::osm(OsmObjectType::Way, 9),
725        };
726
727        let projected = TextIndexDocument::from_record(7, &record.into());
728
729        assert_eq!(projected.address_number, None);
730        assert_eq!(projected.postcode.as_deref(), Some("NW1"));
731        assert!(projected.content_text.contains("baker street"));
732        assert!(projected.autocomplete_subject_text.is_empty());
733    }
734
735    #[test]
736    fn projects_postcodes_and_place_layers() {
737        let postcode = PostcodeRecord {
738            postcode: "M5V".to_string(),
739            geometry: point_geometry(-79.4, 43.6),
740            source: DerivedSourceProvenance::osm_address_records(2),
741        };
742        let place = PlaceRecord {
743            name: "Toronto".to_string(),
744            place_type: "city".to_string(),
745            geometry: point_geometry(-79.4, 43.6),
746            source: SourceProvenance {
747                dataset: "osm".to_string(),
748                object_type: OsmObjectType::Node,
749                object_id: 1,
750                tags: Some(BTreeMap::new()),
751            },
752        };
753
754        let postcode = TextIndexDocument::from_record(1, &postcode.into());
755        let place = TextIndexDocument::from_record(2, &Record::Place(PlaceLayer::Locality, place));
756
757        assert_eq!(postcode.postcode.as_deref(), Some("M5V"));
758        assert_eq!(place.layer, "locality");
759        assert!(place.content_text.contains("toronto"));
760        assert_eq!(postcode.autocomplete_subject_text, "m5v");
761        assert_eq!(place.autocomplete_subject_text, "toronto");
762    }
763}
open_geocode/text_index.rs

open_geocode/
text_index.rs
