pbfhogg 0.5.0

Fast OpenStreetMap PBF reader and writer for Rust. Read, write, and merge .osm.pbf files with pipelined parallel decoding.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

//! Relation-member node-ID collection for ALTW external join.
//!
//! The generic `collect_relation_member_node_ids` in `add_locations_to_ways.rs`
//! reads every blob sequentially via `BlobReader::next()` - at planet scale
//! that's ~100 GB of compressed bytes preread (not decompressed) just to find
//! the ~2K relation blobs. External join already has a full `BlobMeta` table,
//! so this variant preads only the relation blobs directly and decompresses
//! each via `decompress_blob_raw`.

use std::os::unix::fs::FileExt as _;
use std::path::Path;
use std::sync::Arc;

use crate::blob_meta::ElemKind;
use crate::block::PrimitiveBlock;
use crate::elements::{Element, MemberId};

use super::super::Result;
use super::blob_meta::BlobMeta;
use crate::idset::IdSet;

pub(super) struct RelationScanOutput {
    pub(super) member_node_ids: Option<IdSet>,
    pub(super) member_way_ids: Option<IdSet>,
}

#[hotpath::measure]
pub(super) fn collect_relation_member_ids_indexed(
    input: &Path,
    blob_meta: &[BlobMeta],
    want_nodes: bool,
    want_ways: bool,
) -> Result<RelationScanOutput> {
    let file = Arc::new(
        std::fs::File::open(input).map_err(|e| format!("open pbf for relation scan: {e}"))?,
    );

    let mut node_ids = want_nodes.then(IdSet::new);
    let mut way_ids = want_ways.then(IdSet::new);
    let mut read_buf: Vec<u8> = Vec::new();
    let mut decompress_buf: Vec<u8> = Vec::new();
    let mut st_scratch: Vec<(u32, u32)> = Vec::new();
    let mut gr_scratch: Vec<(u32, u32)> = Vec::new();

    for meta in blob_meta
        .iter()
        .filter(|m| matches!(m.kind, ElemKind::Relation))
    {
        read_buf.resize(meta.data_size, 0);
        file.read_exact_at(&mut read_buf, meta.data_offset)
            .map_err(|e| format!("relation scan pread: {e}"))?;
        crate::blob::decompress_blob_raw(&read_buf, &mut decompress_buf)
            .map_err(|e| format!("relation scan decompress: {e}"))?;
        let block = PrimitiveBlock::from_vec_with_scratch(
            std::mem::take(&mut decompress_buf),
            &mut st_scratch,
            &mut gr_scratch,
        )?;
        for element in block.elements_skip_metadata() {
            if let Element::Relation(r) = element {
                let complete = r.tags().any(|(key, value)| {
                    key == "type" && matches!(value, "multipolygon" | "boundary")
                });
                for member in r.members() {
                    match member.id {
                        MemberId::Node(id) if want_nodes && id >= 0 => {
                            node_ids.as_mut().expect("requested").set(id);
                        }
                        MemberId::Way(id) if want_ways && complete && id >= 0 => {
                            way_ids.as_mut().expect("requested").set(id);
                        }
                        _ => {}
                    }
                }
            }
        }
    }

    Ok(RelationScanOutput {
        member_node_ids: node_ids,
        member_way_ids: way_ids,
    })
}