gistools/readers/pmtiles/
pm_spec.rs

1use crate::{parsers::Buffer, util::CompressionFormat};
2use alloc::{string::String, vec::Vec};
3use core::cmp::Ordering;
4use pbf::bit_cast::BitCast;
5
6/// zoom values for each zoom level. Supports up to 27 zooms
7pub const PM_TZ_VALUES: [u64; 27] = [
8    0,
9    1,
10    5,
11    21,
12    85,
13    341,
14    1365,
15    5461,
16    21845,
17    87381,
18    349525,
19    1398101,
20    5592405,
21    22369621,
22    89478485,
23    357913941,
24    1431655765,
25    5726623061,
26    22906492245,
27    91625968981,
28    366503875925,
29    1466015503701,
30    5864062014805,
31    23456248059221,
32    93824992236885,
33    375299968947541,
34    1501199875790165,
35];
36/// the number of bytes in the header
37pub const PM_HEADER_SIZE_BYTES: usize = 127;
38/// the number of bytes in the root
39pub const PM_ROOT_SIZE: usize = 16_384;
40
41/// An array of two numbers representing a point in 2D space
42#[derive(Debug)]
43pub struct PMPoint2D {
44    /// x coordinate
45    pub x: i64,
46    /// y coordinate
47    pub y: i64,
48}
49
50/// A tile, in the format of ZXY
51#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
52pub struct PMTilePos {
53    /// zoom level
54    pub zoom: u8,
55    /// x coordinate
56    pub x: u64,
57    /// y coordinate
58    pub y: u64,
59}
60impl PMTilePos {
61    /// Create a Tile instance from a zoom, x, and y
62    pub fn new(zoom: u8, x: u64, y: u64) -> Self {
63        Self { zoom, x, y }
64    }
65
66    /// Create a Tile instance from an ID
67    pub fn from_id(id: u64) -> Self {
68        let mut acc = 0;
69
70        for z in 0..27 {
71            let num_tiles = (0x1 << z) * (0x1 << z);
72            if acc + num_tiles > id {
73                return Self::from_zoom_pos(z, id - acc);
74            }
75            acc += num_tiles;
76        }
77
78        unreachable!()
79    }
80
81    /// Create a Tile instance from a zoom and position
82    pub fn from_zoom_pos(zoom: u8, pos: u64) -> Self {
83        let n: i64 = 1 << zoom;
84        let mut t = pos as i64;
85        let mut xy = PMPoint2D { x: 0, y: 0 };
86        let mut s: i64 = 1;
87        while s < n {
88            let rx: i64 = 1 & (t / 2);
89            let ry: i64 = 1 & (t ^ rx);
90            rotate(s, &mut xy, rx, ry);
91            xy.x += s * rx;
92            xy.y += s * ry;
93            t /= 4;
94            s *= 2;
95        }
96        Self { zoom, x: xy.x as u64, y: xy.y as u64 }
97    }
98
99    /// Convert a Tile instance to an ID
100    pub fn to_id(&self) -> u64 {
101        if self.zoom > 26
102            || self.x > 2u64.pow(self.zoom as u32) - 1
103            || self.y > 2u64.pow(self.zoom as u32) - 1
104        {
105            unreachable!()
106        }
107
108        let n: u64 = 1 << self.zoom;
109        let mut d: i64 = 0;
110        let mut xy = PMPoint2D { x: self.x as i64, y: self.y as i64 };
111        let mut s: i64 = (n as i64) / 2;
112        loop {
113            let rx = if (xy.x & s) > 0 { 1 } else { 0 };
114            let ry = if (xy.y & s) > 0 { 1 } else { 0 };
115            d += s * s * ((3 * rx) ^ ry);
116            rotate(s, &mut xy, rx, ry);
117            if s <= 1 {
118                break;
119            }
120            s /= 2;
121        }
122
123        PM_TZ_VALUES[self.zoom as usize] + (d as u64)
124    }
125}
126
127/// PMTiles v3 directory entry.
128#[derive(Debug, Clone, Copy, Default, PartialEq)]
129pub struct PMEntry {
130    /// tile ID
131    pub tile_id: u64,
132    /// offset relative to where tile data starts in the file
133    pub offset: u64,
134    /// length in bytes
135    pub length: u32,
136    /// run length
137    pub run_length: u32,
138}
139impl PMEntry {
140    /// Create a new directory entry
141    pub fn new(tile_id: u64, offset: u64, length: u32, run_length: u32) -> Self {
142        Self { tile_id, offset, length, run_length }
143    }
144}
145
146/// PMTiles v3 directory. A collection of Entry instances for storage
147#[derive(Debug, Clone, Default, PartialEq)]
148pub struct PMDirectory {
149    /// entries
150    pub entries: Vec<PMEntry>,
151}
152impl PMDirectory {
153    /// Create a new directory
154    pub fn new(entries: Vec<PMEntry>) -> Self {
155        Self { entries }
156    }
157
158    /// Create a new directory from a buffer
159    pub fn from_buffer(buffer: &mut Buffer) -> Self {
160        let num_entries = buffer.read_varint::<usize>();
161
162        let mut entries: Vec<PMEntry> = Vec::new();
163
164        let mut last_id = 0;
165        for _ in 0..num_entries {
166            let v = buffer.read_varint::<u64>();
167            entries.push(PMEntry::new(last_id + v, 0, 0, 1));
168            last_id += v;
169        }
170
171        // run lengths, lengths, and offsets
172        entries.iter_mut().for_each(|e| e.run_length = buffer.read_varint::<u32>());
173        entries.iter_mut().for_each(|e| e.length = buffer.read_varint::<u32>());
174        for i in 0..num_entries {
175            let v = buffer.read_varint::<u64>();
176            if v == 0 && i > 0 {
177                entries[i].offset = entries[i - 1].offset + entries[i - 1].length as u64;
178            } else {
179                entries[i].offset = v.saturating_sub(1);
180            }
181        }
182
183        Self { entries }
184    }
185
186    /// Serialize the directory into a buffer
187    pub fn serialize(&self) -> Vec<u8> {
188        // then write the entries
189        let mut buffer = Buffer::default();
190
191        buffer.write_varint(self.entries.len().to_u64());
192
193        let mut last_id = 0;
194        for e in &self.entries {
195            buffer.write_varint(e.tile_id - last_id);
196            last_id = e.tile_id;
197        }
198
199        for e in &self.entries {
200            buffer.write_varint(e.run_length);
201        }
202        for e in &self.entries {
203            buffer.write_varint(e.length);
204        }
205        for i in 0..self.entries.len() {
206            if i > 0
207                && self.entries[i].offset
208                    == self.entries[i - 1].offset + self.entries[i - 1].length as u64
209            {
210                buffer.write_varint(0);
211            } else {
212                buffer.write_varint(self.entries[i].offset + 1);
213            }
214        }
215
216        buffer.take()
217    }
218
219    /// Check if the directory is empty
220    pub fn is_empty(&self) -> bool {
221        self.entries.is_empty()
222    }
223
224    /// Get the number of entries
225    pub fn len(&self) -> usize {
226        self.entries.len()
227    }
228
229    /// Get an entry
230    pub fn get(&self, id: u64) -> Option<&PMEntry> {
231        self.entries.iter().find(|e| e.tile_id == id)
232    }
233
234    /// Get an entry mutable
235    pub fn get_mut(&mut self, id: u64) -> Option<&mut PMEntry> {
236        self.entries.iter_mut().find(|e| e.tile_id == id)
237    }
238
239    /// Set an entry
240    pub fn set(&mut self, id: u64, entry: PMEntry) {
241        if let Some(e) = self.get_mut(id) {
242            *e = entry;
243        } else {
244            self.entries.push(entry);
245        }
246    }
247
248    /// Insert an entry
249    pub fn insert(&mut self, entry: PMEntry) {
250        self.entries.push(entry);
251    }
252
253    /// Get the first entry
254    pub fn first(&self) -> Option<&PMEntry> {
255        self.entries.first()
256    }
257
258    /// Get the first entry mutable
259    pub fn first_mut(&mut self) -> Option<&mut PMEntry> {
260        self.entries.first_mut()
261    }
262
263    /// Get the last entry
264    pub fn last(&self) -> Option<&PMEntry> {
265        self.entries.last()
266    }
267
268    /// Get the last entry mutable
269    pub fn last_mut(&mut self) -> Option<&mut PMEntry> {
270        self.entries.last_mut()
271    }
272}
273
274/// Describe the type of tiles stored in the archive.
275/// 0 is unknown/other, 1 is "MVT" vector tiles.
276#[derive(Debug, Copy, Clone, Default, PartialEq)]
277pub enum PMTileType {
278    /// unknown/other.
279    Unknown = 0,
280    /// Vector tiles.
281    #[default]
282    Pbf = 1,
283    /// Image tiles.
284    Png = 2,
285    /// Image tiles.
286    Jpeg = 3,
287    /// Image tiles.
288    Webp = 4,
289    /// Image tiles.
290    Avif = 5,
291}
292impl From<u8> for PMTileType {
293    fn from(value: u8) -> Self {
294        match value {
295            1 => PMTileType::Pbf,
296            2 => PMTileType::Png,
297            3 => PMTileType::Jpeg,
298            4 => PMTileType::Webp,
299            5 => PMTileType::Avif,
300            _ => PMTileType::Unknown,
301        }
302    }
303}
304impl From<PMTileType> for u8 {
305    fn from(t_type: PMTileType) -> Self {
306        match t_type {
307            PMTileType::Unknown => 0,
308            PMTileType::Pbf => 1,
309            PMTileType::Png => 2,
310            PMTileType::Jpeg => 3,
311            PMTileType::Webp => 4,
312            PMTileType::Avif => 5,
313        }
314    }
315}
316impl From<PMTileType> for String {
317    fn from(t_type: PMTileType) -> Self {
318        match t_type {
319            PMTileType::Unknown => "unknown".into(),
320            PMTileType::Pbf => "pbf".into(),
321            PMTileType::Png => "png".into(),
322            PMTileType::Jpeg => "jpeg".into(),
323            PMTileType::Webp => "webp".into(),
324            PMTileType::Avif => "avif".into(),
325        }
326    }
327}
328
329/// PMTiles v3 header storing basic archive-level information.
330#[derive(Debug, Default, PartialEq)]
331pub struct PMHeader {
332    /// Only v3 PMTiles supported
333    pub version: u8,
334    /// the offset in the archive of the root directory
335    pub root_directory_offset: u64,
336    /// the length of the root directory
337    pub root_directory_length: u64,
338    /// the offset in the archive of the JSON metadata
339    pub metadata_offset: u64,
340    /// the length of the metadata
341    pub metadata_length: u64,
342    /// the offset in the archive of the leaf directory
343    pub leaf_directory_offset: u64,
344    /// the length of the leaf directory
345    pub leaf_directory_length: u64,
346    /// the offset in the archive of the tile data
347    pub data_offset: u64,
348    /// the length of the tile data
349    pub data_length: u64,
350    /// the number of addressed tiles
351    pub n_addressed_tiles: u64,
352    /// the number of tile entries
353    pub n_tile_entries: u64,
354    /// the number of tile contents
355    pub n_tile_contents: u64,
356    /// if the archive is clustered
357    pub clustered: bool,
358    /// what kind of compression is used for the Entries and metadata
359    /// This is depreacted and will always be NONE for S2PMTiles
360    pub internal_compression: CompressionFormat,
361    /// what kind of compression is used for the tiles
362    pub tile_compression: CompressionFormat,
363    /// the type of the tiles
364    pub tile_type: PMTileType,
365    /// the min zoom level
366    pub min_zoom: u8,
367    /// the max zoom level
368    pub max_zoom: u8,
369    /// the min longitude
370    pub min_longitude: f32,
371    /// the min latitude
372    pub min_latitude: f32,
373    /// the max longitude
374    pub max_longitude: f32,
375    /// the max latitude
376    pub max_latitude: f32,
377    /// the center zoom level
378    pub center_zoom: u8,
379    /// the center longitude
380    pub center_longitude: f32,
381    /// the center latitude
382    pub center_latitude: f32,
383}
384impl PMHeader {
385    /// Create a new Header from a buffer
386    pub fn from_bytes(buffer: &mut Buffer) -> Self {
387        Self {
388            version: buffer.get_u8_at(7),
389            root_directory_offset: buffer.get_u64_at(8),
390            root_directory_length: buffer.get_u64_at(16),
391            metadata_offset: buffer.get_u64_at(24),
392            metadata_length: buffer.get_u64_at(32),
393            leaf_directory_offset: buffer.get_u64_at(40),
394            leaf_directory_length: buffer.get_u64_at(48),
395            data_offset: buffer.get_u64_at(56),
396            data_length: buffer.get_u64_at(64),
397            n_addressed_tiles: buffer.get_u64_at(72),
398            n_tile_entries: buffer.get_u64_at(80),
399            n_tile_contents: buffer.get_u64_at(88),
400            clustered: buffer.get_u8_at(96) == 1,
401            internal_compression: CompressionFormat::from(buffer.get_u8_at(97)),
402            tile_compression: CompressionFormat::from(buffer.get_u8_at(98)),
403            tile_type: PMTileType::from(buffer.get_u8_at(99)),
404            min_zoom: buffer.get_u8_at(100),
405            max_zoom: buffer.get_u8_at(101),
406            min_longitude: (buffer.get_i32_at(102) as f32) / 10_000_000.0,
407            min_latitude: (buffer.get_i32_at(106) as f32) / 10_000_000.0,
408            max_longitude: (buffer.get_i32_at(110) as f32) / 10_000_000.0,
409            max_latitude: (buffer.get_i32_at(114) as f32) / 10_000_000.0,
410            center_zoom: buffer.get_u8_at(118),
411            center_longitude: (buffer.get_i32_at(119) as f32) / 10_000_000.0,
412            center_latitude: (buffer.get_i32_at(123) as f32) / 10_000_000.0,
413        }
414    }
415
416    /// Write the header to a buffer
417    pub fn to_bytes(&self) -> Buffer {
418        let mut buffer = Buffer::default();
419        // set id
420        buffer.set_u16_at(0, 0x4d50); // set PM
421        // Version number at position 7
422        buffer.set_u8_at(7, 3);
423        // Root directory offset and length at positions 8 and 16
424        buffer.set_u64_at(8, self.root_directory_offset);
425        buffer.set_u64_at(16, self.root_directory_length);
426        // JSON metadata offset and length at positions 24 and 32
427        buffer.set_u64_at(24, self.metadata_offset);
428        buffer.set_u64_at(32, self.metadata_length);
429        // Leaf directory offset and optional length at positions 40 and 48
430        buffer.set_u64_at(40, self.leaf_directory_offset);
431        buffer.set_u64_at(48, self.leaf_directory_length);
432        // Tile data offset and optional length at positions 56 and 64
433        buffer.set_u64_at(56, self.data_offset);
434        buffer.set_u64_at(64, self.data_length);
435        // Number of addressed tiles, tile entries, and tile contents at positions 72, 80, and 88
436        buffer.set_u64_at(72, self.n_addressed_tiles);
437        buffer.set_u64_at(80, self.n_tile_entries);
438        buffer.set_u64_at(88, self.n_tile_contents);
439        // Flags and types at positions 96 through 101
440        buffer.set_u8_at(96, if self.clustered { 1 } else { 0 });
441        buffer.set_u8_at(97, self.internal_compression.into());
442        buffer.set_u8_at(98, self.tile_compression.into());
443        buffer.set_u8_at(99, self.tile_type.into());
444        buffer.set_u8_at(100, self.min_zoom);
445        buffer.set_u8_at(101, self.max_zoom);
446        // Minimum and maximum coordinates
447        buffer.set_i32_at(102, (self.min_longitude * 10_000_000.0) as i32);
448        buffer.set_i32_at(106, (self.min_latitude * 10_000_000.0) as i32);
449        buffer.set_i32_at(110, (self.max_longitude * 10_000_000.0) as i32);
450        buffer.set_i32_at(114, (self.max_latitude * 10_000_000.0) as i32);
451        // Center zoom and center coordinates
452        buffer.set_u8_at(118, self.center_zoom);
453        buffer.set_i32_at(119, (self.center_longitude * 10_000_000.0) as i32);
454        buffer.set_i32_at(123, (self.center_latitude * 10_000_000.0) as i32);
455
456        buffer
457    }
458}
459
460/// rotate xy by n
461pub fn rotate(n: i64, xy: &mut PMPoint2D, rx: i64, ry: i64) {
462    if ry == 0 {
463        if rx == 1 {
464            xy.x = n - 1 - xy.x;
465            xy.y = n - 1 - xy.y;
466        }
467        core::mem::swap(&mut xy.x, &mut xy.y);
468    }
469}
470
471/// Low-level function for looking up a tile_id or leaf directory inside a directory.
472pub fn find_tile(entries: &[PMEntry], tile_id: u64) -> Option<PMEntry> {
473    if entries.is_empty() {
474        return None;
475    }
476    let mut m = 0;
477    let mut n: isize = (entries.len() - 1).try_into().unwrap();
478    while m <= n {
479        let k = (n + m) >> 1;
480        match tile_id.cmp(&entries[k as usize].tile_id) {
481            Ordering::Greater => m = k + 1,
482            Ordering::Less => n = k - 1,
483            Ordering::Equal => return Some(entries[k as usize]),
484        }
485    }
486
487    // at this point, m > n
488    if n >= 0 {
489        let n: usize = n as usize;
490        if entries[n].run_length == 0 {
491            return Some(entries[n]);
492        }
493        if tile_id - entries[n].tile_id < entries[n].run_length as u64 {
494            return Some(entries[n]);
495        }
496    }
497
498    None
499}
gistools/readers/pmtiles/pm_spec.rs

gistools/readers/pmtiles/
pm_spec.rs
