gistools/readers/s2tiles/
mod.rs

1use crate::{
2    data_structures::Cache,
3    parsers::{Buffer, Reader},
4    util::{CompressionFormat, decompress_data},
5};
6use alloc::{collections::BTreeMap, vec, vec::Vec};
7use s2_tilejson::Metadata;
8use s2json::Face;
9
10/// A directory consists of an offset and a length pointing to a node or a leaf.
11/// The maximum value for a 6-byte offset is `281,474,976,710,655`
12/// This is large enough to address 281 TB of byte-indexed data.
13/// - Offset: 6 bytes
14/// - Length: 4 bytes
15#[derive(Debug, Clone, Copy, PartialEq)]
16struct Directory {
17    pub offset: u64,
18    pub length: u32,
19}
20
21const NODE_SIZE: usize = 10; // [offset, length] => [6 bytes, 4 bytes]
22const DIR_SIZE: usize = 1_365 * NODE_SIZE; // (13_650) -> 6 levels, the 6th level has both node and leaf (1+4+16+64+256+1024)*2 => (1365)+1365 => 2_730
23const METADATA_SIZE: usize = 131_072; // 131,072 bytes is 128kB. It is assumed the map metadata AND the S2Tile format metadata is less than 128kB
24const ROOT_DIR_SIZE: usize = DIR_SIZE * 6; // 27_300 * 6 = 163_800
25const ROOT_SIZE: usize = METADATA_SIZE + ROOT_DIR_SIZE;
26// assuming all tiles exist for every face from 0->30 the max leafs to reach depth of 30 is 5
27// root: 6sides * 27_300 bytes/dir = (163_800 bytes)
28// all leafs at 6: 1024 * 6sides * 27_300bytes/dir (0.167731 GB)
29// al leafs at 12: 524_288 * 6sides * 27_300bytes/dir (85.8783744 GB) - obviously most of this is water
30
31/// # S2 Tiles Reader
32///
33/// ## Description
34///
35/// An S2 Tile Reader to store tile and metadata in a cloud optimized format. Similar to PMTiles
36/// but simplified to have as few features as possible.
37///
38/// Reads either a Web Mercator tile or an S2 tile to the folder location given its (zoom, x, y) or (face, zoom, x, y) coordinates.
39///
40/// Reads data via the [S2Tiles specification](https://github.com/Open-S2/s2tiles/blob/master/s2tiles-spec/1.0.0/README.md).
41///
42/// ## Usage
43///
44/// S2TilesReader utilizes any struct that implements the [`Reader`] trait.
45/// Options are [`crate::parsers::BufferReader`], [`crate::parsers::FileReader`], [`crate::parsers::MMapReader`], and [`crate::parsers::FetchReader`].
46///
47/// The methods you have access to:
48/// - [`S2TilesReader::new`]: Create a new S2TilesReader
49/// - [`S2TilesReader::get_metadata`]: Get the metadata of the archive
50/// - [`S2TilesReader::has_tile_wm`]: Check if a WM tile exists in the archive
51/// - [`S2TilesReader::has_tile_s2`]: Check if an S2 tile exists in the archive
52/// - [`S2TilesReader::get_tile_wm`]: Get the bytes of the tile at the given (zoom, x, y) coordinates
53/// - [`S2TilesReader::get_tile_s2`]: Get the bytes of the tile at the given (face, zoom, x, y) coordinates
54///
55/// ```rust
56/// use gistools::{parsers::FileReader, readers::S2TilesReader};
57/// use std::path::PathBuf;
58///
59/// let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
60///     .join("tests/writers/fixtures/example.s2tiles");
61/// let file_reader = FileReader::new(path).unwrap();
62/// let mut reader = S2TilesReader::new(file_reader, None);
63///
64/// smol::block_on(async {
65///
66/// // get the metadata
67/// let metadata = reader.get_metadata().await;
68///
69/// // S2 specific functions
70/// assert!(reader.has_tile_s2(0.into(), 0, 0, 0).await);
71/// let tile = reader.get_tile_s2(0.into(), 0, 0, 0).await;
72///
73/// // WM functions
74/// assert!(reader.has_tile_wm(0, 0, 0).await);
75/// let tile = reader.get_tile_wm(0, 0, 0).await;
76///
77/// });
78/// ```
79///
80/// ## Links
81/// - https://github.com/Open-S2/s2tiles/blob/master/s2tiles-spec/1.0.0/README.md
82#[derive(Debug)]
83pub struct S2TilesReader<R: Reader> {
84    is_setup: bool,
85    version: u16,
86    maxzoom: u8,
87    compression: CompressionFormat,
88    metadata: Option<Metadata>,
89    root_dir: BTreeMap<u8, Buffer>,
90    dir_cache: Cache<u64, Buffer>,
91    reader: R,
92}
93impl<R: Reader> S2TilesReader<R> {
94    /// Create a new S2TilesReader
95    ///
96    /// ## Parameters
97    /// - `reader` - the input reader to parse from
98    /// - `max_size` - the max size of the cache before dumping old data. Defaults to 20.
99    pub fn new(reader: R, max_size: Option<usize>) -> Self {
100        Self {
101            is_setup: false,
102            version: 1,
103            maxzoom: 0,
104            compression: CompressionFormat::Gzip,
105            metadata: None,
106            root_dir: BTreeMap::new(),
107            dir_cache: Cache::new(max_size.unwrap_or(20), None),
108            reader,
109        }
110    }
111
112    /// Get the metadata of the archive
113    ///
114    /// ## Returns
115    /// The metadata of the archive
116    pub async fn get_metadata(&mut self) -> Metadata {
117        if let Some(metadata) = &self.metadata {
118            return metadata.clone();
119        }
120        self.setup().await;
121        self.metadata.clone().unwrap()
122    }
123
124    /// Check if a WM tile exists in the archive
125    ///
126    /// ## Parameters
127    /// - `zoom`: the zoom level of the tile
128    /// - `x`: the x coordinate of the tile
129    /// - `y`: the y coordinate of the tile
130    ///
131    /// ## Returns
132    /// True if the tile exists in the archive
133    pub async fn has_tile_wm(&mut self, zoom: u8, x: u32, y: u32) -> bool {
134        self.setup().await;
135        self.has_tile_s2(0.into(), zoom, x, y).await
136    }
137
138    /// Check if an S2 tile exists in the archive
139    ///
140    /// ## Parameters
141    /// - `face`: the Open S2 projection face
142    /// - `zoom`: the zoom level of the tile
143    /// - `x`: the x coordinate of the tile
144    /// - `y`: the y coordinate of the tile
145    ///
146    /// ## Returns
147    /// True if the tile exists in the archive
148    pub async fn has_tile_s2(&mut self, face: Face, zoom: u8, x: u32, y: u32) -> bool {
149        self.setup().await;
150        // pull in the correct face's directory
151        let dir = self.root_dir.get(&(face as u8)).cloned().unwrap();
152        // now we walk to the next directory as necessary
153        let node = self.walk(dir, zoom, x, y).await; // [offset, length]
154        if let Some(node) = node {
155            let Directory { offset, length } = node;
156            offset != 0 && length != 0
157        } else {
158            false
159        }
160    }
161
162    /// Get the bytes of the tile at the given (zoom, x, y) coordinates
163    ///
164    /// ## Parameters
165    /// - `zoom`: the zoom level of the tile
166    /// - `x`: the x coordinate of the tile
167    /// - `y`: the y coordinate of the tile
168    ///
169    /// ## Returns
170    /// The bytes of the tile at the given (z, x, y) coordinates, or undefined if the tile
171    /// does not exist in the archive.
172    pub async fn get_tile_wm(&mut self, zoom: u8, x: u32, y: u32) -> Option<Vec<u8>> {
173        self.setup().await;
174        self.get_tile_s2(0.into(), zoom, x, y).await
175    }
176
177    /// Get the bytes of the tile at the given (face, zoom, x, y) coordinates
178    ///
179    /// ## Parameters
180    /// - `face`: the Open S2 projection face
181    /// - `zoom`: the zoom level of the tile
182    /// - `x`: the x coordinate of the tile
183    /// - `y`: the y coordinate of the tile
184    ///
185    /// ## Returns
186    /// The bytes of the tile at the given (face, zoom, x, y) coordinates, or undefined if
187    /// the tile does not exist in the archive.
188    pub async fn get_tile_s2(&mut self, face: Face, zoom: u8, x: u32, y: u32) -> Option<Vec<u8>> {
189        self.setup().await;
190
191        // pull in the correct face's directory
192        let dir = self.root_dir.get(&(face as u8)).cloned().unwrap();
193        // now we walk to the next directory as necessary
194        let node = self.walk(dir, zoom, x, y).await; // [offset, length]
195        if let Some(node) = node {
196            let Directory { offset, length } = node;
197
198            // we found the vector file, let's send the details off to the tile worker
199            let data = self.get_range(offset, length as u64).await;
200            Some(decompress_data(&data, self.compression).unwrap())
201        } else {
202            None
203        }
204    }
205
206    /// given position and level, find the tile offset and length
207    ///
208    /// ## Parameters
209    /// - `dir`: the directory to walk
210    /// - `zoom`: the zoom level of the tile
211    /// - `x`: the x coordinate of the tile
212    /// - `y`: the y coordinate of the tile
213    ///
214    /// ## Returns
215    /// The offset and length of the tile if it exists
216    async fn walk(&mut self, mut dir: Buffer, zoom: u8, x: u32, y: u32) -> Option<Directory> {
217        let mut path = get_s2_tile_path(zoom, x, y);
218        let mut offset = 0;
219        let mut length = 0;
220
221        // walk the tree if past zoom 0
222        while !path.is_empty() {
223            // grab position
224            let node_pos = path.remove(0) as usize * NODE_SIZE;
225            // set
226            offset = read_uint_48le(&mut dir, Some(node_pos as usize));
227            length = dir.get_u32_at(node_pos + 6);
228            if length == 0 {
229                return None;
230            }
231            // if we are still walking, grab the new directory
232            if !path.is_empty() {
233                // corner case: if maxzoom matches the zoom and is divisible by 5, the leaf is actually a node
234                if self.maxzoom.is_multiple_of(5)
235                    && zoom == self.maxzoom
236                    && path.len() == 1
237                    && path[0] == 0
238                {
239                    return Some(Directory { offset, length });
240                }
241                // otherwise fetch the directory
242                let next_dir = self.get_dir(offset, length).await;
243                dir = next_dir;
244            }
245        }
246
247        if length == 0 { None } else { Some(Directory { offset, length }) }
248    }
249
250    /// get a directory given an offset and length
251    ///
252    /// ## Parameters
253    /// - `offset`: the offset
254    /// - `length`: the length
255    ///
256    /// ## Returns
257    /// The directory
258    async fn get_dir(&mut self, offset: u64, length: u32) -> Buffer {
259        if let Some(dir) = self.dir_cache.get(&offset) {
260            dir.clone()
261        } else {
262            let data = self.get_range(offset, length as u64).await;
263            let dir = Buffer::new(data);
264            self.dir_cache.set(offset, dir.clone());
265            dir
266        }
267    }
268
269    /// Setup the reader
270    async fn setup(&mut self) {
271        if self.is_setup {
272            return;
273        }
274        self.is_setup = true;
275        // fetch the metadata
276        let data = self.get_range(0, ROOT_SIZE as u64).await;
277        // prep a data view, store in header, build metadata
278        let mut dv = Buffer::new(data.clone());
279        if dv.get_u16_at(0) != 12883 {
280            // the first two bytes are S and 2, we validate
281            panic!("Bad metadata");
282        }
283        // parse the version, maxzoom, and compression
284        self.version = dv.get_u16_at(2);
285        self.maxzoom = dv.get_u8_at(4);
286        self.compression = CompressionFormat::from(dv.get_u8_at(5));
287        // parse the JSON metadata length and offset
288        let m_l = dv.get_u32_at(6);
289        if m_l == 0 {
290            // if the metadata is empty, we failed
291            panic!("Failed to extrapolate metadata");
292        }
293        let meta_data =
294            decompress_data(&data[10..(10 + (m_l as usize))], self.compression).unwrap();
295        self.metadata = Some(serde_json::from_slice(&meta_data).unwrap());
296        // create root directories
297        for face in [0, 1, 2, 3, 4, 5] {
298            let start = METADATA_SIZE + (face as usize) * DIR_SIZE;
299            self.root_dir.insert(face, Buffer::new(data[start..(start + DIR_SIZE)].to_vec()));
300        }
301    }
302
303    /// Get a range of bytes given an offset and length
304    async fn get_range(&mut self, offset: u64, length: u64) -> Vec<u8> {
305        let len = self.reader.len();
306        if len != 0 {
307            // This is not a FetchReader
308            let end = u64::min(len, offset + length);
309            self.reader.slice(Some(offset), Some(end))
310        } else {
311            self.reader.get_slice(offset, Some(length)).await
312        }
313    }
314}
315
316/// read a 48 bit number
317///
318/// ## Parameters
319/// - `buffer`: the buffer
320/// - `offset`: the offset
321///
322/// ## Returns
323/// The number
324fn read_uint_48le(buffer: &mut Buffer, offset: Option<usize>) -> u64 {
325    let offset = offset.unwrap_or(0);
326    buffer.get_u32_at(offset + 2) as u64 * (1 << 16) + buffer.get_u16_at(offset) as u64
327}
328
329/// Get the path to a tile
330///
331/// ## Parameters
332/// - `zoom`: the zoom
333/// - `x`: the x
334/// - `y`: the y
335///
336/// ## Returns
337/// The path as a collection of offsets pointing to the tile Node in the directory
338pub fn get_s2_tile_path(mut zoom: u8, mut x: u32, mut y: u32) -> Vec<u64> {
339    let mut path = vec![];
340
341    while zoom >= 5 {
342        path.push((5, x & 31, y & 31));
343        x >>= 5;
344        y >>= 5;
345        zoom = zoom.saturating_sub(5);
346    }
347    path.push((zoom, x, y));
348
349    path.into_iter()
350        .map(|(zoom, x, y)| {
351            let val = (y as u64) * ((1 << zoom) as u64) + (x as u64);
352            let sum: u64 = (0..zoom).map(|z| (1 << z) * (1 << z)).sum();
353            val + sum
354        })
355        .collect()
356}
gistools/readers/s2tiles/mod.rs

gistools/readers/s2tiles/
mod.rs
