Skip to main content

manby/
lib.rs

1//! manby builds vector tilesets from geographic features, like tippecanoe, but
2//! several times faster. See DESIGN.md.
3//!
4//! [`run`] takes [`Options`] and writes the tileset, and [`join()`] takes
5//! [`JoinOptions`] and joins tilesets as tippecanoe's tile-join does. Nothing
6//! in manby exits the process; errors come back as [`Error`], with the exit
7//! status tippecanoe would use.
8//!
9//! The options are tippecanoe's, so its command line is the shortest way to
10//! them:
11//!
12//! ```no_run
13//! let args = ["manby", "-zg", "--drop-densest-as-needed", "-o", "places.pmtiles", "places.geojson"];
14//! let options = manby::options::parse_args(&args.map(String::from))?;
15//! let summary = manby::run(&options)?;
16//! println!("{} tiles, to zoom {}", summary.tiles, summary.maxzoom);
17//! # Ok::<(), manby::Error>(())
18//! ```
19//!
20//! They are also fields, set on the defaults:
21//!
22//! ```no_run
23//! use manby::options::Output;
24//!
25//! let mut options = manby::Options::default();
26//! options.inputs = vec!["places.geojson".to_string()];
27//! options.output = Output::Pmtiles("places.pmtiles".to_string());
28//! options.maxzoom = 12;
29//! manby::run(&options)?;
30//! # Ok::<(), manby::Error>(())
31//! ```
32//!
33//! # What is public
34//!
35//! The API is what these pages show: [`run`] and [`join()`] with their options
36//! and summaries, and the commands in [`stats`], [`serve`] and [`overzoom`].
37//! The options and summaries are `#[non_exhaustive]`, since each new option
38//! is a new field. Some fields have types of tippecanoe-rs, which is
39//! [`tippecanoe`] here.
40//!
41//! manby's other modules are public for its own tests, examples and
42//! benchmarks. They are hidden from these pages and can change in any
43//! release.
44//!
45//! # One run at a time
46//!
47//! The code that manby borrows from tippecanoe-rs reads its options from
48//! globals, which each run sets, and the memory budget is the process's. Runs
49//! in one process should take turns unless their options are the same.
50
51#[doc(hidden)]
52pub mod buf;
53#[doc(hidden)]
54pub mod clean;
55#[doc(hidden)]
56pub mod command;
57#[doc(hidden)]
58pub mod compress;
59#[doc(hidden)]
60pub mod crs;
61pub mod error;
62#[doc(hidden)]
63pub mod filter;
64#[doc(hidden)]
65pub mod ingest;
66pub mod join;
67#[doc(hidden)]
68pub mod mem;
69#[doc(hidden)]
70pub mod mlt;
71pub mod options;
72#[doc(hidden)]
73pub mod order;
74pub mod overzoom;
75mod pool;
76pub mod serve;
77pub mod shard;
78#[doc(hidden)]
79pub mod sink;
80pub mod stats;
81#[doc(hidden)]
82pub mod store;
83#[doc(hidden)]
84pub mod tiler;
85
86/// tippecanoe-rs, for the types of its that the options have.
87pub use tippecanoe;
88
89use std::collections::BTreeMap;
90
91use tippecanoe::errors::EXIT_NODATA;
92use tippecanoe::mbtiles::{LayermapEntry, Metadata, make_metadata, merge_layermaps};
93use tippecanoe::options::{P_TILE_COMPRESSION, P_TILE_STATS};
94use tippecanoe::projection::tile2lonlat;
95
96pub use error::{Error, Result};
97pub use filter::Filter;
98pub use join::{JoinOptions, JoinSummary, join};
99pub use mlt::{MltProfile, TileFormat};
100pub use options::Options;
101pub use shard::{MergeOptions, MergeSummary, merge};
102
103/// What a run produced.
104#[derive(Clone, Debug, Default)]
105#[non_exhaustive]
106pub struct Summary {
107    /// Features read.
108    pub features: u64,
109    /// Tiles written.
110    pub tiles: u64,
111    /// The lowest zoom written.
112    pub minzoom: i32,
113    /// The highest zoom written.
114    pub maxzoom: i32,
115    /// A tile could not be made small enough, so the tileset stops at
116    /// `maxzoom`, below the zoom that was planned. tippecanoe exits with
117    /// `EXIT_INCOMPLETE` then.
118    pub incomplete: bool,
119}
120
121/// The tiler's options for a run.
122#[doc(hidden)]
123pub fn tile_options(
124    o: &Options,
125    plan: &order::Plan,
126    store: &store::FeatureStore,
127) -> tiler::TileOptions {
128    tiler::TileOptions::new(o, plan, store)
129}
130
131/// Reads the inputs, tiles them and writes the tileset that `options` describes.
132pub fn run(options: &Options) -> Result<Summary> {
133    let mut o = options.clone();
134    for w in o.resolve()? {
135        eprintln!("{}", w);
136    }
137    // TRANSITIONAL: see set_tippecanoe_globals.
138    options::set_tippecanoe_globals(&o, o.cluster_maxzoom);
139    let budget = o.memory_budget();
140    buf::set_budget(budget, std::path::Path::new(&o.tmpdir));
141
142    let sink = sink::open(
143        &o.output,
144        &sink::SinkSettings {
145            force: o.force,
146            allow_existing: o.forcetable,
147            tile_compression: !o.prevent.get(P_TILE_COMPRESSION),
148            tmpdir: o.tmpdir.clone().into(),
149            memory_budget: (budget / 64).clamp(1 << 20, 16 << 20) as usize,
150            program: o.program.clone(),
151            tile_format: o.tile_format,
152            ..Default::default()
153        },
154    )?;
155
156    let mut store = ingest::read_inputs(&o)?;
157    let plan = order::order(&mut store, &o)?;
158    options::set_tippecanoe_globals(&o, plan.cluster_maxzoom);
159
160    let mut topts = tile_options(&o, &plan, &store);
161    let split = o.shard.map(|s| {
162        let zoom = shard::pivot(&o, &plan);
163        let ranges = shard::split(&store, &o, &plan, zoom, s.count);
164        let (lo, hi) = ranges[s.index as usize];
165        topts.shard = Some(shard::Range {
166            index: s.index,
167            zoom,
168            lo,
169            hi,
170            minzoom: plan.minzoom,
171        });
172        (s, zoom, ranges)
173    });
174    let stats = tiler::tile_with(&store, &topts, sink.as_ref())?;
175
176    // The tiler can also extend the zoom levels beyond the plan.
177    let minzoom = plan.minzoom;
178    let maxzoom = stats.maxzoom;
179    let incomplete = stats.incomplete;
180    if incomplete {
181        if maxzoom > minzoom {
182            eprintln!(
183                "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM {} ***",
184                maxzoom
185            );
186        } else {
187            return Err(Error::new(
188                EXIT_NODATA,
189                format!("{}: No zoom levels were successfully written", o.program),
190            ));
191        }
192    }
193
194    let inputs = MetaInputs::new(&o, &plan, &store);
195    let metadata = inputs.metadata(&o.tileset_name(), &stats);
196    match split {
197        Some((s, zoom, ranges)) => {
198            let tiling = stats.shard.clone().unwrap_or_default();
199            if !tiling.again.is_empty() {
200                eprintln!("{}", shard::again_note(s, &tiling.again));
201            }
202            let record = shard::Record {
203                index: s.index,
204                count: s.count,
205                zoom,
206                fingerprint: shard::fingerprint(&inputs, &plan, &store, zoom, &ranges),
207                ranges,
208                name: o.name.clone(),
209                meta: inputs,
210                tiling,
211            };
212            let mut extra = record.meta.extra();
213            extra.push((shard::KEY.to_string(), record.to_json()));
214            sink.finish_with(&metadata, &extra)?;
215        }
216        None => sink.finish_with(&metadata, &inputs.extra())?,
217    }
218
219    Ok(Summary {
220        features: store.features.len() as u64,
221        tiles: stats.tiles,
222        minzoom,
223        maxzoom,
224        incomplete,
225    })
226}
227
228/// The metadata's key for `--h3`'s resolutions and zooms.
229pub(crate) const H3_KEY: &str = "manby_h3";
230
231/// What a run's metadata is made of besides its name and what the tiling
232/// found, which `manby merge` gets from each shard.
233#[doc(hidden)]
234#[derive(Clone, Debug, Default)]
235pub struct MetaInputs {
236    pub minzoom: i32,
237    /// The input's bounding boxes in world coordinates: the whole one, and
238    /// the two that put the antimeridian in different places.
239    pub bbox: [i64; 4],
240    pub bbox1: [i64; 4],
241    pub bbox2: [i64; 4],
242    pub attribution: Option<String>,
243    pub description: Option<String>,
244    pub tilestats: bool,
245    pub attribute_descriptions: BTreeMap<String, String>,
246    pub commandline: String,
247    pub basezoom: i32,
248    pub droprate: f64,
249    pub retain_points_multiplier: i32,
250    /// The input's layers.
251    pub layermap: BTreeMap<String, LayermapEntry>,
252    /// How the output stores its tiles.
253    pub tile_compression: bool,
254    pub tile_format: TileFormat,
255    /// `--h3`: the hexagons' resolutions and zooms, as JSON, for
256    /// `manby serve`'s map.
257    pub h3: Option<String>,
258}
259
260impl MetaInputs {
261    pub fn new(o: &Options, plan: &order::Plan, store: &store::FeatureStore) -> MetaInputs {
262        MetaInputs {
263            minzoom: plan.minzoom,
264            bbox: store.bbox,
265            bbox1: store.bbox1,
266            bbox2: store.bbox2,
267            attribution: o.attribution.clone(),
268            description: o.description.clone(),
269            tilestats: !o.prevent.get(P_TILE_STATS),
270            attribute_descriptions: o.attribute_descriptions.clone(),
271            commandline: o.commandline.clone(),
272            basezoom: plan.basezoom,
273            droprate: plan.droprate,
274            retain_points_multiplier: o.retain_points_multiplier,
275            layermap: store.layermap(),
276            tile_compression: !o.prevent.get(P_TILE_COMPRESSION),
277            tile_format: o.tile_format,
278            h3: o.h3.then(|| ingest::h3::Setup::new(o).json()),
279        }
280    }
281
282    /// What the metadata has beyond tippecanoe's: `--h3`'s resolutions and
283    /// zooms, as `manby_h3`.
284    pub fn extra(&self) -> Vec<(String, String)> {
285        self.h3
286            .iter()
287            .map(|h| (H3_KEY.to_string(), h.clone()))
288            .collect()
289    }
290
291    /// The tileset's metadata, as tippecanoe makes it.
292    pub fn metadata(&self, name: &str, stats: &tiler::TileStats) -> Metadata {
293        let maxzoom = stats.maxzoom;
294
295        // mbtiles-style bounding box and center
296        let (midx, midy) = (stats.midx, stats.midy);
297        let (mut minlon, mut maxlat) = tile2lonlat(midx as i64, midy as i64, maxzoom);
298        let (mut maxlon, mut minlat) = tile2lonlat(
299            midx.wrapping_add(1) as i64,
300            midy.wrapping_add(1) as i64,
301            maxzoom,
302        );
303
304        let mut midlat = (maxlat + minlat) / 2.0;
305        let mut midlon = (maxlon + minlon) / 2.0;
306
307        let b = self.bbox;
308        (minlon, maxlat) = tile2lonlat(b[0], b[1], 32);
309        (maxlon, minlat) = tile2lonlat(b[2], b[3], 32);
310
311        midlat = midlat.clamp(minlat, maxlat);
312        midlon = midlon.clamp(minlon, maxlon);
313
314        // antimeridian-aware bounding box: whichever of the two is narrower
315        let (b1, b2) = (self.bbox1, self.bbox2);
316        let nb = if b2[2] - b2[0] < b1[2] - b1[0] {
317            b2
318        } else {
319            b1
320        };
321        let (minlon2, maxlat2) = tile2lonlat(nb[0], nb[1], 32);
322        let (maxlon2, minlat2) = tile2lonlat(nb[2], nb[3], 32);
323
324        let mut merged = merge_layermaps(&[self.layermap.clone(), stats.layermap.clone()]);
325        for v in merged.values_mut() {
326            v.minzoom = self.minzoom;
327            v.maxzoom = maxzoom;
328        }
329
330        make_metadata(
331            name,
332            self.minzoom,
333            maxzoom,
334            minlat,
335            minlon,
336            maxlat,
337            maxlon,
338            minlat2,
339            minlon2,
340            maxlat2,
341            maxlon2,
342            midlat,
343            midlon,
344            self.attribution.as_deref(),
345            &merged,
346            true,
347            self.description.as_deref(),
348            self.tilestats,
349            &self.attribute_descriptions,
350            "tippecanoe",
351            &self.commandline,
352            &stats.strategies,
353            self.basezoom,
354            self.droprate,
355            self.retain_points_multiplier,
356        )
357    }
358}