versatiles_container 4.10.0

//! This module provides functionality for reading tile data from a directory structure.
//!
//! The directory path must be **absolute**.
//!
//! Recognized metadata files include `meta.json`, `tiles.json`, `metadata.json` and their compressed variants with `.gz` or `.br` extensions.
//!
//! Tile files must follow the naming pattern:
//! ```text
//! <root>/<z>/<x>/<y>.<format>[.<compression>]
//! ```
//! where `<z>`, `<x>`, and `<y>` are zoom level and tile coordinates, `<format>` is the tile format (e.g., `png`, `pbf`), and `<compression>` is optional (e.g., `br`, `gz`).
//!
//! Examples:
//! | Path               | Description                  |
//! |--------------------|------------------------------|
//! | `/tiles/3/2/1.png` | Uncompressed PNG tile        |
//! | `/tiles/4/2/1.pbf.br` | Brotli compressed PBF tile  |
//! | `/tiles/meta.json`  | Metadata file                |
//!
//! All tiles must share the same **format** and **compression**. If multiple formats or compressions are detected, an error is returned.
//!
//! Bounds, minimum zoom, and maximum zoom are inferred from the discovered tiles and merged with any metadata files found.
//!
//! ## Usage
//! ```no_run
//! use versatiles_container::*;
//! use versatiles_core::*;
//! use std::path::Path;
//! use tokio;
//!
//! #[tokio::main]
//! async fn main() {
//!     let mut reader = DirectoryReader::open(Path::new("/absolute/path/to/tiles")).unwrap();
//!     let tile_data = reader.tile(&TileCoord::new(3, 1, 2).unwrap()).await.unwrap();
//! }
//! ```
//!
//! ## Errors
//! Errors are returned if the directory is not absolute, does not exist, is not a directory, contains no tiles, or if tiles have inconsistent formats or compressions.

use std::{
	collections::HashMap,
	fmt::Debug,
	fs,
	path::{Path, PathBuf},
	sync::Arc,
};

use anyhow::{Result, bail, ensure};
use async_trait::async_trait;
use itertools::Itertools;
#[cfg(feature = "cli")]
use versatiles_core::utils::PrettyPrint;
use versatiles_core::{
	Blob, TileBBox, TileCompression, TileCoord, TileFormat, TileJSON, TilePyramid, TileStream, compression::decompress,
};
use versatiles_derive::context;

#[cfg(feature = "cli")]
use crate::TilesRuntime;
use crate::{SourceType, Tile, TileSource, TileSourceMetadata, Traversal};

/// A reader for tiles stored in a directory structure.
///
/// This struct merges `TileJSON` metadata from recognized files such as `meta.json`, `tiles.json`, or `metadata.json` (and their compressed variants),
/// and infers a tile pyramid from the folder hierarchy to provide tile reading functionality.
///
/// The directory structure is expected as:
/// ```text
/// <root>/<z>/<x>/<y>.<format>[.<compression>]]
/// ```
/// where `<z>`, `<x>`, and `<y>` are tile coordinates, `<format>` is the tile format, and `<compression>` is optional.
pub struct DirectoryReader {
	tilejson: TileJSON,
	dir: PathBuf,
	tile_map: Arc<HashMap<TileCoord, PathBuf>>,
	metadata: TileSourceMetadata,
}

/// The entries of `dir` that this reader can do anything with, paired with their
/// names and sorted by name.
///
/// Two kinds are skipped rather than reported. An entry that cannot be read at
/// all — a permission, or a file removed while the directory is being scanned —
/// says nothing about the tiles beside it. And a name that is not valid UTF-8
/// cannot be a zoom level, an x column or a `y.ext` tile in any case; a
/// directory is free to contain one, so reading it must not end the process.
fn usable_entries(dir: &Path) -> Result<impl Iterator<Item = (String, fs::DirEntry)>> {
	Ok(fs::read_dir(dir)?
		.filter_map(Result::ok)
		.filter_map(|entry| Some((entry.file_name().into_string().ok()?, entry)))
		.sorted_unstable_by(|a, b| a.0.cmp(&b.0)))
}

impl DirectoryReader {
	/// Opens a directory and initializes a `DirectoryReader`.
	///
	/// The provided path must be **absolute**.
	///
	/// This function scans the directory structure for tiles and metadata files.
	/// It requires that all tiles have a uniform tile format and compression type, otherwise it returns an error.
	/// Metadata files (`meta.json`, `tiles.json`, `metadata.json` and their `.gz`/`.br` variants) are merged into the `TileJSON`.
	/// Bounds, minzoom, and maxzoom are inferred from the directory's tile pyramid and merged with metadata.
	///
	/// The returned `DirectoryReader` contains `TileSourceMetadata` which specify the tile format, compression, and tile pyramid.
	///
	/// # Arguments
	///
	/// * `dir` - An absolute path to the directory containing the tiles.
	///
	/// # Errors
	///
	/// Returns an error if the directory does not exist, is not a directory, contains no tiles, or contains inconsistent tile formats or compressions.
	#[context("opening tiles directory {:?}", dir)]
	pub fn open(dir: &Path) -> Result<DirectoryReader>
	where
		Self: Sized,
	{
		log::trace!("read {dir:?}");

		ensure!(dir.is_absolute(), "path {dir:?} must be absolute");
		ensure!(dir.exists(), "path {dir:?} does not exist");
		ensure!(dir.is_dir(), "path {dir:?} is not a directory");

		let mut tilejson = TileJSON::default();
		let mut tile_map = HashMap::new();
		let mut container_form: Option<TileFormat> = None;
		let mut container_comp: Option<TileCompression> = None;
		for (name1, entry1) in usable_entries(dir)? {
			// z level
			if let Ok(level) = name1.parse::<u8>() {
				for (name2, entry2) in usable_entries(&entry1.path())? {
					// x level
					let Ok(x) = name2.parse::<u32>() else {
						continue;
					};

					for (mut filename, entry3) in usable_entries(&entry2.path())? {
						// y level
						let file_comp = TileCompression::from_filename(&mut filename);
						let this_form = TileFormat::from_filename(&mut filename);

						if this_form.is_none() {
							continue;
						}
						let file_form = this_form.expect("checked is_none above");

						let numeric3 = filename.parse::<u32>();
						if numeric3.is_err() {
							continue;
						}
						let y = numeric3?;

						if let Some(form) = container_form {
							if form != file_form {
								let mut r = [form, file_form];
								r.sort();
								bail!("found multiple tile formats: {r:?}");
							}
						} else {
							container_form = Some(file_form);
						}

						if let Some(comp) = container_comp {
							if comp != file_comp {
								let mut r = [comp, file_comp];
								r.sort();
								bail!("found multiple tile compressions: {r:?}");
							}
						} else {
							container_comp = Some(file_comp);
						}

						let coord = TileCoord::new(level, x, y)?;
						tile_map.insert(coord, entry3.path());
					}
				}
			} else {
				match name1.as_str() {
					"meta.json" | "tiles.json" | "metadata.json" => {
						tilejson.merge(&TileJSON::try_from_blob_or_default(&Self::read(&entry1.path())?))?;
					}
					"meta.json.gz" | "tiles.json.gz" | "metadata.json.gz" => {
						tilejson.merge(&TileJSON::try_from_blob_or_default(&decompress(
							Self::read(&entry1.path())?,
							&TileCompression::Gzip,
						)?))?;
					}
					"meta.json.br" | "tiles.json.br" | "metadata.json.br" => {
						tilejson.merge(&TileJSON::try_from_blob_or_default(&decompress(
							Self::read(&entry1.path())?,
							&TileCompression::Brotli,
						)?))?;
					}
					&_ => {}
				}
			}
		}

		if tile_map.is_empty() {
			bail!("no tiles found");
		}

		let tile_pyramid = TilePyramid::from_tile_coords(tile_map.keys().copied());

		let tile_format = container_form.context("tile format must be specified")?;
		let tile_compression = container_comp.context("tile compression must be specified")?;

		tilejson.update_from_pyramid(&tile_pyramid);

		Ok(DirectoryReader {
			tilejson,
			dir: dir.to_path_buf(),
			tile_map: Arc::new(tile_map),
			metadata: TileSourceMetadata::new(tile_format, tile_compression, Traversal::ANY, Some(tile_pyramid)),
		})
	}

	/// Reads a file into a `Blob`.
	#[context("reading file '{}'", path.display())]
	fn read(path: &Path) -> Result<Blob> {
		Ok(Blob::from(fs::read(path)?))
	}

	/// Internal helper to look up and read a tile by coordinate.
	fn lookup_tile(
		coord: &TileCoord,
		tile_map: &HashMap<TileCoord, PathBuf>,
		tile_compression: TileCompression,
		tile_format: TileFormat,
	) -> Result<Option<Tile>> {
		if let Some(path) = tile_map.get(coord) {
			Self::read(path).map(|blob| Some(Tile::from_blob(blob, tile_compression, tile_format)))
		} else {
			Ok(None)
		}
	}
}

/// Implements the `TileSource` for `DirectoryReader`.
///
/// Provides the container name ("directory"), access to tile reading parameters,
/// ability to override the tile compression, access to `TileJSON` metadata,
/// and asynchronous fetching of tile data by coordinate.
#[async_trait]
impl TileSource for DirectoryReader {
	fn source_type(&self) -> Arc<SourceType> {
		SourceType::new_container("directory", self.dir.to_str().expect("directory path is utf-8"))
	}

	fn metadata(&self) -> &TileSourceMetadata {
		&self.metadata
	}

	fn tilejson(&self) -> &TileJSON {
		&self.tilejson
	}

	async fn tile_pyramid(&self) -> Result<Arc<TilePyramid>> {
		self
			.metadata
			.get_or_compute_tile_pyramid(|| Ok(TilePyramid::from_tile_coords(self.tile_map.keys().copied())))
	}

	#[context("fetching tile {:?} from directory '{}'", coord, self.dir.display())]
	async fn tile(&self, coord: &TileCoord) -> Result<Option<Tile>> {
		log::trace!("tile {coord:?}");
		Self::lookup_tile(
			coord,
			&self.tile_map,
			*self.metadata.tile_compression(),
			*self.metadata.tile_format(),
		)
	}

	async fn tile_stream(&self, bbox: TileBBox) -> Result<TileStream<'static, Tile>> {
		log::trace!("directory::tile_stream {bbox:?}");
		let tile_map = Arc::clone(&self.tile_map);
		let tile_compression = *self.metadata.tile_compression();
		let tile_format = *self.metadata.tile_format();

		Ok(TileStream::from_bbox_parallel(bbox, move |coord| {
			DirectoryReader::lookup_tile(&coord, &tile_map, tile_compression, tile_format).ok()?
		}))
	}

	async fn tile_coord_stream(&self, bbox: TileBBox) -> Result<TileStream<'static, ()>> {
		let tile_map = Arc::clone(&self.tile_map);
		Ok(TileStream::from_bbox_parallel(bbox, move |coord| {
			tile_map.get(&coord).map(|_| ())
		}))
	}

	async fn tile_size_stream(&self, bbox: TileBBox) -> Result<TileStream<'static, u32>> {
		let tile_map = Arc::clone(&self.tile_map);
		Ok(TileStream::from_bbox_parallel(bbox, move |coord| {
			let path = tile_map.get(&coord)?;
			let size = std::fs::metadata(path).ok()?.len();
			u32::try_from(size).ok()
		}))
	}

	#[cfg(feature = "cli")]
	async fn probe_container(&self, print: &mut PrettyPrint, _runtime: &TilesRuntime) -> Result<()> {
		print.add_key_value("directory", &self.dir.display().to_string()).await;
		print.add_key_value("tile count", &self.tile_map.len()).await;
		Ok(())
	}
}

impl Debug for DirectoryReader {
	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
		f.debug_struct("DirectoryReader")
			.field("source_type", &self.source_type())
			.field("parameters", &self.metadata())
			.finish()
	}
}

#[cfg(test)]
mod tests {
	use std::fs::{self};

	use assert_fs::{
		TempDir,
		fixture::{FileWriteStr, PathChild},
	};
	use versatiles_core::{assert_wildcard, compression::compress};

	use super::*;

	#[tokio::test]
	async fn tile_reader_new() -> Result<()> {
		let dir = TempDir::new()?;
		dir.child(".DS_Store").write_str("")?;
		dir.child("3/2/1.png").write_str("test tile data")?;
		dir.child("meta.json").write_str(r#"{"type":"dummy"}"#)?;

		let reader = DirectoryReader::open(&dir)?;

		assert_eq!(
			reader.tilejson().stringify(),
			"{\"bounds\":[-90,66.51326,-45,79.171335],\"maxzoom\":3,\"minzoom\":3,\"tilejson\":\"3.0.0\",\"type\":\"dummy\"}"
		);

		let mut tile_data = reader.tile(&TileCoord::new(3, 2, 1)?).await?.unwrap();
		assert_eq!(
			tile_data.as_blob(reader.metadata().tile_compression())?,
			&Blob::from("test tile data")
		);

		assert!(reader.tile(&TileCoord::new(2, 2, 1)?).await?.is_none());

		Ok(())
	}

	/// A tile directory can contain a file whose name is not UTF-8 — copied from
	/// another system, or simply written that way. Reading the directory used to
	/// panic on it at `into_string().expect("filesystem name is utf-8")`; the
	/// tiles beside it are still perfectly readable.
	#[cfg(unix)]
	#[tokio::test]
	async fn non_utf8_filenames_are_skipped_not_fatal() -> Result<()> {
		use std::{ffi::OsStr, os::unix::ffi::OsStrExt};

		let dir = TempDir::new()?;
		dir.child("3/2/1.png").write_str("test tile data")?;

		// A neighbour of the tile, and a whole zoom-level directory, both named in
		// bytes that are not UTF-8. Not every filesystem accepts such a name —
		// APFS rejects it with EILSEQ — so where it cannot be created there is
		// nothing to test and the reader is checked on the tile alone.
		let odd_file = dir.path().join("3/2").join(OsStr::from_bytes(b"\xff\xfe.png"));
		let odd_dir = dir.path().join(OsStr::from_bytes(b"\xff"));
		let odd_names_supported = fs::write(&odd_file, "junk").is_ok() && fs::create_dir(&odd_dir).is_ok();
		if !odd_names_supported {
			eprintln!("note: this filesystem rejects non-UTF-8 names; the interesting half of the test is skipped");
		}

		let reader = DirectoryReader::open(&dir)?;
		let mut tile = reader.tile(&TileCoord::new(3, 2, 1)?).await?.unwrap();
		assert_eq!(
			tile.as_blob(reader.metadata().tile_compression())?,
			&Blob::from("test tile data")
		);

		Ok(())
	}

	#[tokio::test]
	async fn open_path_with_nonexistent_directory() -> Result<()> {
		let dir = TempDir::new()?;

		let msg = DirectoryReader::open(&dir.join("dont_exist"))
			.unwrap_err()
			.chain()
			.last()
			.unwrap()
			.to_string();
		assert_eq!(&msg[msg.len() - 16..], "\" does not exist");

		Ok(())
	}

	#[tokio::test]
	async fn open_path_with_unsupported_file_format() -> Result<()> {
		let dir = TempDir::new()?;
		dir.child("3/2/1.unknown").write_str("unsupported format")?;

		assert_eq!(
			DirectoryReader::open(dir.path())
				.unwrap_err()
				.chain()
				.last()
				.unwrap()
				.to_string(),
			"no tiles found",
		);

		Ok(())
	}

	#[tokio::test]
	async fn read_compressed_meta_files() -> Result<()> {
		let dir = TempDir::new().unwrap();
		fs::write(
			dir.path().join("meta.json.gz"),
			compress(Blob::from(r#"{"type":"dummy data"}"#), &TileCompression::Gzip)
				.unwrap()
				.as_slice(),
		)
		.unwrap();
		fs::create_dir_all(dir.path().join("2/1")).unwrap();
		fs::write(dir.path().join("2/1/0.png"), "tile at 2/1/0").unwrap();

		let reader = DirectoryReader::open(&dir).unwrap();
		assert_eq!(
			reader.tilejson().stringify(),
			"{\"bounds\":[-90,66.51326,0,85.051129],\"maxzoom\":2,\"minzoom\":2,\"tilejson\":\"3.0.0\",\"type\":\"dummy data\"}"
		);

		Ok(())
	}

	#[tokio::test]
	async fn complex_directory_structure() -> Result<()> {
		let dir = TempDir::new().unwrap();
		fs::create_dir_all(dir.path().join("3/2")).unwrap();
		fs::write(dir.path().join("3/2/1.png"), "tile at 3/2/1").unwrap();
		fs::write(dir.path().join("meta.json"), r#"{"type":"dummy data"}"#).unwrap();

		let reader = DirectoryReader::open(&dir).unwrap();
		let coord = TileCoord::new(3, 2, 1).unwrap();
		let blob = reader
			.tile(&coord)
			.await
			.unwrap()
			.unwrap()
			.into_blob(reader.metadata().tile_compression())?;

		assert_eq!(blob, Blob::from("tile at 3/2/1"));

		Ok(())
	}

	#[tokio::test]
	async fn incorrect_format_and_compression_handling() -> Result<()> {
		let dir = TempDir::new().unwrap();
		fs::create_dir_all(dir.path().join("3/2")).unwrap();
		fs::write(dir.path().join("3/2/1.txt"), "wrong format").unwrap();

		assert_eq!(
			&DirectoryReader::open(&dir)
				.unwrap_err()
				.chain()
				.last()
				.unwrap()
				.to_string(),
			"no tiles found",
			"Should error on incorrect tile format"
		);

		Ok(())
	}

	#[tokio::test]
	async fn error_different_tile_formats() -> Result<()> {
		let dir = TempDir::new()?;
		dir.child("3/2/1.png").write_str("test tile data")?;
		dir.child("4/2/1.jpg").write_str("test tile data")?;

		assert_eq!(
			DirectoryReader::open(&dir)
				.unwrap_err()
				.chain()
				.last()
				.unwrap()
				.to_string(),
			"found multiple tile formats: [JPG, PNG]"
		);

		Ok(())
	}

	#[tokio::test]
	async fn error_different_tile_compressions() -> Result<()> {
		let dir = TempDir::new()?;
		dir.child("3/2/1.pbf").write_str("test tile data")?;
		dir.child("4/2/1.pbf.br").write_str("test tile data")?;

		assert_eq!(
			DirectoryReader::open(&dir)
				.unwrap_err()
				.chain()
				.last()
				.unwrap()
				.to_string(),
			"found multiple tile compressions: [Uncompressed, Brotli]"
		);

		Ok(())
	}

	#[tokio::test]
	async fn test_minor_functions() -> Result<()> {
		let dir = assert_fs::TempDir::new()?;
		dir.child("meta.json").write_str("{\"key\": \"value\"}")?;
		dir.child("3/2/1.png.br").write_str("tile data")?;

		let reader = DirectoryReader::open(dir.path())?;

		assert_eq!(
			reader.source_type().to_string(),
			format!("container 'directory' ('{}')", dir.path().to_str().unwrap())
		);

		assert_wildcard!(
			format!("{reader:?}"),
			"DirectoryReader { source_type: Container { name: \"directory\", uri: \"*\" }, parameters: TileSourceMetadata { tile_compression: Brotli, tile_format: PNG, traversal: Traversal(AnyOrder,full), tile_pyramid: RwLock { data: * poisoned: false, .. } } }"
		);

		assert_eq!(
			reader.tilejson().stringify(),
			"{\"bounds\":[-90,66.51326,-45,79.171335],\"key\":\"value\",\"maxzoom\":3,\"minzoom\":3,\"tilejson\":\"3.0.0\"}"
		);

		Ok(())
	}

	#[tokio::test]
	async fn tile_size_stream() -> Result<()> {
		let dir = TempDir::new()?;
		dir.child("2/0/0.png").write_str("short")?;
		dir.child("2/0/1.png").write_str("a longer tile")?;
		dir.child("2/1/0.png").write_str("medium tile")?;

		let reader = DirectoryReader::open(&dir)?;
		let bbox = TileBBox::from_min_and_max(2, 0, 0, 1, 1)?;

		let mut sizes: Vec<(TileCoord, u32)> = reader.tile_size_stream(bbox).await?.to_vec().await;
		sizes.sort_by_key(|(c, _)| (c.y, c.x));

		assert_eq!(sizes.len(), 3);
		assert_eq!(sizes[0].1, 5); // "short"
		assert_eq!(sizes[1].1, 11); // "medium tile"
		assert_eq!(sizes[2].1, 13); // "a longer tile"

		Ok(())
	}

	#[cfg(feature = "cli")]
	#[tokio::test]
	async fn probe() -> Result<()> {
		use versatiles_core::utils::PrettyPrint;

		let dir = TempDir::new()?;
		dir.child("2/0/0.png").write_str("tile data")?;
		dir.child("2/1/0.png").write_str("tile data")?;

		let reader = DirectoryReader::open(&dir)?;
		let runtime = crate::TilesRuntime::default();

		let mut printer = PrettyPrint::new();
		reader
			.probe_container(&mut printer.category("container").await, &runtime)
			.await?;
		let output = printer.stringify().await;
		assert!(output.contains("tile count: 2"), "unexpected output: {output}");
		assert!(output.contains("directory:"), "unexpected output: {output}");

		Ok(())
	}

	#[tokio::test]
	async fn tile_stream_matches_individual_reads() -> Result<()> {
		let dir = TempDir::new()?;
		// Create a small grid of tiles
		dir.child("2/0/0.png").write_str("tile_0_0")?;
		dir.child("2/0/1.png").write_str("tile_0_1")?;
		dir.child("2/1/0.png").write_str("tile_1_0")?;
		dir.child("2/1/1.png").write_str("tile_1_1")?;

		let reader = DirectoryReader::open(&dir)?;
		let bbox = TileBBox::from_min_and_max(2, 0, 0, 1, 1)?;

		// Get all tiles via stream
		let stream = reader.tile_stream(bbox).await?;
		let stream_tiles: Vec<_> = stream.to_vec().await;
		assert_eq!(stream_tiles.len(), 4);

		// Verify each streamed tile matches individual read
		for (coord, mut tile) in stream_tiles {
			let stream_blob = tile.as_blob(reader.metadata().tile_compression())?;
			let single_blob = reader
				.tile(&coord)
				.await?
				.expect("tile should exist")
				.into_blob(reader.metadata().tile_compression())?;
			assert_eq!(
				stream_blob.as_slice(),
				single_blob.as_slice(),
				"blob mismatch at {coord:?}"
			);
		}

		Ok(())
	}
}