dezoomify_rs/
lib.rs

1#![allow(clippy::upper_case_acronyms)]
2
3use std::{fmt, fs, io};
4use std::env::current_dir;
5use std::error::Error;
6use std::io::BufRead;
7use std::path::PathBuf;
8
9use futures::stream::StreamExt;
10use indicatif::{ProgressBar, ProgressStyle};
11use itertools::Itertools;
12use log::{debug, info};
13use reqwest::Client;
14
15pub use arguments::Arguments;
16use dezoomer::{TileFetchResult, ZoomLevel, ZoomLevelIter};
17use dezoomer::{Dezoomer, DezoomerError, DezoomerInput, ZoomLevels};
18use dezoomer::TileReference;
19pub use errors::ZoomError;
20use network::{client, fetch_uri};
21use output_file::get_outname;
22use tile::Tile;
23pub use vec2d::Vec2d;
24
25use crate::dezoomer::PageContents;
26use crate::encoder::tile_buffer::TileBuffer;
27use crate::network::TileDownloader;
28use crate::output_file::reserve_output_file;
29
30mod arguments;
31mod encoder;
32pub mod dezoomer;
33pub mod tile;
34mod vec2d;
35mod errors;
36mod output_file;
37mod network;
38
39pub mod auto;
40pub mod custom_yaml;
41pub mod dzi;
42pub mod generic;
43pub mod google_arts_and_culture;
44pub mod iiif;
45pub mod pff;
46pub mod zoomify;
47pub mod krpano;
48pub mod nypl;
49pub mod iipimage;
50mod json_utils;
51
52fn stdin_line() -> Result<String, ZoomError> {
53    let stdin = std::io::stdin();
54    let mut lines = stdin.lock().lines();
55    let first_line = lines.next().ok_or_else(|| {
56        let err_msg = "Encountered end of standard input while reading a line";
57        io::Error::new(io::ErrorKind::UnexpectedEof, err_msg)
58    })?;
59    Ok(first_line?)
60}
61
62async fn list_tiles(
63    dezoomer: &mut dyn Dezoomer,
64    http: &Client,
65    uri: &str,
66) -> Result<ZoomLevels, ZoomError> {
67    let mut i = DezoomerInput {
68        uri: String::from(uri),
69        contents: PageContents::Unknown,
70    };
71    loop {
72        match dezoomer.zoom_levels(&i) {
73            Ok(levels) => return Ok(levels),
74            Err(DezoomerError::NeedsData { uri }) => {
75                let contents = fetch_uri(&uri, http).await.into();
76                debug!("Response for metadata file '{}': {:?}", uri, &contents);
77                i.uri = uri;
78                i.contents = contents;
79            }
80            Err(e) => return Err(e.into()),
81        }
82    }
83}
84
85/// An interactive level picker
86fn level_picker(mut levels: Vec<ZoomLevel>) -> Result<ZoomLevel, ZoomError> {
87    println!("Found the following zoom levels:");
88    for (i, level) in levels.iter().enumerate() {
89        println!("{: >2}. {}", i, level.name());
90    }
91    loop {
92        println!("Which level do you want to download? ");
93        let line = stdin_line()?;
94        if let Ok(idx) = line.parse::<usize>() {
95            if levels.get(idx).is_some() {
96                return Ok(levels.swap_remove(idx));
97            }
98        }
99        println!("'{}' is not a valid level number", line);
100    }
101}
102
103fn choose_level(mut levels: Vec<ZoomLevel>, args: &Arguments) -> Result<ZoomLevel, ZoomError> {
104    match levels.len() {
105        0 => Err(ZoomError::NoLevels),
106        1 => Ok(levels.swap_remove(0)),
107        _ => {
108            let pos = args
109                .best_size(levels.iter().filter_map(|l| l.size_hint()))
110                .and_then(|best_size| {
111                    levels
112                        .iter()
113                        .find_position(|&l| l.size_hint() == Some(best_size))
114                });
115            if let Some((i, _)) = pos {
116                Ok(levels.swap_remove(i))
117            } else {
118                level_picker(levels)
119            }
120        }
121    }
122}
123
124fn progress_bar(n: usize) -> ProgressBar {
125    let progress = ProgressBar::new(n as u64);
126    progress.set_style(
127        ProgressStyle::default_bar()
128            .template("[ETA:{eta}] {bar:40.cyan/blue} {pos:>4}/{len:4} {msg}")
129            .expect("Invalid indicatif progress bar template")
130            .progress_chars("##-"),
131    );
132    progress
133}
134
135async fn find_zoomlevel(args: &Arguments) -> Result<ZoomLevel, ZoomError> {
136    let mut dezoomer = args.find_dezoomer()?;
137    let uri = args.choose_input_uri()?;
138    let http_client = client(args.headers(), args, Some(&uri))?;
139    info!("Trying to locate a zoomable image...");
140    let zoom_levels: Vec<ZoomLevel> = list_tiles(dezoomer.as_mut(), &http_client, &uri).await?;
141    info!("Found {} zoom levels", zoom_levels.len());
142    choose_level(zoom_levels, args)
143}
144
145pub async fn dezoomify(args: &Arguments) -> Result<PathBuf, ZoomError> {
146    let zoom_level = find_zoomlevel(args).await?;
147    let base_dir = current_dir()?;
148    let outname = get_outname(&args.outfile, &zoom_level.title(), &base_dir,zoom_level.size_hint());
149    let save_as = fs::canonicalize(outname.as_path()).unwrap_or_else(|_e| outname.clone());
150    reserve_output_file(&save_as)?;
151    let tile_buffer: TileBuffer = TileBuffer::new(save_as.clone(), args.compression).await?;
152    info!("Dezooming {}", zoom_level.name());
153    dezoomify_level(args, zoom_level, tile_buffer).await?;
154    Ok(save_as)
155}
156
157pub async fn dezoomify_level(
158    args: &Arguments,
159    mut zoom_level: ZoomLevel,
160    tile_buffer: TileBuffer,
161) -> Result<(), ZoomError> {
162    let level_headers = zoom_level.http_headers();
163    let downloader = TileDownloader {
164        http_client: client(level_headers.iter().chain(args.headers()), args, None)?,
165        post_process_fn: zoom_level.post_process_fn(),
166        retries: args.retries,
167        retry_delay: args.retry_delay,
168        tile_storage_folder: args.tile_storage_folder.clone(),
169    };
170
171    info!("Creating canvas");
172    let mut canvas = tile_buffer;
173
174    let progress = progress_bar(10);
175    let mut total_tiles = 0u64;
176    let mut successful_tiles = 0u64;
177
178
179    progress.set_message("Computing the URLs of the image tiles...");
180
181    let mut zoom_level_iter = ZoomLevelIter::new(&mut zoom_level);
182    let mut last_count = 0;
183    let mut last_successes = 0;
184    while let Some(tile_refs) = zoom_level_iter.next_tile_references() {
185        last_count = tile_refs.len() as u64;
186        total_tiles += last_count;
187        progress.set_length(total_tiles);
188
189        progress.set_message("Requesting the tiles...");
190
191        let mut stream = futures::stream::iter(tile_refs)
192            .map(|tile_ref: TileReference| downloader.download_tile(tile_ref))
193            .buffer_unordered(args.parallelism);
194
195        last_successes = 0;
196        let mut tile_size = None;
197
198        if let Some(size) = zoom_level_iter.size_hint() {
199            canvas.set_size(size).await?;
200        }
201
202        while let Some(tile_result) = stream.next().await {
203            debug!("Received tile result: {:?}", tile_result);
204            progress.inc(1);
205            let tile = match tile_result {
206                Ok(tile) => {
207                    progress.set_message(format!("Loaded tile at {}", tile.position()));
208                    tile_size.replace(tile.size());
209                    last_successes += 1;
210                    Some(tile)
211                }
212                Err(err) => {
213                    // If a tile download fails, we replace it with an empty tile
214                    progress.set_message(err.to_string());
215                    let position = err.tile_reference.position;
216                    tile_size.and_then(|tile_size| {
217                        zoom_level_iter.size_hint().map(|canvas_size| {
218                            let size = max_size_in_rect(position, tile_size, canvas_size);
219                            Tile::empty(position, size)
220                        })
221                    })
222                }
223            };
224            if let Some(tile) = tile { canvas.add_tile(tile).await; }
225        }
226        successful_tiles += last_successes;
227        zoom_level_iter.set_fetch_result(TileFetchResult {
228            count: last_count,
229            successes: last_successes,
230            tile_size,
231        });
232    }
233
234    if successful_tiles == 0 { return Err(ZoomError::NoTile); }
235
236    progress.set_message("Downloaded all tiles. Finalizing the image file.");
237    canvas.finalize().await?;
238
239    progress.finish_with_message("Finished tile download");
240
241    if last_successes < last_count {
242        let destination = canvas.destination().to_string_lossy().to_string();
243        Err(ZoomError::PartialDownload { successful_tiles, total_tiles, destination })
244    } else {
245        Ok(())
246    }
247}
248
249#[derive(Debug)]
250pub struct TileDownloadError {
251    tile_reference: TileReference,
252    cause: ZoomError,
253}
254
255impl fmt::Display for TileDownloadError {
256    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
257        write!(f, "Unable to download tile '{}'. Cause: {}", self.tile_reference.url, self.cause)
258    }
259}
260
261impl Error for TileDownloadError {}
262
263/// Returns the maximal size a tile can have in order to fit in a canvas of the given size
264pub fn max_size_in_rect(position: Vec2d, tile_size: Vec2d, canvas_size: Vec2d) -> Vec2d {
265    (position + tile_size).min(canvas_size) - position
266}
dezoomify_rs/lib.rs

dezoomify_rs/
lib.rs
