1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
//! OneIO creates readers and writers for files that just works, local or remote, compressed or raw.
//!
//! ## Usage and Feature Flags
//!
//! Enable all compression algorithms and handle remote files (default)
//! ```toml
//! oneio = "0.16"
//! ```
//!
//! Select from supported feature flags
//! ```toml
//! oneio = {version = "0.16", features = ["remote", "gz"]}
//! ```
//!
//! Supported feature flags:
//! - `all` (default): all flags (`["gz", "bz", "lz", "remote", "json"]`)
//! - `remote`: allow reading from remote files, including http(s) and ftp
//! - `remote-rustls`: use `rustls` instead of `native-tls` for remote files via https
//! - `rustls`: use `rustls` instead of `native-tls` for remote files via https or S3, if either are enabled
//! - `gz`: support `gzip` files
//! - `bz`: support `bzip2` files
//! - `lz`: support `lz4` files
//! - `json`: allow reading JSON content into structs directly
//!
//! ## Reader
//!
//! The returned reader implements BufRead, and handles decompression from the following types:
//! - `gzip`: files ending with `gz` or `gzip`
//! - `bzip2`: files ending with `bz` or `bz2`
//! - `lz4`: files ending with `lz4` or `lz`
//!
//! It also handles reading from remote or local files transparently.
//!
//! ### Examples
//!
//! Read all into string:
//! ```rust
//! const TEST_TEXT: &str = "OneIO test file.
//! This is a test.";
//!
//! let mut reader = oneio::get_reader("https://spaces.bgpkit.org/oneio/test_data.txt.gz").unwrap();
//!
//! let mut text = "".to_string();
//! reader.read_to_string(&mut text).unwrap();
//! assert_eq!(text.as_str(), TEST_TEXT);
//! ```
//!
//! Read into lines:
//! ```rust
//! use std::io::BufRead;
//! use std::io::BufReader;
//! const TEST_TEXT: &str = "OneIO test file.
//! This is a test.";
//!
//! let lines = oneio::read_lines("https://spaces.bgpkit.org/oneio/test_data.txt.gz").unwrap()
//! .map(|line| line.unwrap()).collect::<Vec<String>>();
//!
//! assert_eq!(lines.len(), 2);
//! assert_eq!(lines[0].as_str(), "OneIO test file.");
//! assert_eq!(lines[1].as_str(), "This is a test.");
//! ```
//!
//! Cache data first, then read
//! ```rust
//! use std::io::BufRead;
//! use std::io::BufReader;
//! const TEST_TEXT: &str = "OneIO test file.
//! This is a test.";
//!
//! let reader = BufReader::new(oneio::get_cache_reader("https://spaces.bgpkit.org/oneio/test_data.txt.gz", "/tmp/oneio/cache/", None, false).unwrap());
//! let lines = reader.lines().into_iter().map(|line| line.unwrap()).collect::<Vec<String>>();
//!
//! assert_eq!(lines.len(), 2);
//! assert_eq!(lines[0].as_str(), "OneIO test file.");
//! assert_eq!(lines[1].as_str(), "This is a test.");
//!
//! let mut reader = oneio::get_reader("/tmp/oneio/cache/test_data.txt.gz").unwrap();
//! let mut text = "".to_string();
//! reader.read_to_string(&mut text).unwrap();
//! assert_eq!(text.as_str(), TEST_TEXT);
//! ```
//!
//! Read remote content with custom headers
//! ```no_run
//! use std::collections::HashMap;
//! let mut reader = oneio::get_remote_reader(
//! "https://SOME_REMOTE_RESOURCE_PROTECTED_BY_ACCESS_TOKEN",
//! HashMap::from([("X-Custom-Auth-Key".to_string(), "TOKEN".to_string())])
//! ).unwrap();
//! let mut text = "".to_string();
//! reader.read_to_string(&mut text).unwrap();
//! println!("{}", text);
//! ```
//!
//! ## Writer
//!
//! [get_writer] returns a generic writer that implements [std::io::Write],
//! and handles decompression from the following types:
//! - `gzip`: files ending with `gz` or `gzip`
//! - `bzip2`: files ending with `bz` or `bz2`
//!
//! **Note: lz4 writer is not currently supported.**
//!
//! ### Example
//!
//! ```rust
//! let to_read_file = "https://spaces.bgpkit.org/oneio/test_data.txt.gz";
//! let to_write_file = "/tmp/test_write.txt.bz2";
//!
//! // read text from remote gzip file
//! let mut text = "".to_string();
//! oneio::get_reader(to_read_file).unwrap().read_to_string(&mut text).unwrap();
//!
//! // write the same text to a local bz2 file
//! let mut writer = oneio::get_writer(to_write_file).unwrap();
//! writer.write_all(text.as_ref()).unwrap();
//! drop(writer);
//!
//! // read from the newly-generated bz2 file
//! let mut new_text = "".to_string();
//! oneio::get_reader(to_write_file).unwrap().read_to_string(&mut new_text).unwrap();
//!
//! // compare the decompressed content of the remote and local files
//! assert_eq!(text.as_str(), new_text.as_str());
//! std::fs::remove_file(to_write_file).unwrap();
//! ```
#![doc(
html_logo_url = "https://raw.githubusercontent.com/bgpkit/assets/main/logos/icon-transparent.png",
html_favicon_url = "https://raw.githubusercontent.com/bgpkit/assets/main/logos/favicon.ico"
)]
mod error;
mod oneio;
pub use error::OneIoError;
#[cfg(feature = "digest")]
pub use crate::oneio::digest::*;
#[cfg(feature = "remote")]
pub use crate::oneio::remote::*;
#[cfg(feature = "s3")]
pub use crate::oneio::s3::*;
pub use crate::oneio::*;