use std::collections::BTreeMap;
use std::fmt;
use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicBool;
use serde::{Deserialize, Serialize};
use crate::error::{AuthCause, Error};
use crate::params::Router;
use crate::Result;
use super::store;
pub const ROUTER_PATH: &str = ".spider/router.json";
const ROUTER_FILE: &str = "router.json";
pub const MAX_ROUTER_FILE_BYTES: usize = 64 * 1024;
pub const MODES: &[&str] = &["fallback", "first", "off"];
pub const FUNDINGS: &[&str] = &["any", "own"];
const REDACTED: &str = "<redacted>";
static WARNED_ABOUT_PERMISSIONS: AtomicBool = AtomicBool::new(false);
#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct StoredRouter {
#[serde(default)]
pub router: Router,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_options: Option<BTreeMap<String, serde_json::Value>>,
}
impl fmt::Debug for StoredRouter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let credentials = self.router.credentials.as_ref().map(|held| {
held.keys()
.map(|name| (name.as_str(), REDACTED))
.collect::<BTreeMap<_, _>>()
});
let options = self
.provider_options
.as_ref()
.map(|held| held.keys().map(String::as_str).collect::<Vec<_>>());
f.debug_struct("StoredRouter")
.field("mode", &self.router.mode)
.field("provider", &self.router.provider)
.field("token", &self.router.token.as_ref().map(|_| REDACTED))
.field("credentials", &credentials)
.field("funding", &self.router.funding)
.field("provider_options", &options)
.finish()
}
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[non_exhaustive]
pub enum RouterConfigError {
#[error("router.mode has to be fallback, first or off")]
Mode,
#[error("router.funding has to be any or own")]
Funding,
#[error("router.mode first needs router.provider")]
FirstNeedsProvider,
#[error(
"router.mode first needs router.token, or a router.credentials entry for the provider"
)]
FirstNeedsKey,
#[error("router.provider is empty")]
EmptyProvider,
#[error("router.token is empty")]
EmptyToken,
#[error("router.credentials.{0} is empty")]
EmptyCredential(String),
}
impl From<RouterConfigError> for Error {
fn from(error: RouterConfigError) -> Error {
Error::Config(error.to_string())
}
}
impl StoredRouter {
pub fn validate(&self) -> std::result::Result<(), RouterConfigError> {
let router = &self.router;
if let Some(mode) = &router.mode {
if !MODES.contains(&mode.as_str()) {
return Err(RouterConfigError::Mode);
}
}
if let Some(funding) = &router.funding {
if !FUNDINGS.contains(&funding.as_str()) {
return Err(RouterConfigError::Funding);
}
}
if router.provider.as_deref().is_some_and(blank) {
return Err(RouterConfigError::EmptyProvider);
}
if router.token.as_deref().is_some_and(blank) {
return Err(RouterConfigError::EmptyToken);
}
for (name, value) in router.credentials.iter().flatten() {
if blank(value) {
return Err(RouterConfigError::EmptyCredential(name.clone()));
}
}
if router.mode.as_deref() == Some("first") {
let Some(provider) = router.provider.as_deref() else {
return Err(RouterConfigError::FirstNeedsProvider);
};
let credential = router.credentials.iter().flatten().any(|(name, _)| {
name == provider
|| name
.strip_prefix(provider)
.is_some_and(|rest| rest.starts_with('_'))
});
if router.token.is_none() && !credential {
return Err(RouterConfigError::FirstNeedsKey);
}
}
Ok(())
}
}
fn blank(value: &str) -> bool {
value.trim().is_empty()
}
pub fn path() -> Option<PathBuf> {
store::spider_dir().map(|dir| dir.join(ROUTER_FILE))
}
pub fn load() -> Result<Option<StoredRouter>> {
match path() {
Some(path) => load_from(&path),
None => Ok(None),
}
}
pub fn load_from(path: &Path) -> Result<Option<StoredRouter>> {
use std::io::Read;
let file = match std::fs::File::open(path) {
Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(local(format!("could not open ~/{ROUTER_PATH}: {e}"))),
};
let mut bytes = Vec::new();
file.take(MAX_ROUTER_FILE_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|e| local(format!("could not read ~/{ROUTER_PATH}: {e}")))?;
if bytes.len() > MAX_ROUTER_FILE_BYTES {
return Err(Error::Config(format!(
"~/{ROUTER_PATH} is larger than {MAX_ROUTER_FILE_BYTES} bytes, so it was not read"
)));
}
if let Some(mode) = store::wide_permissions(path) {
let fix = "store the router again";
store::warn_wide_permissions(path, mode, &WARNED_ABOUT_PERMISSIONS, fix);
}
let config: StoredRouter = serde_json::from_slice(&bytes).map_err(|e| {
Error::Config(format!(
"~/{ROUTER_PATH} is not a stored router (line {}, column {})",
e.line(),
e.column()
))
})?;
config.validate()?;
Ok(Some(config))
}
pub fn store(config: &StoredRouter) -> Result<PathBuf> {
config.validate()?;
let bytes = serde_json::to_vec_pretty(config)
.map_err(|_| Error::Config("the router could not be encoded".to_string()))?;
if bytes.len() > MAX_ROUTER_FILE_BYTES {
return Err(Error::Config(format!(
"the router is larger than {MAX_ROUTER_FILE_BYTES} bytes, so it was not stored"
)));
}
store::write_private_file(ROUTER_FILE, &bytes)
.map_err(|e| local(format!("could not write ~/{ROUTER_PATH}: {e}")))
}
pub fn clear() -> Result<bool> {
let Some(path) = path() else {
return Ok(false);
};
match std::fs::remove_file(&path) {
Ok(()) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(local(format!("could not delete ~/{ROUTER_PATH}: {e}"))),
}
}
fn local(message: String) -> Error {
Error::Auth {
cause: AuthCause::Local,
message,
}
}
#[cfg(test)]
pub(crate) mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::indexing_slicing
)]
use super::*;
const TOKEN: &str = "synthetic-provider-token";
const PASSWORD: &str = "synthetic-provider-password";
fn router(mode: &str) -> StoredRouter {
StoredRouter {
router: Router {
mode: Some(mode.to_string()),
provider: Some("zyte".to_string()),
token: Some(TOKEN.to_string()),
credentials: None,
funding: Some("own".to_string()),
},
provider_options: None,
}
}
struct TempDir(PathBuf);
impl TempDir {
fn new(tag: &str) -> TempDir {
let unique = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let path = std::env::temp_dir().join(format!("spider-router-{tag}-{unique}"));
std::fs::create_dir_all(&path).unwrap();
TempDir(path)
}
fn write(&self, contents: &[u8]) -> PathBuf {
let path = self.0.join(ROUTER_FILE);
std::fs::write(&path, contents).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
}
path
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
#[test]
fn the_path_constant_names_the_file_inside_the_spider_directory() {
assert_eq!(ROUTER_PATH, format!("{}/{ROUTER_FILE}", store::SPIDER_DIR));
}
#[test]
fn every_mode_and_funding_the_service_reads_is_accepted() {
for mode in MODES {
for funding in FUNDINGS {
let mut config = router(mode);
config.router.funding = Some((*funding).to_string());
assert_eq!(config.validate(), Ok(()), "{mode} {funding}");
}
}
assert_eq!(StoredRouter::default().validate(), Ok(()));
}
#[test]
fn a_mode_or_funding_outside_the_set_is_refused() {
assert_eq!(router("Fallback").validate(), Err(RouterConfigError::Mode));
let mut config = router("fallback");
config.router.funding = Some("mine".to_string());
assert_eq!(config.validate(), Err(RouterConfigError::Funding));
}
#[test]
fn first_needs_a_provider_and_a_key_for_it() {
let mut config = router("first");
config.router.provider = None;
assert_eq!(
config.validate(),
Err(RouterConfigError::FirstNeedsProvider)
);
let mut config = router("first");
config.router.token = None;
assert_eq!(config.validate(), Err(RouterConfigError::FirstNeedsKey));
config.router.provider = Some("oxylabs".to_string());
config.router.credentials = Some(BTreeMap::from([
("oxylabs_username".to_string(), "name".to_string()),
("oxylabs_password".to_string(), PASSWORD.to_string()),
]));
assert_eq!(config.validate(), Ok(()));
config.router.provider = Some("oxy".to_string());
assert_eq!(config.validate(), Err(RouterConfigError::FirstNeedsKey));
config.router.mode = Some("fallback".to_string());
assert_eq!(config.validate(), Ok(()));
}
#[test]
fn a_blank_token_credential_or_provider_is_refused() {
let mut config = router("fallback");
config.router.token = Some(" ".to_string());
assert_eq!(config.validate(), Err(RouterConfigError::EmptyToken));
let mut config = router("fallback");
config.router.credentials = Some(BTreeMap::from([(
"oxylabs_password".to_string(),
String::new(),
)]));
assert_eq!(
config.validate(),
Err(RouterConfigError::EmptyCredential(
"oxylabs_password".to_string()
))
);
let mut config = router("fallback");
config.router.provider = Some(String::new());
assert_eq!(config.validate(), Err(RouterConfigError::EmptyProvider));
}
#[test]
fn an_unknown_provider_is_allowed() {
let mut config = router("first");
config.router.provider = Some("a-provider-added-later".to_string());
assert_eq!(config.validate(), Ok(()));
}
#[test]
fn debug_prints_names_and_no_secret() {
let mut config = router("fallback");
config.router.credentials = Some(BTreeMap::from([(
"oxylabs_password".to_string(),
PASSWORD.to_string(),
)]));
config.provider_options = Some(BTreeMap::from([(
"zyte".to_string(),
serde_json::json!({"geolocation": "synthetic-option-value"}),
)]));
let printed = format!("{config:?} {config:#?}");
for secret in [TOKEN, PASSWORD, "synthetic-option-value", "geolocation"] {
assert!(!printed.contains(secret), "{printed}");
}
for name in ["oxylabs_password", "zyte", "fallback", "own", REDACTED] {
assert!(printed.contains(name), "{name} missing from {printed}");
}
}
#[test]
fn a_missing_file_is_no_router() {
let dir = TempDir::new("missing");
assert_eq!(load_from(&dir.0.join(ROUTER_FILE)).unwrap(), None);
}
#[test]
fn a_file_round_trips() {
let dir = TempDir::new("round-trip");
let mut config = router("first");
config.provider_options = Some(BTreeMap::from([(
"zyte".to_string(),
serde_json::json!({"geolocation": "US"}),
)]));
let path = dir.write(&serde_json::to_vec(&config).unwrap());
assert_eq!(load_from(&path).unwrap(), Some(config));
}
#[test]
fn a_file_past_the_cap_is_not_read() {
let dir = TempDir::new("oversized");
let mut padded = b"{\"router\":{},\"provider_options\":{\"x\":\"".to_vec();
padded.resize(MAX_ROUTER_FILE_BYTES + 1, b'a');
let path = dir.write(&padded);
let error = load_from(&path).unwrap_err().to_string();
assert!(error.contains("larger than"), "{error}");
}
#[test]
fn a_malformed_file_names_a_position_and_no_value() {
let dir = TempDir::new("malformed");
let text = format!(r#"{{"router":{{"token":["{TOKEN}"]}}}}"#);
let path = dir.write(text.as_bytes());
let error = load_from(&path).unwrap_err();
let printed = format!("{error} {error:?}");
assert!(printed.contains("line 1"), "{printed}");
assert!(!printed.contains(TOKEN), "{printed}");
}
#[test]
fn a_stored_file_that_does_not_validate_is_an_error() {
let dir = TempDir::new("invalid");
let text = format!(r#"{{"router":{{"mode":"sideways","token":"{TOKEN}"}}}}"#);
let path = dir.write(text.as_bytes());
let error = load_from(&path).unwrap_err();
let printed = format!("{error} {error:?}");
assert!(printed.contains("router.mode"), "{printed}");
assert!(!printed.contains("sideways") && !printed.contains(TOKEN));
}
pub(crate) fn every_error(dir: &Path) -> Vec<Error> {
let mut out: Vec<Error> = Vec::new();
let invalid = [
{
let mut config = router("sideways");
config.router.token = Some(TOKEN.to_string());
config
},
{
let mut config = router("fallback");
config.router.funding = Some(TOKEN.to_string());
config
},
{
let mut config = router("first");
config.router.provider = None;
config
},
{
let mut config = router("first");
config.router.token = None;
config
},
{
let mut config = router("fallback");
config.router.provider = Some(" ".to_string());
config
},
{
let mut config = router("fallback");
config.router.token = Some(" ".to_string());
config
},
{
let mut config = router("fallback");
config.router.credentials =
Some(BTreeMap::from([("zyte".to_string(), " ".to_string())]));
config
},
];
for config in &invalid {
out.push(config.validate().unwrap_err().into());
out.push(store(config).unwrap_err());
}
let oversized = dir.join("oversized.json");
let mut padded = format!(r#"{{"router":{{"token":"{TOKEN}"}},"x":""#).into_bytes();
padded.resize(MAX_ROUTER_FILE_BYTES + 1, b'a');
std::fs::write(&oversized, padded).unwrap();
out.push(load_from(&oversized).unwrap_err());
let malformed = dir.join("malformed.json");
std::fs::write(&malformed, format!(r#"{{"router":{{"token":{TOKEN}}}}}"#)).unwrap();
out.push(load_from(&malformed).unwrap_err());
let wrong_type = dir.join("wrong-type.json");
std::fs::write(&wrong_type, format!(r#"{{"router":"{TOKEN}"}}"#)).unwrap();
out.push(load_from(&wrong_type).unwrap_err());
let directory = dir.join("directory.json");
std::fs::create_dir_all(&directory).unwrap();
out.extend(load_from(&directory).err());
out
}
#[test]
fn every_error_here_names_a_field_and_no_value() {
let dir = TempDir::new("errors");
let errors = every_error(&dir.0);
assert_eq!(errors.len(), 18, "an error path stopped being covered");
for error in errors {
let printed = format!("{error} {error:?}");
assert!(!printed.contains(TOKEN), "{printed}");
}
}
}