1use std::collections::BTreeMap;
21use std::fmt;
22use std::path::{Path, PathBuf};
23use std::sync::atomic::AtomicBool;
24
25use serde::{Deserialize, Serialize};
26
27use crate::error::{AuthCause, Error};
28use crate::params::Router;
29use crate::Result;
30
31use super::store;
32
33pub const ROUTER_PATH: &str = ".spider/router.json";
35
36const ROUTER_FILE: &str = "router.json";
38
39pub const MAX_ROUTER_FILE_BYTES: usize = 64 * 1024;
42
43pub const MODES: &[&str] = &["fallback", "first", "off"];
45
46pub const FUNDINGS: &[&str] = &["any", "own"];
48
49const REDACTED: &str = "<redacted>";
51
52static WARNED_ABOUT_PERMISSIONS: AtomicBool = AtomicBool::new(false);
54
55#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
61pub struct StoredRouter {
62 #[serde(default)]
64 pub router: Router,
65 #[serde(default, skip_serializing_if = "Option::is_none")]
68 pub provider_options: Option<BTreeMap<String, serde_json::Value>>,
69}
70
71impl fmt::Debug for StoredRouter {
72 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75 let credentials = self.router.credentials.as_ref().map(|held| {
76 held.keys()
77 .map(|name| (name.as_str(), REDACTED))
78 .collect::<BTreeMap<_, _>>()
79 });
80 let options = self
81 .provider_options
82 .as_ref()
83 .map(|held| held.keys().map(String::as_str).collect::<Vec<_>>());
84 f.debug_struct("StoredRouter")
85 .field("mode", &self.router.mode)
86 .field("provider", &self.router.provider)
87 .field("token", &self.router.token.as_ref().map(|_| REDACTED))
88 .field("credentials", &credentials)
89 .field("funding", &self.router.funding)
90 .field("provider_options", &options)
91 .finish()
92 }
93}
94
95#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
99#[non_exhaustive]
100pub enum RouterConfigError {
101 #[error("router.mode has to be fallback, first or off")]
103 Mode,
104 #[error("router.funding has to be any or own")]
106 Funding,
107 #[error("router.mode first needs router.provider")]
109 FirstNeedsProvider,
110 #[error(
112 "router.mode first needs router.token, or a router.credentials entry for the provider"
113 )]
114 FirstNeedsKey,
115 #[error("router.provider is empty")]
117 EmptyProvider,
118 #[error("router.token is empty")]
120 EmptyToken,
121 #[error("router.credentials.{0} is empty")]
123 EmptyCredential(String),
124}
125
126impl From<RouterConfigError> for Error {
127 fn from(error: RouterConfigError) -> Error {
128 Error::Config(error.to_string())
129 }
130}
131
132impl StoredRouter {
133 pub fn validate(&self) -> std::result::Result<(), RouterConfigError> {
139 let router = &self.router;
140 if let Some(mode) = &router.mode {
141 if !MODES.contains(&mode.as_str()) {
142 return Err(RouterConfigError::Mode);
143 }
144 }
145 if let Some(funding) = &router.funding {
146 if !FUNDINGS.contains(&funding.as_str()) {
147 return Err(RouterConfigError::Funding);
148 }
149 }
150 if router.provider.as_deref().is_some_and(blank) {
151 return Err(RouterConfigError::EmptyProvider);
152 }
153 if router.token.as_deref().is_some_and(blank) {
154 return Err(RouterConfigError::EmptyToken);
155 }
156 for (name, value) in router.credentials.iter().flatten() {
157 if blank(value) {
158 return Err(RouterConfigError::EmptyCredential(name.clone()));
159 }
160 }
161 if router.mode.as_deref() == Some("first") {
162 let Some(provider) = router.provider.as_deref() else {
163 return Err(RouterConfigError::FirstNeedsProvider);
164 };
165 let credential = router.credentials.iter().flatten().any(|(name, _)| {
166 name == provider
167 || name
168 .strip_prefix(provider)
169 .is_some_and(|rest| rest.starts_with('_'))
170 });
171 if router.token.is_none() && !credential {
172 return Err(RouterConfigError::FirstNeedsKey);
173 }
174 }
175 Ok(())
176 }
177}
178
179fn blank(value: &str) -> bool {
180 value.trim().is_empty()
181}
182
183pub fn path() -> Option<PathBuf> {
185 store::spider_dir().map(|dir| dir.join(ROUTER_FILE))
186}
187
188pub fn load() -> Result<Option<StoredRouter>> {
193 match path() {
194 Some(path) => load_from(&path),
195 None => Ok(None),
196 }
197}
198
199pub fn load_from(path: &Path) -> Result<Option<StoredRouter>> {
204 use std::io::Read;
205
206 let file = match std::fs::File::open(path) {
207 Ok(file) => file,
208 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
209 Err(e) => return Err(local(format!("could not open ~/{ROUTER_PATH}: {e}"))),
210 };
211 let mut bytes = Vec::new();
212 file.take(MAX_ROUTER_FILE_BYTES as u64 + 1)
213 .read_to_end(&mut bytes)
214 .map_err(|e| local(format!("could not read ~/{ROUTER_PATH}: {e}")))?;
215 if bytes.len() > MAX_ROUTER_FILE_BYTES {
216 return Err(Error::Config(format!(
217 "~/{ROUTER_PATH} is larger than {MAX_ROUTER_FILE_BYTES} bytes, so it was not read"
218 )));
219 }
220 if let Some(mode) = store::wide_permissions(path) {
221 let fix = "store the router again";
222 store::warn_wide_permissions(path, mode, &WARNED_ABOUT_PERMISSIONS, fix);
223 }
224 let config: StoredRouter = serde_json::from_slice(&bytes).map_err(|e| {
227 Error::Config(format!(
228 "~/{ROUTER_PATH} is not a stored router (line {}, column {})",
229 e.line(),
230 e.column()
231 ))
232 })?;
233 config.validate()?;
234 Ok(Some(config))
235}
236
237pub fn store(config: &StoredRouter) -> Result<PathBuf> {
240 config.validate()?;
241 let bytes = serde_json::to_vec_pretty(config)
242 .map_err(|_| Error::Config("the router could not be encoded".to_string()))?;
243 if bytes.len() > MAX_ROUTER_FILE_BYTES {
244 return Err(Error::Config(format!(
245 "the router is larger than {MAX_ROUTER_FILE_BYTES} bytes, so it was not stored"
246 )));
247 }
248 store::write_private_file(ROUTER_FILE, &bytes)
251 .map_err(|e| local(format!("could not write ~/{ROUTER_PATH}: {e}")))
252}
253
254pub fn clear() -> Result<bool> {
256 let Some(path) = path() else {
257 return Ok(false);
258 };
259 match std::fs::remove_file(&path) {
260 Ok(()) => Ok(true),
261 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
262 Err(e) => Err(local(format!("could not delete ~/{ROUTER_PATH}: {e}"))),
263 }
264}
265
266fn local(message: String) -> Error {
268 Error::Auth {
269 cause: AuthCause::Local,
270 message,
271 }
272}
273
274#[cfg(test)]
275pub(crate) mod tests {
276 #![allow(
279 clippy::unwrap_used,
280 clippy::expect_used,
281 clippy::panic,
282 clippy::indexing_slicing
283 )]
284 use super::*;
285
286 const TOKEN: &str = "synthetic-provider-token";
287 const PASSWORD: &str = "synthetic-provider-password";
288
289 fn router(mode: &str) -> StoredRouter {
290 StoredRouter {
291 router: Router {
292 mode: Some(mode.to_string()),
293 provider: Some("zyte".to_string()),
294 token: Some(TOKEN.to_string()),
295 credentials: None,
296 funding: Some("own".to_string()),
297 },
298 provider_options: None,
299 }
300 }
301
302 struct TempDir(PathBuf);
305
306 impl TempDir {
307 fn new(tag: &str) -> TempDir {
308 let unique = std::time::SystemTime::now()
309 .duration_since(std::time::UNIX_EPOCH)
310 .map(|d| d.as_nanos())
311 .unwrap_or(0);
312 let path = std::env::temp_dir().join(format!("spider-router-{tag}-{unique}"));
313 std::fs::create_dir_all(&path).unwrap();
314 TempDir(path)
315 }
316
317 fn write(&self, contents: &[u8]) -> PathBuf {
320 let path = self.0.join(ROUTER_FILE);
321 std::fs::write(&path, contents).unwrap();
322 #[cfg(unix)]
323 {
324 use std::os::unix::fs::PermissionsExt;
325 std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
326 }
327 path
328 }
329 }
330
331 impl Drop for TempDir {
332 fn drop(&mut self) {
333 let _ = std::fs::remove_dir_all(&self.0);
334 }
335 }
336
337 #[test]
338 fn the_path_constant_names_the_file_inside_the_spider_directory() {
339 assert_eq!(ROUTER_PATH, format!("{}/{ROUTER_FILE}", store::SPIDER_DIR));
340 }
341
342 #[test]
343 fn every_mode_and_funding_the_service_reads_is_accepted() {
344 for mode in MODES {
345 for funding in FUNDINGS {
346 let mut config = router(mode);
347 config.router.funding = Some((*funding).to_string());
348 assert_eq!(config.validate(), Ok(()), "{mode} {funding}");
349 }
350 }
351 assert_eq!(StoredRouter::default().validate(), Ok(()));
352 }
353
354 #[test]
355 fn a_mode_or_funding_outside_the_set_is_refused() {
356 assert_eq!(router("Fallback").validate(), Err(RouterConfigError::Mode));
357 let mut config = router("fallback");
358 config.router.funding = Some("mine".to_string());
359 assert_eq!(config.validate(), Err(RouterConfigError::Funding));
360 }
361
362 #[test]
363 fn first_needs_a_provider_and_a_key_for_it() {
364 let mut config = router("first");
365 config.router.provider = None;
366 assert_eq!(
367 config.validate(),
368 Err(RouterConfigError::FirstNeedsProvider)
369 );
370
371 let mut config = router("first");
372 config.router.token = None;
373 assert_eq!(config.validate(), Err(RouterConfigError::FirstNeedsKey));
374
375 config.router.provider = Some("oxylabs".to_string());
376 config.router.credentials = Some(BTreeMap::from([
377 ("oxylabs_username".to_string(), "name".to_string()),
378 ("oxylabs_password".to_string(), PASSWORD.to_string()),
379 ]));
380 assert_eq!(config.validate(), Ok(()));
381
382 config.router.provider = Some("oxy".to_string());
384 assert_eq!(config.validate(), Err(RouterConfigError::FirstNeedsKey));
385
386 config.router.mode = Some("fallback".to_string());
389 assert_eq!(config.validate(), Ok(()));
390 }
391
392 #[test]
393 fn a_blank_token_credential_or_provider_is_refused() {
394 let mut config = router("fallback");
395 config.router.token = Some(" ".to_string());
396 assert_eq!(config.validate(), Err(RouterConfigError::EmptyToken));
397
398 let mut config = router("fallback");
399 config.router.credentials = Some(BTreeMap::from([(
400 "oxylabs_password".to_string(),
401 String::new(),
402 )]));
403 assert_eq!(
404 config.validate(),
405 Err(RouterConfigError::EmptyCredential(
406 "oxylabs_password".to_string()
407 ))
408 );
409
410 let mut config = router("fallback");
411 config.router.provider = Some(String::new());
412 assert_eq!(config.validate(), Err(RouterConfigError::EmptyProvider));
413 }
414
415 #[test]
416 fn an_unknown_provider_is_allowed() {
417 let mut config = router("first");
418 config.router.provider = Some("a-provider-added-later".to_string());
419 assert_eq!(config.validate(), Ok(()));
420 }
421
422 #[test]
423 fn debug_prints_names_and_no_secret() {
424 let mut config = router("fallback");
425 config.router.credentials = Some(BTreeMap::from([(
426 "oxylabs_password".to_string(),
427 PASSWORD.to_string(),
428 )]));
429 config.provider_options = Some(BTreeMap::from([(
430 "zyte".to_string(),
431 serde_json::json!({"geolocation": "synthetic-option-value"}),
432 )]));
433 let printed = format!("{config:?} {config:#?}");
434 for secret in [TOKEN, PASSWORD, "synthetic-option-value", "geolocation"] {
435 assert!(!printed.contains(secret), "{printed}");
436 }
437 for name in ["oxylabs_password", "zyte", "fallback", "own", REDACTED] {
438 assert!(printed.contains(name), "{name} missing from {printed}");
439 }
440 }
441
442 #[test]
443 fn a_missing_file_is_no_router() {
444 let dir = TempDir::new("missing");
445 assert_eq!(load_from(&dir.0.join(ROUTER_FILE)).unwrap(), None);
446 }
447
448 #[test]
449 fn a_file_round_trips() {
450 let dir = TempDir::new("round-trip");
451 let mut config = router("first");
452 config.provider_options = Some(BTreeMap::from([(
453 "zyte".to_string(),
454 serde_json::json!({"geolocation": "US"}),
455 )]));
456 let path = dir.write(&serde_json::to_vec(&config).unwrap());
457 assert_eq!(load_from(&path).unwrap(), Some(config));
458 }
459
460 #[test]
461 fn a_file_past_the_cap_is_not_read() {
462 let dir = TempDir::new("oversized");
463 let mut padded = b"{\"router\":{},\"provider_options\":{\"x\":\"".to_vec();
464 padded.resize(MAX_ROUTER_FILE_BYTES + 1, b'a');
465 let path = dir.write(&padded);
466 let error = load_from(&path).unwrap_err().to_string();
467 assert!(error.contains("larger than"), "{error}");
468 }
469
470 #[test]
471 fn a_malformed_file_names_a_position_and_no_value() {
472 let dir = TempDir::new("malformed");
473 let text = format!(r#"{{"router":{{"token":["{TOKEN}"]}}}}"#);
474 let path = dir.write(text.as_bytes());
475 let error = load_from(&path).unwrap_err();
476 let printed = format!("{error} {error:?}");
477 assert!(printed.contains("line 1"), "{printed}");
478 assert!(!printed.contains(TOKEN), "{printed}");
479 }
480
481 #[test]
482 fn a_stored_file_that_does_not_validate_is_an_error() {
483 let dir = TempDir::new("invalid");
484 let text = format!(r#"{{"router":{{"mode":"sideways","token":"{TOKEN}"}}}}"#);
485 let path = dir.write(text.as_bytes());
486 let error = load_from(&path).unwrap_err();
487 let printed = format!("{error} {error:?}");
488 assert!(printed.contains("router.mode"), "{printed}");
489 assert!(!printed.contains("sideways") && !printed.contains(TOKEN));
490 }
491
492 pub(crate) fn every_error(dir: &Path) -> Vec<Error> {
494 let mut out: Vec<Error> = Vec::new();
495 let invalid = [
496 {
497 let mut config = router("sideways");
498 config.router.token = Some(TOKEN.to_string());
499 config
500 },
501 {
502 let mut config = router("fallback");
503 config.router.funding = Some(TOKEN.to_string());
504 config
505 },
506 {
507 let mut config = router("first");
508 config.router.provider = None;
509 config
510 },
511 {
512 let mut config = router("first");
513 config.router.token = None;
514 config
515 },
516 {
517 let mut config = router("fallback");
518 config.router.provider = Some(" ".to_string());
519 config
520 },
521 {
522 let mut config = router("fallback");
523 config.router.token = Some(" ".to_string());
524 config
525 },
526 {
527 let mut config = router("fallback");
528 config.router.credentials =
529 Some(BTreeMap::from([("zyte".to_string(), " ".to_string())]));
530 config
531 },
532 ];
533 for config in &invalid {
534 out.push(config.validate().unwrap_err().into());
535 out.push(store(config).unwrap_err());
536 }
537 let oversized = dir.join("oversized.json");
538 let mut padded = format!(r#"{{"router":{{"token":"{TOKEN}"}},"x":""#).into_bytes();
539 padded.resize(MAX_ROUTER_FILE_BYTES + 1, b'a');
540 std::fs::write(&oversized, padded).unwrap();
541 out.push(load_from(&oversized).unwrap_err());
542 let malformed = dir.join("malformed.json");
543 std::fs::write(&malformed, format!(r#"{{"router":{{"token":{TOKEN}}}}}"#)).unwrap();
544 out.push(load_from(&malformed).unwrap_err());
545 let wrong_type = dir.join("wrong-type.json");
546 std::fs::write(&wrong_type, format!(r#"{{"router":"{TOKEN}"}}"#)).unwrap();
548 out.push(load_from(&wrong_type).unwrap_err());
549 let directory = dir.join("directory.json");
551 std::fs::create_dir_all(&directory).unwrap();
552 out.extend(load_from(&directory).err());
553 out
554 }
555
556 #[test]
557 fn every_error_here_names_a_field_and_no_value() {
558 let dir = TempDir::new("errors");
559 let errors = every_error(&dir.0);
560 assert_eq!(errors.len(), 18, "an error path stopped being covered");
561 for error in errors {
562 let printed = format!("{error} {error:?}");
563 assert!(!printed.contains(TOKEN), "{printed}");
564 }
565 }
566}