Skip to main content

spider_cloud_agent/auth/
router.rs

1//! A provider fallback kept on this machine, for every request that did not
2//! name one.
3//!
4//! The request parameter `router` puts an outside scraping provider behind the
5//! service's own fetch (`fallback`), in front of it (`first`), or out of the
6//! request (`off`), and `provider_options` carries vendor settings for routes
7//! your own key pays for. Setting both on every operation is repetitive, and
8//! the token in it is a key. This module keeps one copy in
9//! `~/.spider/router.json`, written the way the credentials file is written:
10//! owner only from the moment it exists, and replaced whole.
11//!
12//! A client built with [`crate::SpiderBuilder::stored_router`] reads the file
13//! once, and every page operation then fills in `router` and
14//! `provider_options` where the caller left them unset. A caller who set
15//! `router` gets exactly what they set, `mode: off` included.
16//!
17//! Nothing here prints a key. `Debug` on [`StoredRouter`] is written by hand,
18//! and no error names a value, only the field it came from.
19
20use std::collections::BTreeMap;
21use std::fmt;
22use std::path::{Path, PathBuf};
23use std::sync::atomic::AtomicBool;
24
25use serde::{Deserialize, Serialize};
26
27use crate::error::{AuthCause, Error};
28use crate::params::Router;
29use crate::Result;
30
31use super::store;
32
33/// Where the stored router lives, relative to the home directory.
34pub const ROUTER_PATH: &str = ".spider/router.json";
35
36/// The file name inside `~/.spider`.
37const ROUTER_FILE: &str = "router.json";
38
39/// The most a router file may hold and still be read. A router with every
40/// provider's keys and options is a few kilobytes.
41pub const MAX_ROUTER_FILE_BYTES: usize = 64 * 1024;
42
43/// The modes the service reads.
44pub const MODES: &[&str] = &["fallback", "first", "off"];
45
46/// The funding rules the service reads.
47pub const FUNDINGS: &[&str] = &["any", "own"];
48
49/// Shown wherever a key would otherwise be printed.
50const REDACTED: &str = "<redacted>";
51
52/// Once per process, as for the credentials file.
53static WARNED_ABOUT_PERMISSIONS: AtomicBool = AtomicBool::new(false);
54
55/// A `router` parameter and its `provider_options`, as stored.
56///
57/// `Debug` prints the mode, the provider, the funding rule, the credential
58/// names and the provider names under the options. The token and every
59/// credential value print as `<redacted>`, and no option value is printed.
60#[derive(Clone, Default, PartialEq, Serialize, Deserialize)]
61pub struct StoredRouter {
62    /// The `router` parameter every request without one receives.
63    #[serde(default)]
64    pub router: Router,
65    /// The `provider_options` every request without them receives, keyed by
66    /// provider name.
67    #[serde(default, skip_serializing_if = "Option::is_none")]
68    pub provider_options: Option<BTreeMap<String, serde_json::Value>>,
69}
70
71impl fmt::Debug for StoredRouter {
72    /// Written by hand so a token cannot reach a log line through a derived
73    /// `Debug` on something that holds one of these.
74    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75        let credentials = self.router.credentials.as_ref().map(|held| {
76            held.keys()
77                .map(|name| (name.as_str(), REDACTED))
78                .collect::<BTreeMap<_, _>>()
79        });
80        let options = self
81            .provider_options
82            .as_ref()
83            .map(|held| held.keys().map(String::as_str).collect::<Vec<_>>());
84        f.debug_struct("StoredRouter")
85            .field("mode", &self.router.mode)
86            .field("provider", &self.router.provider)
87            .field("token", &self.router.token.as_ref().map(|_| REDACTED))
88            .field("credentials", &credentials)
89            .field("funding", &self.router.funding)
90            .field("provider_options", &options)
91            .finish()
92    }
93}
94
95/// Why a router cannot be stored or used.
96///
97/// Every message names the field and never the value in it.
98#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
99#[non_exhaustive]
100pub enum RouterConfigError {
101    /// `mode` is not one of [`MODES`].
102    #[error("router.mode has to be fallback, first or off")]
103    Mode,
104    /// `funding` is not one of [`FUNDINGS`].
105    #[error("router.funding has to be any or own")]
106    Funding,
107    /// `mode` is `first` and no provider is named.
108    #[error("router.mode first needs router.provider")]
109    FirstNeedsProvider,
110    /// `mode` is `first` and there is no key for the provider.
111    #[error(
112        "router.mode first needs router.token, or a router.credentials entry for the provider"
113    )]
114    FirstNeedsKey,
115    /// `provider` is present and blank.
116    #[error("router.provider is empty")]
117    EmptyProvider,
118    /// `token` is present and blank.
119    #[error("router.token is empty")]
120    EmptyToken,
121    /// A credential is present and blank. Holds the credential's name.
122    #[error("router.credentials.{0} is empty")]
123    EmptyCredential(String),
124}
125
126impl From<RouterConfigError> for Error {
127    fn from(error: RouterConfigError) -> Error {
128        Error::Config(error.to_string())
129    }
130}
131
132impl StoredRouter {
133    /// Check the values the service reads as a closed set, and that `first`
134    /// has something to call.
135    ///
136    /// An unknown provider name is allowed, because the service adds providers
137    /// without a release of this crate.
138    pub fn validate(&self) -> std::result::Result<(), RouterConfigError> {
139        let router = &self.router;
140        if let Some(mode) = &router.mode {
141            if !MODES.contains(&mode.as_str()) {
142                return Err(RouterConfigError::Mode);
143            }
144        }
145        if let Some(funding) = &router.funding {
146            if !FUNDINGS.contains(&funding.as_str()) {
147                return Err(RouterConfigError::Funding);
148            }
149        }
150        if router.provider.as_deref().is_some_and(blank) {
151            return Err(RouterConfigError::EmptyProvider);
152        }
153        if router.token.as_deref().is_some_and(blank) {
154            return Err(RouterConfigError::EmptyToken);
155        }
156        for (name, value) in router.credentials.iter().flatten() {
157            if blank(value) {
158                return Err(RouterConfigError::EmptyCredential(name.clone()));
159            }
160        }
161        if router.mode.as_deref() == Some("first") {
162            let Some(provider) = router.provider.as_deref() else {
163                return Err(RouterConfigError::FirstNeedsProvider);
164            };
165            let credential = router.credentials.iter().flatten().any(|(name, _)| {
166                name == provider
167                    || name
168                        .strip_prefix(provider)
169                        .is_some_and(|rest| rest.starts_with('_'))
170            });
171            if router.token.is_none() && !credential {
172                return Err(RouterConfigError::FirstNeedsKey);
173            }
174        }
175        Ok(())
176    }
177}
178
179fn blank(value: &str) -> bool {
180    value.trim().is_empty()
181}
182
183/// `~/.spider/router.json`, or `None` when there is no home directory.
184pub fn path() -> Option<PathBuf> {
185    store::spider_dir().map(|dir| dir.join(ROUTER_FILE))
186}
187
188/// The stored router, or `None` when no file is there.
189///
190/// A file that is there and cannot be used is an error rather than no router,
191/// because a fallback the caller thinks is on and is not costs pages.
192pub fn load() -> Result<Option<StoredRouter>> {
193    match path() {
194        Some(path) => load_from(&path),
195        None => Ok(None),
196    }
197}
198
199/// The router stored at a path, or `None` when no file is there.
200///
201/// The read stops at [`MAX_ROUTER_FILE_BYTES`]. A file readable by anyone but
202/// its owner is reported once per process on Unix and left as it is.
203pub fn load_from(path: &Path) -> Result<Option<StoredRouter>> {
204    use std::io::Read;
205
206    let file = match std::fs::File::open(path) {
207        Ok(file) => file,
208        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
209        Err(e) => return Err(local(format!("could not open ~/{ROUTER_PATH}: {e}"))),
210    };
211    let mut bytes = Vec::new();
212    file.take(MAX_ROUTER_FILE_BYTES as u64 + 1)
213        .read_to_end(&mut bytes)
214        .map_err(|e| local(format!("could not read ~/{ROUTER_PATH}: {e}")))?;
215    if bytes.len() > MAX_ROUTER_FILE_BYTES {
216        return Err(Error::Config(format!(
217            "~/{ROUTER_PATH} is larger than {MAX_ROUTER_FILE_BYTES} bytes, so it was not read"
218        )));
219    }
220    if let Some(mode) = store::wide_permissions(path) {
221        let fix = "store the router again";
222        store::warn_wide_permissions(path, mode, &WARNED_ABOUT_PERMISSIONS, fix);
223    }
224    // A serde message quotes the value it choked on, and that value can be a
225    // key. Only the position is kept.
226    let config: StoredRouter = serde_json::from_slice(&bytes).map_err(|e| {
227        Error::Config(format!(
228            "~/{ROUTER_PATH} is not a stored router (line {}, column {})",
229            e.line(),
230            e.column()
231        ))
232    })?;
233    config.validate()?;
234    Ok(Some(config))
235}
236
237/// Write the router to `~/.spider/router.json`, owner only, replacing what was
238/// there. Returns the path.
239pub fn store(config: &StoredRouter) -> Result<PathBuf> {
240    config.validate()?;
241    let bytes = serde_json::to_vec_pretty(config)
242        .map_err(|_| Error::Config("the router could not be encoded".to_string()))?;
243    if bytes.len() > MAX_ROUTER_FILE_BYTES {
244        return Err(Error::Config(format!(
245            "the router is larger than {MAX_ROUTER_FILE_BYTES} bytes, so it was not stored"
246        )));
247    }
248    // The directory is created at its owner only mode if it is missing, and
249    // the file is created owner only beside the path and renamed over it.
250    store::write_private_file(ROUTER_FILE, &bytes)
251        .map_err(|e| local(format!("could not write ~/{ROUTER_PATH}: {e}")))
252}
253
254/// Delete the stored router. True when there was one to delete.
255pub fn clear() -> Result<bool> {
256    let Some(path) = path() else {
257        return Ok(false);
258    };
259    match std::fs::remove_file(&path) {
260        Ok(()) => Ok(true),
261        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
262        Err(e) => Err(local(format!("could not delete ~/{ROUTER_PATH}: {e}"))),
263    }
264}
265
266/// A problem on this machine rather than with the router.
267fn local(message: String) -> Error {
268    Error::Auth {
269        cause: AuthCause::Local,
270        message,
271    }
272}
273
274#[cfg(test)]
275pub(crate) mod tests {
276    // A test may unwrap and may panic: a test that cannot set itself up should
277    // fail loudly rather than quietly measure nothing.
278    #![allow(
279        clippy::unwrap_used,
280        clippy::expect_used,
281        clippy::panic,
282        clippy::indexing_slicing
283    )]
284    use super::*;
285
286    const TOKEN: &str = "synthetic-provider-token";
287    const PASSWORD: &str = "synthetic-provider-password";
288
289    fn router(mode: &str) -> StoredRouter {
290        StoredRouter {
291            router: Router {
292                mode: Some(mode.to_string()),
293                provider: Some("zyte".to_string()),
294                token: Some(TOKEN.to_string()),
295                credentials: None,
296                funding: Some("own".to_string()),
297            },
298            provider_options: None,
299        }
300    }
301
302    /// A directory of our own, removed when the test ends. Nothing here reads
303    /// or writes the developer's `~/.spider`.
304    struct TempDir(PathBuf);
305
306    impl TempDir {
307        fn new(tag: &str) -> TempDir {
308            let unique = std::time::SystemTime::now()
309                .duration_since(std::time::UNIX_EPOCH)
310                .map(|d| d.as_nanos())
311                .unwrap_or(0);
312            let path = std::env::temp_dir().join(format!("spider-router-{tag}-{unique}"));
313            std::fs::create_dir_all(&path).unwrap();
314            TempDir(path)
315        }
316
317        /// Written owner only, so no test here sets off the permission
318        /// warning that the credentials tests count.
319        fn write(&self, contents: &[u8]) -> PathBuf {
320            let path = self.0.join(ROUTER_FILE);
321            std::fs::write(&path, contents).unwrap();
322            #[cfg(unix)]
323            {
324                use std::os::unix::fs::PermissionsExt;
325                std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
326            }
327            path
328        }
329    }
330
331    impl Drop for TempDir {
332        fn drop(&mut self) {
333            let _ = std::fs::remove_dir_all(&self.0);
334        }
335    }
336
337    #[test]
338    fn the_path_constant_names_the_file_inside_the_spider_directory() {
339        assert_eq!(ROUTER_PATH, format!("{}/{ROUTER_FILE}", store::SPIDER_DIR));
340    }
341
342    #[test]
343    fn every_mode_and_funding_the_service_reads_is_accepted() {
344        for mode in MODES {
345            for funding in FUNDINGS {
346                let mut config = router(mode);
347                config.router.funding = Some((*funding).to_string());
348                assert_eq!(config.validate(), Ok(()), "{mode} {funding}");
349            }
350        }
351        assert_eq!(StoredRouter::default().validate(), Ok(()));
352    }
353
354    #[test]
355    fn a_mode_or_funding_outside_the_set_is_refused() {
356        assert_eq!(router("Fallback").validate(), Err(RouterConfigError::Mode));
357        let mut config = router("fallback");
358        config.router.funding = Some("mine".to_string());
359        assert_eq!(config.validate(), Err(RouterConfigError::Funding));
360    }
361
362    #[test]
363    fn first_needs_a_provider_and_a_key_for_it() {
364        let mut config = router("first");
365        config.router.provider = None;
366        assert_eq!(
367            config.validate(),
368            Err(RouterConfigError::FirstNeedsProvider)
369        );
370
371        let mut config = router("first");
372        config.router.token = None;
373        assert_eq!(config.validate(), Err(RouterConfigError::FirstNeedsKey));
374
375        config.router.provider = Some("oxylabs".to_string());
376        config.router.credentials = Some(BTreeMap::from([
377            ("oxylabs_username".to_string(), "name".to_string()),
378            ("oxylabs_password".to_string(), PASSWORD.to_string()),
379        ]));
380        assert_eq!(config.validate(), Ok(()));
381
382        // A credential for another provider is not a key for this one.
383        config.router.provider = Some("oxy".to_string());
384        assert_eq!(config.validate(), Err(RouterConfigError::FirstNeedsKey));
385
386        // Fallback needs neither: the provider may already be armed on the
387        // account.
388        config.router.mode = Some("fallback".to_string());
389        assert_eq!(config.validate(), Ok(()));
390    }
391
392    #[test]
393    fn a_blank_token_credential_or_provider_is_refused() {
394        let mut config = router("fallback");
395        config.router.token = Some("  ".to_string());
396        assert_eq!(config.validate(), Err(RouterConfigError::EmptyToken));
397
398        let mut config = router("fallback");
399        config.router.credentials = Some(BTreeMap::from([(
400            "oxylabs_password".to_string(),
401            String::new(),
402        )]));
403        assert_eq!(
404            config.validate(),
405            Err(RouterConfigError::EmptyCredential(
406                "oxylabs_password".to_string()
407            ))
408        );
409
410        let mut config = router("fallback");
411        config.router.provider = Some(String::new());
412        assert_eq!(config.validate(), Err(RouterConfigError::EmptyProvider));
413    }
414
415    #[test]
416    fn an_unknown_provider_is_allowed() {
417        let mut config = router("first");
418        config.router.provider = Some("a-provider-added-later".to_string());
419        assert_eq!(config.validate(), Ok(()));
420    }
421
422    #[test]
423    fn debug_prints_names_and_no_secret() {
424        let mut config = router("fallback");
425        config.router.credentials = Some(BTreeMap::from([(
426            "oxylabs_password".to_string(),
427            PASSWORD.to_string(),
428        )]));
429        config.provider_options = Some(BTreeMap::from([(
430            "zyte".to_string(),
431            serde_json::json!({"geolocation": "synthetic-option-value"}),
432        )]));
433        let printed = format!("{config:?} {config:#?}");
434        for secret in [TOKEN, PASSWORD, "synthetic-option-value", "geolocation"] {
435            assert!(!printed.contains(secret), "{printed}");
436        }
437        for name in ["oxylabs_password", "zyte", "fallback", "own", REDACTED] {
438            assert!(printed.contains(name), "{name} missing from {printed}");
439        }
440    }
441
442    #[test]
443    fn a_missing_file_is_no_router() {
444        let dir = TempDir::new("missing");
445        assert_eq!(load_from(&dir.0.join(ROUTER_FILE)).unwrap(), None);
446    }
447
448    #[test]
449    fn a_file_round_trips() {
450        let dir = TempDir::new("round-trip");
451        let mut config = router("first");
452        config.provider_options = Some(BTreeMap::from([(
453            "zyte".to_string(),
454            serde_json::json!({"geolocation": "US"}),
455        )]));
456        let path = dir.write(&serde_json::to_vec(&config).unwrap());
457        assert_eq!(load_from(&path).unwrap(), Some(config));
458    }
459
460    #[test]
461    fn a_file_past_the_cap_is_not_read() {
462        let dir = TempDir::new("oversized");
463        let mut padded = b"{\"router\":{},\"provider_options\":{\"x\":\"".to_vec();
464        padded.resize(MAX_ROUTER_FILE_BYTES + 1, b'a');
465        let path = dir.write(&padded);
466        let error = load_from(&path).unwrap_err().to_string();
467        assert!(error.contains("larger than"), "{error}");
468    }
469
470    #[test]
471    fn a_malformed_file_names_a_position_and_no_value() {
472        let dir = TempDir::new("malformed");
473        let text = format!(r#"{{"router":{{"token":["{TOKEN}"]}}}}"#);
474        let path = dir.write(text.as_bytes());
475        let error = load_from(&path).unwrap_err();
476        let printed = format!("{error} {error:?}");
477        assert!(printed.contains("line 1"), "{printed}");
478        assert!(!printed.contains(TOKEN), "{printed}");
479    }
480
481    #[test]
482    fn a_stored_file_that_does_not_validate_is_an_error() {
483        let dir = TempDir::new("invalid");
484        let text = format!(r#"{{"router":{{"mode":"sideways","token":"{TOKEN}"}}}}"#);
485        let path = dir.write(text.as_bytes());
486        let error = load_from(&path).unwrap_err();
487        let printed = format!("{error} {error:?}");
488        assert!(printed.contains("router.mode"), "{printed}");
489        assert!(!printed.contains("sideways") && !printed.contains(TOKEN));
490    }
491
492    /// Every error this module makes, built the way it builds them.
493    pub(crate) fn every_error(dir: &Path) -> Vec<Error> {
494        let mut out: Vec<Error> = Vec::new();
495        let invalid = [
496            {
497                let mut config = router("sideways");
498                config.router.token = Some(TOKEN.to_string());
499                config
500            },
501            {
502                let mut config = router("fallback");
503                config.router.funding = Some(TOKEN.to_string());
504                config
505            },
506            {
507                let mut config = router("first");
508                config.router.provider = None;
509                config
510            },
511            {
512                let mut config = router("first");
513                config.router.token = None;
514                config
515            },
516            {
517                let mut config = router("fallback");
518                config.router.provider = Some(" ".to_string());
519                config
520            },
521            {
522                let mut config = router("fallback");
523                config.router.token = Some(" ".to_string());
524                config
525            },
526            {
527                let mut config = router("fallback");
528                config.router.credentials =
529                    Some(BTreeMap::from([("zyte".to_string(), " ".to_string())]));
530                config
531            },
532        ];
533        for config in &invalid {
534            out.push(config.validate().unwrap_err().into());
535            out.push(store(config).unwrap_err());
536        }
537        let oversized = dir.join("oversized.json");
538        let mut padded = format!(r#"{{"router":{{"token":"{TOKEN}"}},"x":""#).into_bytes();
539        padded.resize(MAX_ROUTER_FILE_BYTES + 1, b'a');
540        std::fs::write(&oversized, padded).unwrap();
541        out.push(load_from(&oversized).unwrap_err());
542        let malformed = dir.join("malformed.json");
543        std::fs::write(&malformed, format!(r#"{{"router":{{"token":{TOKEN}}}}}"#)).unwrap();
544        out.push(load_from(&malformed).unwrap_err());
545        let wrong_type = dir.join("wrong-type.json");
546        // serde quotes a string it did not expect, which is the value.
547        std::fs::write(&wrong_type, format!(r#"{{"router":"{TOKEN}"}}"#)).unwrap();
548        out.push(load_from(&wrong_type).unwrap_err());
549        // A directory where the file should be cannot be read.
550        let directory = dir.join("directory.json");
551        std::fs::create_dir_all(&directory).unwrap();
552        out.extend(load_from(&directory).err());
553        out
554    }
555
556    #[test]
557    fn every_error_here_names_a_field_and_no_value() {
558        let dir = TempDir::new("errors");
559        let errors = every_error(&dir.0);
560        assert_eq!(errors.len(), 18, "an error path stopped being covered");
561        for error in errors {
562            let printed = format!("{error} {error:?}");
563            assert!(!printed.contains(TOKEN), "{printed}");
564        }
565    }
566}