Skip to main content

ctlog/
lib.rs

1//! # ctlog - canonical Certificate Transparency (crt.sh) client
2//!
3//! Single source of truth for *"query a Certificate Transparency log for
4//! every name ever certified under a domain, and turn the response into a
5//! clean, deduplicated host list"* across the Santh fleet.
6//!
7//! Before this crate existed, the crt.sh query+parse was reimplemented in
8//! at least five places with divergent behavior:
9//!
10//! - `wafrift_recon` - bounded chunked read, 30s timeout, typed errors,
11//!   case-normalized, sorted+deduped. The most defensive copy.
12//! - `gossan` subdomain `ct` source and `origin` `ssl_cert` scanner -
13//!   bounded read via their own helpers, `*.` stripped, lowercased.
14//! - `scald` subdomain discovery - retry/backoff, URL expansion.
15//! - `karyx` origin-discovery `certificate` - **unbounded** `Response::json`
16//!   (OOM on a hostile mirror), **no wildcard** in the query URL (so it
17//!   only saw apex certs and missed every subdomain), and **no case
18//!   normalization** (a mixed-case base domain leaked into results).
19//!
20//! Folding them onto this crate removes the duplication and fixes karyx's
21//! three latent bugs in one move. The crate is split into two layers so
22//! consumers that already own a transport stack don't pay for one they
23//! won't use:
24//!
25//! - **Pure layer** ([`crtsh_query_url`], [`parse_crtsh_hostnames`],
26//!   [`parse_crtsh_subdomains`], [`CrtShEntry`]) - JSON + a defensive URL
27//!   encoder, no async/HTTP. Consumed by gossan (its `ScanClient` +
28//!   governor rate-limiter) and scald (its retry/backoff client), which
29//!   feed an already-fetched body in.
30//! - **Fetch layer** ([`discover_subdomains_ct`],
31//!   [`discover_subdomains_ct_with`], behind the `fetch` feature) - the
32//!   chunk-bounded, timeout-guarded read wafrift hardened. Consumed by
33//!   wafrift and karyx, which have no bespoke crt.sh transport.
34//!
35//! ## Normalization contract
36//!
37//! [`parse_crtsh_hostnames`] / [`parse_crtsh_subdomains`] apply the union
38//! of every consumer's normalization, which is also the most complete:
39//!
40//! 1. Split each `name_value` on `\n` (crt.sh packs the full SAN set of a
41//!    cert into one newline-joined field).
42//! 2. Trim surrounding whitespace.
43//! 3. Strip a leading wildcard label `*.` so `*.api.example.com` still
44//!    yields the concrete `api.example.com` host - a real subdomain the
45//!    drop-the-whole-entry copies (wafrift, scald, karyx) silently lost.
46//! 4. Lowercase (ASCII) so a mixed-case apex compares equal to the
47//!    queried domain and doesn't leak into a subdomain list.
48//! 5. Drop empty strings and anything still containing `*`.
49//! 6. Sort and deduplicate.
50//!
51//! ## Safe defaults
52//!
53//! - **Input size (parse):** bounded by the caller - the parse functions
54//!   take an in-memory `&str` the caller already size-limited. The fetch
55//!   layer enforces [`CT_RESPONSE_MAX_BYTES`] itself.
56//! - **Outbound network:** none in the pure layer; the fetch layer makes
57//!   exactly one GET to crt.sh under [`CT_QUERY_TIMEOUT`].
58//! - **Process spawning / filesystem writes / credential exposure:** none.
59
60#![warn(missing_docs)]
61#![forbid(unsafe_code)]
62#![warn(clippy::pedantic)]
63#![cfg_attr(
64    not(test),
65    deny(
66        clippy::unwrap_used,
67        clippy::expect_used,
68        clippy::todo,
69        clippy::unimplemented,
70        clippy::panic
71    )
72)]
73#![allow(
74    clippy::module_name_repetitions,
75    clippy::must_use_candidate,
76    clippy::missing_errors_doc
77)]
78
79use serde::{Deserialize, Serialize};
80
81/// One row of a crt.sh `output=json` response.
82///
83/// crt.sh returns far more fields (issuer, serial, `not_before`, …); only
84/// `name_value` - the newline-joined CN+SAN set - is load-bearing for name
85/// discovery, and unknown fields are ignored by serde, so this stays
86/// deliberately minimal.
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct CrtShEntry {
89    /// Newline-separated common-name + subject-alternative-name set.
90    pub name_value: String,
91}
92
93/// Errors returned by the `ctlog` parse and fetch layers.
94#[derive(Debug, thiserror::Error)]
95pub enum CtError {
96    /// The crt.sh body was not the expected `Vec<CrtShEntry>` JSON shape.
97    #[error("failed to parse crt.sh response: {0}")]
98    Parse(#[from] serde_json::Error),
99
100    /// The outbound crt.sh request failed at the transport layer (DNS,
101    /// TCP, TLS, or the [`CT_QUERY_TIMEOUT`] elapsed).
102    #[cfg(feature = "fetch")]
103    #[error("crt.sh request failed: {0}")]
104    Transport(#[from] reqwest::Error),
105
106    /// crt.sh answered with a non-success status (502 when overloaded,
107    /// 403 when rate-limiting the caller's IP are the common ones).
108    #[cfg(feature = "fetch")]
109    #[error("crt.sh returned status {0}")]
110    BadStatus(reqwest::StatusCode),
111
112    /// The streamed response exceeded [`CT_RESPONSE_MAX_BYTES`]; buffering
113    /// was aborted rather than letting a hostile/runaway mirror OOM the
114    /// scanner.
115    #[cfg(feature = "fetch")]
116    #[error("crt.sh response exceeded {limit} byte cap (got {got}+ bytes) - refusing to buffer")]
117    ResponseTooLarge {
118        /// The configured byte cap.
119        limit: usize,
120        /// The size reached when the cap was tripped.
121        got: usize,
122    },
123}
124
125/// Clean an input domain by trimming whitespace, stripping leading wildcard labels
126/// (`*.` or `.`), and stripping trailing dots (`.`).
127#[inline]
128pub(crate) fn clean_domain(domain: &str) -> &str {
129    let mut s = domain.trim();
130    loop {
131        let prev = s;
132        if let Some(rest) = s.strip_prefix("*.") {
133            s = rest;
134        }
135        if let Some(rest) = s.strip_prefix('*') {
136            s = rest;
137        }
138        s = s.trim_matches('.');
139        if s == prev {
140            break;
141        }
142    }
143    s
144}
145
146/// Build the canonical crt.sh JSON query URL for `domain`.
147///
148/// Uses the SQL-`LIKE` wildcard prefix `%.` so the query returns every
149/// name certified *under* the domain, not just apex certs - the missing
150/// wildcard was the bug that made karyx's origin discovery blind to
151/// subdomains. The domain itself is percent-encoded so a hostile or
152/// malformed target (spaces, `&`, `#`) can't break out of the query
153/// component.
154///
155/// ```
156/// assert_eq!(
157///     ctlog::crtsh_query_url("Example.com"),
158///     "https://crt.sh/?q=%.Example.com&output=json",
159/// );
160/// ```
161pub fn crtsh_query_url(domain: &str) -> String {
162    format!(
163        "https://crt.sh/?q=%.{}&output=json",
164        urlencoding::encode(clean_domain(domain))
165    )
166}
167
168/// Build a crt.sh query URL that matches certificates for the apex `domain`.
169///
170/// This is the companion to [`crtsh_query_url`]; an apex-only certificate
171/// that never lists a wildcard or subdomain SAN is missed by the `%.` query,
172/// but its subdomains (if any) still appear here. The two result sets are
173/// unioned by the fetch layer.
174///
175/// ```
176/// assert_eq!(
177///     ctlog::crtsh_apex_query_url("Example.com"),
178///     "https://crt.sh/?q=Example.com&output=json",
179/// );
180/// ```
181pub fn crtsh_apex_query_url(domain: &str) -> String {
182    format!(
183        "https://crt.sh/?q={}&output=json",
184        urlencoding::encode(clean_domain(domain))
185    )
186}
187
188/// Parse a crt.sh `output=json` body into the full set of normalized
189/// host names, *including* the queried apex.
190///
191/// Applies the [normalization contract](crate#normalization-contract).
192/// Use this for origin discovery, where the apex resolving to a non-CDN
193/// address is itself a candidate worth probing.
194///
195/// ```
196/// let body = r#"[
197///     {"name_value": "*.api.example.com\nexample.com"},
198///     {"name_value": "WWW.example.com"}
199/// ]"#;
200/// let names = ctlog::parse_crtsh_hostnames(body).unwrap();
201/// assert_eq!(names, ["api.example.com", "example.com", "www.example.com"]);
202/// ```
203pub fn parse_crtsh_hostnames(body: &str) -> Result<Vec<String>, CtError> {
204    normalized_names(body)
205}
206
207/// Parse a crt.sh `output=json` body into normalized *subdomains*,
208/// excluding the queried `domain` itself.
209///
210/// Applies the [normalization contract](crate#normalization-contract)
211/// then drops the apex. Use this for subdomain enumeration, where the
212/// queried domain is already known and only newly discovered names
213/// matter.
214///
215/// ```
216/// let body = r#"[
217///     {"name_value": "api.example.com\n*.example.com"},
218///     {"name_value": "Example.COM"}
219/// ]"#;
220/// // The apex (in any case) and the apex-wildcard are both excluded.
221/// let subs = ctlog::parse_crtsh_subdomains(body, "example.com").unwrap();
222/// assert_eq!(subs, ["api.example.com"]);
223/// ```
224pub fn parse_crtsh_subdomains(body: &str, domain: &str) -> Result<Vec<String>, CtError> {
225    let cleaned = clean_domain(domain);
226    let domain_lower = cleaned.to_ascii_lowercase();
227    let mut names = normalized_names(body)?;
228    names.retain(|n| *n != domain_lower);
229    Ok(names)
230}
231
232/// The shared normalization core behind both public parse functions.
233fn normalized_names(body: &str) -> Result<Vec<String>, CtError> {
234    if body.trim().is_empty() {
235        return Ok(Vec::new());
236    }
237
238    let entries: Vec<CrtShEntry> = serde_json::from_str(body)?;
239
240    // Push directly into the result. The previous `flat_map` collected each
241    // entry's names into a throwaway `Vec<String>` before flattening (one heap
242    // Vec per CT-log entry, of which a busy domain has thousands) (Law 7); a
243    // plain nested loop keeps `e.name_value` alive across the inner iteration
244    // without that intermediate allocation, and fuses the filter into the walk.
245    let mut names: Vec<String> = Vec::new();
246    for e in entries {
247        for raw in e.name_value.split('\n') {
248            let trimmed = raw.trim();
249            // Strip any leading wildcard labels and surrounding dots.
250            let mut s = trimmed;
251            loop {
252                let prev = s;
253                if let Some(rest) = s.strip_prefix("*.") {
254                    s = rest;
255                }
256                if let Some(rest) = s.strip_prefix('*') {
257                    s = rest;
258                }
259                s = s.trim_matches('.');
260                if s == prev {
261                    break;
262                }
263            }
264            let normalized = s.to_ascii_lowercase();
265
266            // Drop anything that can never be a DNS name: empties, residual
267            // wildcards, consecutive dots, whitespace, control characters, and
268            // invalid hostname symbols (URL schemes, paths, credentials, etc.).
269            if !normalized.is_empty()
270                && !normalized.contains('*')
271                && !normalized.contains("..")
272                && !normalized.chars().any(|c| {
273                    c.is_whitespace()
274                        || c.is_control()
275                        || (!c.is_ascii_alphanumeric() && c != '.' && c != '-' && c != '_')
276                })
277            {
278                names.push(normalized);
279            }
280        }
281    }
282
283    names.sort();
284    names.dedup();
285    Ok(names)
286}
287
288#[cfg(feature = "fetch")]
289mod fetch_impl {
290    use super::{parse_crtsh_subdomains, CtError};
291    use serde::de::Error as _;
292    use std::time::Duration;
293
294    /// Timeout for outbound CT-log queries. crt.sh routinely takes
295    /// 10-20s and occasionally hangs entirely; without a cap a discovery
296    /// run would be a DoS-on-self for every blocked-up upstream.
297    pub const CT_QUERY_TIMEOUT: Duration = Duration::from_secs(30);
298
299    /// Hard cap on crt.sh response body size. A real CT-log JSON for a
300    /// busy domain is a few MB at most; an adversarial / misbehaving
301    /// mirror that streams multi-GB nonsense would otherwise OOM the
302    /// scanner before the JSON parser ever ran.
303    pub const CT_RESPONSE_MAX_BYTES: usize = 32 * 1024 * 1024; // 32 MiB
304
305    /// Maximum retry attempts for the fetch layer. crt.sh commonly returns
306    /// 502 when overloaded; a single transient failure should not kill a
307    /// discovery run.
308    pub const CT_MAX_RETRIES: u32 = 3;
309
310    /// Base backoff between retries. Grows linearly with the attempt.
311    pub const CT_RETRY_BACKOFF: Duration = Duration::from_millis(500);
312
313    /// Tunable options for the CT fetch layer.
314    ///
315    /// Callers that need a different endpoint, timeout (via the `reqwest`
316    /// client), or byte cap can override the defaults here.
317    #[derive(Debug, Clone)]
318    pub struct CtOptions {
319        /// Hard cap on the buffered crt.sh response body in bytes.
320        ///
321        /// Defaults to [`CT_RESPONSE_MAX_BYTES`] (32 MiB).
322        pub max_response_bytes: usize,
323        /// Base URL for crt.sh-style queries.
324        ///
325        /// Defaults to `https://crt.sh/` and must not include a trailing
326        /// slash. Override this in tests that drive a local mock server.
327        pub base_url: String,
328    }
329
330    // `CtOptions::default()` resolves through this trait impl (`Default` is
331    // in the prelude); there is deliberately no inherent `default()` method,
332    // which `clippy::should_implement_trait` flags as shadowing the trait.
333    impl Default for CtOptions {
334        fn default() -> Self {
335            Self {
336                max_response_bytes: CT_RESPONSE_MAX_BYTES,
337                base_url: String::from("https://crt.sh"),
338            }
339        }
340    }
341
342    /// Query crt.sh for `domain` over a fresh default client and return
343    /// the deduplicated subdomains.
344    ///
345    /// Convenience wrapper over [`discover_subdomains_ct_with`] for callers
346    /// that have no client to share; it builds one with
347    /// [`CT_QUERY_TIMEOUT`] applied.
348    ///
349    /// # Single-source risk
350    ///
351    /// The default endpoint is `https://crt.sh/`. If crt.sh is down or
352    /// rate-limits the caller, there is no built-in fallback aggregator
353    /// (certspotter, Censys, etc.). Use [`discover_subdomains_ct_with_options`]
354    /// and point [`CtOptions::base_url`] at an alternate crt.sh-compatible
355    /// mirror to distribute the dependency.
356    pub async fn discover_subdomains_ct(domain: &str) -> Result<Vec<String>, CtError> {
357        let client = reqwest::Client::builder()
358            .timeout(CT_QUERY_TIMEOUT)
359            .build()?;
360        discover_subdomains_ct_with(&client, domain).await
361    }
362
363    /// Query crt.sh for `domain` reusing the caller's `client` (and its
364    /// connection pool / proxy / UA config) and return the deduplicated
365    /// subdomains.
366    ///
367    /// Retries transient failures (5xx, 429, transport errors) up to
368    /// [`CT_MAX_RETRIES`] with a linear [`CT_RETRY_BACKOFF`]. The body is
369    /// read chunk-by-chunk and aborted past [`CT_RESPONSE_MAX_BYTES`]
370    /// rather than buffered whole, so a hostile mirror cannot OOM the
371    /// caller regardless of its advertised `Content-Length`.
372    ///
373    /// See [`discover_subdomains_ct`] for the single-source risk note.
374    pub async fn discover_subdomains_ct_with(
375        client: &reqwest::Client,
376        domain: &str,
377    ) -> Result<Vec<String>, CtError> {
378        discover_subdomains_ct_with_options(client, domain, CtOptions::default()).await
379    }
380
381    /// Query crt.sh for `domain` reusing the caller's `client` and tunable
382    /// [`CtOptions`].
383    ///
384    /// Behaves like [`discover_subdomains_ct_with`] but lets the caller
385    /// override the response size cap.
386    pub async fn discover_subdomains_ct_with_options(
387        client: &reqwest::Client,
388        domain: &str,
389        options: CtOptions,
390    ) -> Result<Vec<String>, CtError> {
391        let clean = super::clean_domain(domain);
392        if clean.is_empty() {
393            return Ok(Vec::new());
394        }
395
396        let base = options.base_url.trim_end_matches('/');
397        let encoded = urlencoding::encode(clean);
398        let wildcard_url = format!("{base}/?q=%.{encoded}&output=json");
399        let apex_url = format!("{base}/?q={encoded}&output=json");
400
401        let wildcard_res = discover_subdomains_ct_with_url(client, &wildcard_url, clean, &options).await;
402        let apex_res = discover_subdomains_ct_with_url(client, &apex_url, clean, &options).await;
403
404        match (wildcard_res, apex_res) {
405            (Ok(mut names), Ok(mut apex_names)) => {
406                names.append(&mut apex_names);
407                names.sort_unstable();
408                names.dedup();
409                Ok(names)
410            }
411            (Ok(names), Err(e)) => {
412                tracing::warn!(domain, error = %e, "apex query failed, returning wildcard results");
413                Ok(names)
414            }
415            (Err(e), Ok(apex_names)) => {
416                tracing::warn!(domain, error = %e, "wildcard query failed, returning apex results");
417                Ok(apex_names)
418            }
419            (Err(e1), Err(_)) => Err(e1),
420        }
421    }
422
423    /// Internal test seam: query an explicit `url` so `fetch` tests can
424    /// drive a local `wiremock` server rather than the real `crt.sh` endpoint.
425    pub(crate) async fn discover_subdomains_ct_with_url(
426        client: &reqwest::Client,
427        url: &str,
428        domain: &str,
429        options: &CtOptions,
430    ) -> Result<Vec<String>, CtError> {
431        tracing::debug!(domain, "querying crt.sh for CT logs");
432
433        let mut last_err: Option<CtError> = None;
434        for attempt in 0..CT_MAX_RETRIES {
435            if attempt > 0 {
436                let delay = CT_RETRY_BACKOFF * attempt;
437                tracing::warn!(attempt, domain, "crt.sh query retry after {delay:?}");
438                tokio::time::sleep(delay).await;
439            }
440
441            match try_query(client, url, domain, options).await {
442                Ok(names) => return Ok(names),
443                Err(e) if is_retryable(&e) => {
444                    tracing::warn!(attempt, error = %e, "crt.sh query failed; will retry if attempts remain");
445                    last_err = Some(e);
446                }
447                Err(e) => return Err(e),
448            }
449        }
450
451        Err(last_err.unwrap_or_else(|| CtError::BadStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE)))
452    }
453
454    async fn try_query(
455        client: &reqwest::Client,
456        url: &str,
457        domain: &str,
458        options: &CtOptions,
459    ) -> Result<Vec<String>, CtError> {
460        let max_response_bytes = options.max_response_bytes;
461        let mut res = client.get(url).send().await?;
462        if !res.status().is_success() {
463            return Err(CtError::BadStatus(res.status()));
464        }
465
466        let mut body = Vec::with_capacity(64 * 1024);
467        while let Some(chunk) = res.chunk().await? {
468            if body.len() + chunk.len() > max_response_bytes {
469                return Err(CtError::ResponseTooLarge {
470                    limit: max_response_bytes,
471                    got: body.len() + chunk.len(),
472                });
473            }
474            body.extend_from_slice(&chunk);
475        }
476
477        // crt.sh emits UTF-8 JSON. A stray invalid byte is not silently
478        // lossy-replaced (which could create bogus hostnames with U+FFFD);
479        // instead it becomes a parse error so the caller knows the response
480        // was corrupted.
481        let text = String::from_utf8(body).map_err(|e| {
482            CtError::Parse(serde_json::Error::custom(format!(
483                "CT response contained invalid UTF-8: {e}"
484            )))
485        })?;
486
487        // crt.sh and upstream proxies (Cloudflare, nginx) frequently emit 200 OK
488        // status headers with HTML, JSON error objects, or plain text error pages
489        // (e.g. Gateway Time-out or {"error": "..."}).
490        // Catch non-array responses early and surface as SERVICE_UNAVAILABLE so the
491        // caller's retry loop is triggered rather than failing immediately on JSON parse.
492        let trimmed_text = text.trim_start();
493        if !trimmed_text.starts_with('[') {
494            return Err(CtError::BadStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE));
495        }
496
497        let names = parse_crtsh_subdomains(&text, domain)?;
498        tracing::debug!(found = names.len(), "discovered subdomains via CT logs");
499        Ok(names)
500    }
501
502    pub(crate) fn is_retryable(err: &CtError) -> bool {
503        match err {
504            CtError::Transport(_) => true,
505            CtError::BadStatus(status) => {
506                status.is_server_error()
507                    || *status == reqwest::StatusCode::TOO_MANY_REQUESTS
508                    || *status == reqwest::StatusCode::REQUEST_TIMEOUT
509            }
510            _ => false,
511        }
512    }
513}
514
515#[cfg(feature = "fetch")]
516pub use fetch_impl::{
517    discover_subdomains_ct, discover_subdomains_ct_with, discover_subdomains_ct_with_options,
518    CtOptions, CT_MAX_RETRIES, CT_QUERY_TIMEOUT, CT_RESPONSE_MAX_BYTES, CT_RETRY_BACKOFF,
519};
520// Only the feature-gated `fetch_tests` module uses this, so it must be gated on
521// `feature = "fetch"` too - otherwise `cargo test` (default features) fails to
522// compile with `unresolved import fetch_impl` (the module only exists under the
523// `fetch` feature).
524#[cfg(all(test, feature = "fetch"))]
525pub(crate) use fetch_impl::discover_subdomains_ct_with_url;
526
527#[cfg(test)]
528mod tests;