ctlog/lib.rs
1//! # ctlog - canonical Certificate Transparency (crt.sh) client
2//!
3//! Single source of truth for *"query a Certificate Transparency log for
4//! every name ever certified under a domain, and turn the response into a
5//! clean, deduplicated host list"* across the Santh fleet.
6//!
7//! Before this crate existed, the crt.sh query+parse was reimplemented in
8//! at least five places with divergent behavior:
9//!
10//! - `wafrift_recon` - bounded chunked read, 30s timeout, typed errors,
11//! case-normalized, sorted+deduped. The most defensive copy.
12//! - `gossan` subdomain `ct` source and `origin` `ssl_cert` scanner -
13//! bounded read via their own helpers, `*.` stripped, lowercased.
14//! - `scald` subdomain discovery - retry/backoff, URL expansion.
15//! - `karyx` origin-discovery `certificate` - **unbounded** `Response::json`
16//! (OOM on a hostile mirror), **no wildcard** in the query URL (so it
17//! only saw apex certs and missed every subdomain), and **no case
18//! normalization** (a mixed-case base domain leaked into results).
19//!
20//! Folding them onto this crate removes the duplication and fixes karyx's
21//! three latent bugs in one move. The crate is split into two layers so
22//! consumers that already own a transport stack don't pay for one they
23//! won't use:
24//!
25//! - **Pure layer** ([`crtsh_query_url`], [`parse_crtsh_hostnames`],
26//! [`parse_crtsh_subdomains`], [`CrtShEntry`]) - JSON + a defensive URL
27//! encoder, no async/HTTP. Consumed by gossan (its `ScanClient` +
28//! governor rate-limiter) and scald (its retry/backoff client), which
29//! feed an already-fetched body in.
30//! - **Fetch layer** ([`discover_subdomains_ct`],
31//! [`discover_subdomains_ct_with`], behind the `fetch` feature) - the
32//! chunk-bounded, timeout-guarded read wafrift hardened. Consumed by
33//! wafrift and karyx, which have no bespoke crt.sh transport.
34//!
35//! ## Normalization contract
36//!
37//! [`parse_crtsh_hostnames`] / [`parse_crtsh_subdomains`] apply the union
38//! of every consumer's normalization, which is also the most complete:
39//!
40//! 1. Split each `name_value` on `\n` (crt.sh packs the full SAN set of a
41//! cert into one newline-joined field).
42//! 2. Trim surrounding whitespace.
43//! 3. Strip a leading wildcard label `*.` so `*.api.example.com` still
44//! yields the concrete `api.example.com` host - a real subdomain the
45//! drop-the-whole-entry copies (wafrift, scald, karyx) silently lost.
46//! 4. Lowercase (ASCII) so a mixed-case apex compares equal to the
47//! queried domain and doesn't leak into a subdomain list.
48//! 5. Drop empty strings and anything still containing `*`.
49//! 6. Sort and deduplicate.
50//!
51//! ## Safe defaults
52//!
53//! - **Input size (parse):** bounded by the caller - the parse functions
54//! take an in-memory `&str` the caller already size-limited. The fetch
55//! layer enforces [`CT_RESPONSE_MAX_BYTES`] itself.
56//! - **Outbound network:** none in the pure layer; the fetch layer makes
57//! exactly one GET to crt.sh under [`CT_QUERY_TIMEOUT`].
58//! - **Process spawning / filesystem writes / credential exposure:** none.
59
60#![warn(missing_docs)]
61#![forbid(unsafe_code)]
62#![warn(clippy::pedantic)]
63#![cfg_attr(
64 not(test),
65 deny(
66 clippy::unwrap_used,
67 clippy::expect_used,
68 clippy::todo,
69 clippy::unimplemented,
70 clippy::panic
71 )
72)]
73#![allow(
74 clippy::module_name_repetitions,
75 clippy::must_use_candidate,
76 clippy::missing_errors_doc
77)]
78
79use serde::{Deserialize, Serialize};
80
81/// One row of a crt.sh `output=json` response.
82///
83/// crt.sh returns far more fields (issuer, serial, `not_before`, …); only
84/// `name_value` - the newline-joined CN+SAN set - is load-bearing for name
85/// discovery, and unknown fields are ignored by serde, so this stays
86/// deliberately minimal.
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct CrtShEntry {
89 /// Newline-separated common-name + subject-alternative-name set.
90 pub name_value: String,
91}
92
93/// Errors returned by the `ctlog` parse and fetch layers.
94#[derive(Debug, thiserror::Error)]
95pub enum CtError {
96 /// The crt.sh body was not the expected `Vec<CrtShEntry>` JSON shape.
97 #[error("failed to parse crt.sh response: {0}")]
98 Parse(#[from] serde_json::Error),
99
100 /// The outbound crt.sh request failed at the transport layer (DNS,
101 /// TCP, TLS, or the [`CT_QUERY_TIMEOUT`] elapsed).
102 #[cfg(feature = "fetch")]
103 #[error("crt.sh request failed: {0}")]
104 Transport(#[from] reqwest::Error),
105
106 /// crt.sh answered with a non-success status (502 when overloaded,
107 /// 403 when rate-limiting the caller's IP are the common ones).
108 #[cfg(feature = "fetch")]
109 #[error("crt.sh returned status {0}")]
110 BadStatus(reqwest::StatusCode),
111
112 /// The streamed response exceeded [`CT_RESPONSE_MAX_BYTES`]; buffering
113 /// was aborted rather than letting a hostile/runaway mirror OOM the
114 /// scanner.
115 #[cfg(feature = "fetch")]
116 #[error("crt.sh response exceeded {limit} byte cap (got {got}+ bytes) - refusing to buffer")]
117 ResponseTooLarge {
118 /// The configured byte cap.
119 limit: usize,
120 /// The size reached when the cap was tripped.
121 got: usize,
122 },
123}
124
125/// Clean an input domain by trimming whitespace, stripping leading wildcard labels
126/// (`*.` or `.`), and stripping trailing dots (`.`).
127#[inline]
128pub(crate) fn clean_domain(domain: &str) -> &str {
129 let mut s = domain.trim();
130 loop {
131 let prev = s;
132 if let Some(rest) = s.strip_prefix("*.") {
133 s = rest;
134 }
135 if let Some(rest) = s.strip_prefix('*') {
136 s = rest;
137 }
138 s = s.trim_matches('.');
139 if s == prev {
140 break;
141 }
142 }
143 s
144}
145
146/// Build the canonical crt.sh JSON query URL for `domain`.
147///
148/// Uses the SQL-`LIKE` wildcard prefix `%.` so the query returns every
149/// name certified *under* the domain, not just apex certs - the missing
150/// wildcard was the bug that made karyx's origin discovery blind to
151/// subdomains. The domain itself is percent-encoded so a hostile or
152/// malformed target (spaces, `&`, `#`) can't break out of the query
153/// component.
154///
155/// ```
156/// assert_eq!(
157/// ctlog::crtsh_query_url("Example.com"),
158/// "https://crt.sh/?q=%.Example.com&output=json",
159/// );
160/// ```
161pub fn crtsh_query_url(domain: &str) -> String {
162 format!(
163 "https://crt.sh/?q=%.{}&output=json",
164 urlencoding::encode(clean_domain(domain))
165 )
166}
167
168/// Build a crt.sh query URL that matches certificates for the apex `domain`.
169///
170/// This is the companion to [`crtsh_query_url`]; an apex-only certificate
171/// that never lists a wildcard or subdomain SAN is missed by the `%.` query,
172/// but its subdomains (if any) still appear here. The two result sets are
173/// unioned by the fetch layer.
174///
175/// ```
176/// assert_eq!(
177/// ctlog::crtsh_apex_query_url("Example.com"),
178/// "https://crt.sh/?q=Example.com&output=json",
179/// );
180/// ```
181pub fn crtsh_apex_query_url(domain: &str) -> String {
182 format!(
183 "https://crt.sh/?q={}&output=json",
184 urlencoding::encode(clean_domain(domain))
185 )
186}
187
188/// Parse a crt.sh `output=json` body into the full set of normalized
189/// host names, *including* the queried apex.
190///
191/// Applies the [normalization contract](crate#normalization-contract).
192/// Use this for origin discovery, where the apex resolving to a non-CDN
193/// address is itself a candidate worth probing.
194///
195/// ```
196/// let body = r#"[
197/// {"name_value": "*.api.example.com\nexample.com"},
198/// {"name_value": "WWW.example.com"}
199/// ]"#;
200/// let names = ctlog::parse_crtsh_hostnames(body).unwrap();
201/// assert_eq!(names, ["api.example.com", "example.com", "www.example.com"]);
202/// ```
203pub fn parse_crtsh_hostnames(body: &str) -> Result<Vec<String>, CtError> {
204 normalized_names(body)
205}
206
207/// Parse a crt.sh `output=json` body into normalized *subdomains*,
208/// excluding the queried `domain` itself.
209///
210/// Applies the [normalization contract](crate#normalization-contract)
211/// then drops the apex. Use this for subdomain enumeration, where the
212/// queried domain is already known and only newly discovered names
213/// matter.
214///
215/// ```
216/// let body = r#"[
217/// {"name_value": "api.example.com\n*.example.com"},
218/// {"name_value": "Example.COM"}
219/// ]"#;
220/// // The apex (in any case) and the apex-wildcard are both excluded.
221/// let subs = ctlog::parse_crtsh_subdomains(body, "example.com").unwrap();
222/// assert_eq!(subs, ["api.example.com"]);
223/// ```
224pub fn parse_crtsh_subdomains(body: &str, domain: &str) -> Result<Vec<String>, CtError> {
225 let cleaned = clean_domain(domain);
226 let domain_lower = cleaned.to_ascii_lowercase();
227 let mut names = normalized_names(body)?;
228 names.retain(|n| *n != domain_lower);
229 Ok(names)
230}
231
232/// The shared normalization core behind both public parse functions.
233fn normalized_names(body: &str) -> Result<Vec<String>, CtError> {
234 if body.trim().is_empty() {
235 return Ok(Vec::new());
236 }
237
238 let entries: Vec<CrtShEntry> = serde_json::from_str(body)?;
239
240 // Push directly into the result. The previous `flat_map` collected each
241 // entry's names into a throwaway `Vec<String>` before flattening (one heap
242 // Vec per CT-log entry, of which a busy domain has thousands) (Law 7); a
243 // plain nested loop keeps `e.name_value` alive across the inner iteration
244 // without that intermediate allocation, and fuses the filter into the walk.
245 let mut names: Vec<String> = Vec::new();
246 for e in entries {
247 for raw in e.name_value.split('\n') {
248 let trimmed = raw.trim();
249 // Strip any leading wildcard labels and surrounding dots.
250 let mut s = trimmed;
251 loop {
252 let prev = s;
253 if let Some(rest) = s.strip_prefix("*.") {
254 s = rest;
255 }
256 if let Some(rest) = s.strip_prefix('*') {
257 s = rest;
258 }
259 s = s.trim_matches('.');
260 if s == prev {
261 break;
262 }
263 }
264 let normalized = s.to_ascii_lowercase();
265
266 // Drop anything that can never be a DNS name: empties, residual
267 // wildcards, consecutive dots, whitespace, control characters, and
268 // invalid hostname symbols (URL schemes, paths, credentials, etc.).
269 if !normalized.is_empty()
270 && !normalized.contains('*')
271 && !normalized.contains("..")
272 && !normalized.chars().any(|c| {
273 c.is_whitespace()
274 || c.is_control()
275 || (!c.is_ascii_alphanumeric() && c != '.' && c != '-' && c != '_')
276 })
277 {
278 names.push(normalized);
279 }
280 }
281 }
282
283 names.sort();
284 names.dedup();
285 Ok(names)
286}
287
288#[cfg(feature = "fetch")]
289mod fetch_impl {
290 use super::{parse_crtsh_subdomains, CtError};
291 use serde::de::Error as _;
292 use std::time::Duration;
293
294 /// Timeout for outbound CT-log queries. crt.sh routinely takes
295 /// 10-20s and occasionally hangs entirely; without a cap a discovery
296 /// run would be a DoS-on-self for every blocked-up upstream.
297 pub const CT_QUERY_TIMEOUT: Duration = Duration::from_secs(30);
298
299 /// Hard cap on crt.sh response body size. A real CT-log JSON for a
300 /// busy domain is a few MB at most; an adversarial / misbehaving
301 /// mirror that streams multi-GB nonsense would otherwise OOM the
302 /// scanner before the JSON parser ever ran.
303 pub const CT_RESPONSE_MAX_BYTES: usize = 32 * 1024 * 1024; // 32 MiB
304
305 /// Maximum retry attempts for the fetch layer. crt.sh commonly returns
306 /// 502 when overloaded; a single transient failure should not kill a
307 /// discovery run.
308 pub const CT_MAX_RETRIES: u32 = 3;
309
310 /// Base backoff between retries. Grows linearly with the attempt.
311 pub const CT_RETRY_BACKOFF: Duration = Duration::from_millis(500);
312
313 /// Tunable options for the CT fetch layer.
314 ///
315 /// Callers that need a different endpoint, timeout (via the `reqwest`
316 /// client), or byte cap can override the defaults here.
317 #[derive(Debug, Clone)]
318 pub struct CtOptions {
319 /// Hard cap on the buffered crt.sh response body in bytes.
320 ///
321 /// Defaults to [`CT_RESPONSE_MAX_BYTES`] (32 MiB).
322 pub max_response_bytes: usize,
323 /// Base URL for crt.sh-style queries.
324 ///
325 /// Defaults to `https://crt.sh/` and must not include a trailing
326 /// slash. Override this in tests that drive a local mock server.
327 pub base_url: String,
328 }
329
330 // `CtOptions::default()` resolves through this trait impl (`Default` is
331 // in the prelude); there is deliberately no inherent `default()` method,
332 // which `clippy::should_implement_trait` flags as shadowing the trait.
333 impl Default for CtOptions {
334 fn default() -> Self {
335 Self {
336 max_response_bytes: CT_RESPONSE_MAX_BYTES,
337 base_url: String::from("https://crt.sh"),
338 }
339 }
340 }
341
342 /// Query crt.sh for `domain` over a fresh default client and return
343 /// the deduplicated subdomains.
344 ///
345 /// Convenience wrapper over [`discover_subdomains_ct_with`] for callers
346 /// that have no client to share; it builds one with
347 /// [`CT_QUERY_TIMEOUT`] applied.
348 ///
349 /// # Single-source risk
350 ///
351 /// The default endpoint is `https://crt.sh/`. If crt.sh is down or
352 /// rate-limits the caller, there is no built-in fallback aggregator
353 /// (certspotter, Censys, etc.). Use [`discover_subdomains_ct_with_options`]
354 /// and point [`CtOptions::base_url`] at an alternate crt.sh-compatible
355 /// mirror to distribute the dependency.
356 pub async fn discover_subdomains_ct(domain: &str) -> Result<Vec<String>, CtError> {
357 let client = reqwest::Client::builder()
358 .timeout(CT_QUERY_TIMEOUT)
359 .build()?;
360 discover_subdomains_ct_with(&client, domain).await
361 }
362
363 /// Query crt.sh for `domain` reusing the caller's `client` (and its
364 /// connection pool / proxy / UA config) and return the deduplicated
365 /// subdomains.
366 ///
367 /// Retries transient failures (5xx, 429, transport errors) up to
368 /// [`CT_MAX_RETRIES`] with a linear [`CT_RETRY_BACKOFF`]. The body is
369 /// read chunk-by-chunk and aborted past [`CT_RESPONSE_MAX_BYTES`]
370 /// rather than buffered whole, so a hostile mirror cannot OOM the
371 /// caller regardless of its advertised `Content-Length`.
372 ///
373 /// See [`discover_subdomains_ct`] for the single-source risk note.
374 pub async fn discover_subdomains_ct_with(
375 client: &reqwest::Client,
376 domain: &str,
377 ) -> Result<Vec<String>, CtError> {
378 discover_subdomains_ct_with_options(client, domain, CtOptions::default()).await
379 }
380
381 /// Query crt.sh for `domain` reusing the caller's `client` and tunable
382 /// [`CtOptions`].
383 ///
384 /// Behaves like [`discover_subdomains_ct_with`] but lets the caller
385 /// override the response size cap.
386 pub async fn discover_subdomains_ct_with_options(
387 client: &reqwest::Client,
388 domain: &str,
389 options: CtOptions,
390 ) -> Result<Vec<String>, CtError> {
391 let clean = super::clean_domain(domain);
392 if clean.is_empty() {
393 return Ok(Vec::new());
394 }
395
396 let base = options.base_url.trim_end_matches('/');
397 let encoded = urlencoding::encode(clean);
398 let wildcard_url = format!("{base}/?q=%.{encoded}&output=json");
399 let apex_url = format!("{base}/?q={encoded}&output=json");
400
401 let wildcard_res = discover_subdomains_ct_with_url(client, &wildcard_url, clean, &options).await;
402 let apex_res = discover_subdomains_ct_with_url(client, &apex_url, clean, &options).await;
403
404 match (wildcard_res, apex_res) {
405 (Ok(mut names), Ok(mut apex_names)) => {
406 names.append(&mut apex_names);
407 names.sort_unstable();
408 names.dedup();
409 Ok(names)
410 }
411 (Ok(names), Err(e)) => {
412 tracing::warn!(domain, error = %e, "apex query failed, returning wildcard results");
413 Ok(names)
414 }
415 (Err(e), Ok(apex_names)) => {
416 tracing::warn!(domain, error = %e, "wildcard query failed, returning apex results");
417 Ok(apex_names)
418 }
419 (Err(e1), Err(_)) => Err(e1),
420 }
421 }
422
423 /// Internal test seam: query an explicit `url` so `fetch` tests can
424 /// drive a local `wiremock` server rather than the real `crt.sh` endpoint.
425 pub(crate) async fn discover_subdomains_ct_with_url(
426 client: &reqwest::Client,
427 url: &str,
428 domain: &str,
429 options: &CtOptions,
430 ) -> Result<Vec<String>, CtError> {
431 tracing::debug!(domain, "querying crt.sh for CT logs");
432
433 let mut last_err: Option<CtError> = None;
434 for attempt in 0..CT_MAX_RETRIES {
435 if attempt > 0 {
436 let delay = CT_RETRY_BACKOFF * attempt;
437 tracing::warn!(attempt, domain, "crt.sh query retry after {delay:?}");
438 tokio::time::sleep(delay).await;
439 }
440
441 match try_query(client, url, domain, options).await {
442 Ok(names) => return Ok(names),
443 Err(e) if is_retryable(&e) => {
444 tracing::warn!(attempt, error = %e, "crt.sh query failed; will retry if attempts remain");
445 last_err = Some(e);
446 }
447 Err(e) => return Err(e),
448 }
449 }
450
451 Err(last_err.unwrap_or_else(|| CtError::BadStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE)))
452 }
453
454 async fn try_query(
455 client: &reqwest::Client,
456 url: &str,
457 domain: &str,
458 options: &CtOptions,
459 ) -> Result<Vec<String>, CtError> {
460 let max_response_bytes = options.max_response_bytes;
461 let mut res = client.get(url).send().await?;
462 if !res.status().is_success() {
463 return Err(CtError::BadStatus(res.status()));
464 }
465
466 let mut body = Vec::with_capacity(64 * 1024);
467 while let Some(chunk) = res.chunk().await? {
468 if body.len() + chunk.len() > max_response_bytes {
469 return Err(CtError::ResponseTooLarge {
470 limit: max_response_bytes,
471 got: body.len() + chunk.len(),
472 });
473 }
474 body.extend_from_slice(&chunk);
475 }
476
477 // crt.sh emits UTF-8 JSON. A stray invalid byte is not silently
478 // lossy-replaced (which could create bogus hostnames with U+FFFD);
479 // instead it becomes a parse error so the caller knows the response
480 // was corrupted.
481 let text = String::from_utf8(body).map_err(|e| {
482 CtError::Parse(serde_json::Error::custom(format!(
483 "CT response contained invalid UTF-8: {e}"
484 )))
485 })?;
486
487 // crt.sh and upstream proxies (Cloudflare, nginx) frequently emit 200 OK
488 // status headers with HTML, JSON error objects, or plain text error pages
489 // (e.g. Gateway Time-out or {"error": "..."}).
490 // Catch non-array responses early and surface as SERVICE_UNAVAILABLE so the
491 // caller's retry loop is triggered rather than failing immediately on JSON parse.
492 let trimmed_text = text.trim_start();
493 if !trimmed_text.starts_with('[') {
494 return Err(CtError::BadStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE));
495 }
496
497 let names = parse_crtsh_subdomains(&text, domain)?;
498 tracing::debug!(found = names.len(), "discovered subdomains via CT logs");
499 Ok(names)
500 }
501
502 pub(crate) fn is_retryable(err: &CtError) -> bool {
503 match err {
504 CtError::Transport(_) => true,
505 CtError::BadStatus(status) => {
506 status.is_server_error()
507 || *status == reqwest::StatusCode::TOO_MANY_REQUESTS
508 || *status == reqwest::StatusCode::REQUEST_TIMEOUT
509 }
510 _ => false,
511 }
512 }
513}
514
515#[cfg(feature = "fetch")]
516pub use fetch_impl::{
517 discover_subdomains_ct, discover_subdomains_ct_with, discover_subdomains_ct_with_options,
518 CtOptions, CT_MAX_RETRIES, CT_QUERY_TIMEOUT, CT_RESPONSE_MAX_BYTES, CT_RETRY_BACKOFF,
519};
520// Only the feature-gated `fetch_tests` module uses this, so it must be gated on
521// `feature = "fetch"` too - otherwise `cargo test` (default features) fails to
522// compile with `unresolved import fetch_impl` (the module only exists under the
523// `fetch` feature).
524#[cfg(all(test, feature = "fetch"))]
525pub(crate) use fetch_impl::discover_subdomains_ct_with_url;
526
527#[cfg(test)]
528mod tests;