keyhog_core/

source.rs

//! Source trait and chunk types: the abstraction for pluggable input backends.

// Debt bucket: 9 items predating the crate floor raising `missing_docs` to
// `warn`. Remove this allow once every Source-trait item is documented.
#![allow(missing_docs)]

use crate::SensitiveString;
use serde::{Deserialize, Serialize};
use std::sync::{Arc, LazyLock};
use thiserror::Error;
/// Canonical default window size (1 MiB) for streaming source chunks.
pub const DEFAULT_WINDOW_SIZE_BYTES: usize = 1024 * 1024;

/// Canonical default window overlap (128 KiB) between adjacent streaming source windows.
///
/// Single canonical owner shared across scanner seam reassembly, filesystem chunking,
/// stdin streaming, archive member extraction, and benchmark harnesses.
///
/// 128 KiB covers PEM-encoded RSA-8192 keys, large JWTs, and multi-line concatenated
/// secrets with generous margin while bounding per-window re-scan overhead.
pub const DEFAULT_WINDOW_OVERLAP_BYTES: usize = 128 * 1024;

/// Machine-readable reason a requested source surface was not fully scanned.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SourceCoverageGapKind {
    /// The source denied access, did not exist, or returned an unreadable response.
    Inaccessible,
    /// A configured request, item, or byte limit stopped the scan early.
    Truncated,
}

impl std::fmt::Display for SourceCoverageGapKind {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter.write_str(match self {
            Self::Inaccessible => "inaccessible",
            Self::Truncated => "truncated",
        })
    }
}

/// A scannable chunk of text with metadata about where it came from.
///
/// # Examples
///
/// ```rust
/// use keyhog_core::{Chunk, ChunkMetadata};
///
/// let chunk = Chunk {
///     data: "API_KEY=sk_live_example".into(),
///     metadata: ChunkMetadata {
///         source_type: "filesystem".into(),
///         path: Some("app.env".into()),
///         ..Default::default()
///     },
/// };
///
/// assert_eq!(chunk.metadata.path.as_deref(), Some("app.env"));
/// ```
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Chunk {
    /// UTF-8 text content to scan.
    pub data: SensitiveString,
    /// Provenance details used in findings and reporters.
    pub metadata: ChunkMetadata,
}

impl From<String> for Chunk {
    fn from(data: String) -> Self {
        Self {
            data: data.into(),
            metadata: ChunkMetadata::default(),
        }
    }
}

impl From<&str> for Chunk {
    fn from(data: &str) -> Self {
        Self::from(data.to_string())
    }
}

/// Metadata that tracks the source location for a scanned chunk.
///
/// # Examples
///
/// ```rust
/// use keyhog_core::ChunkMetadata;
///
/// let metadata = ChunkMetadata {
///     source_type: "git-diff".into(),
///     path: Some("src/lib.rs".into()),
///     commit: Some("abc123".into()),
///     author: Some("Dev".into()),
///     date: Some("2026-03-26T00:00:00Z".into()),
///     ..Default::default()
/// };
///
/// assert_eq!(&*metadata.source_type, "git-diff");
/// ```
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ChunkMetadata {
    /// `Arc<str>` (not `String`) so cloning a chunk's metadata, done per decode
    /// sub-chunk, where every sub-chunk of a file shares the same `source_type`
    /// and `path`: is a refcount bump, not a fresh heap allocation + copy of
    /// each string. Mirrors the `Arc<str>` convention already used by
    /// `MatchLocation` in `finding.rs`; serialized through the same
    /// `serde_arc_str` helpers so no `serde` `rc` feature is needed.
    #[serde(with = "crate::finding::serde_arc_str")]
    pub source_type: Arc<str>,
    #[serde(with = "crate::finding::serde_arc_str_opt")]
    pub path: Option<Arc<str>>,
    #[serde(with = "crate::finding::serde_arc_str_opt")]
    pub commit: Option<Arc<str>>,
    #[serde(with = "crate::finding::serde_arc_str_opt")]
    pub author: Option<Arc<str>>,
    #[serde(with = "crate::finding::serde_arc_str_opt")]
    pub date: Option<Arc<str>>,
    pub base_offset: usize,
    /// Number of lines that precede `base_offset` in the original file -
    /// the line-number analog of `base_offset`. Zero for whole-file chunks
    /// (single-pass mmap, stdin, http, git diffs). Non-zero only when a
    /// source slices one file into multiple chunks (the filesystem
    /// `>window_size` windowed path), where each window after the first
    /// starts partway through the file. The scanner computes a match's
    /// line number *within the chunk text* and adds this base so the
    /// reported line is the absolute file line, not the per-window one -
    /// exactly mirroring how `base_offset` makes the byte offset absolute.
    /// Without it, a secret on line 584307 of a 70 MiB file was reported
    /// at the window-local line (e.g. line 2), making findings impossible
    /// to locate.
    #[serde(default)]
    pub base_line: usize,
    /// File mtime in nanoseconds since UNIX epoch, when the source can
    /// surface it cheaply (filesystem walks). Optional because non-fs
    /// sources (stdin, http, git diffs) don't have a meaningful mtime.
    /// Populated to drive the merkle-index metadata fast-path.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub mtime_ns: Option<u64>,
    /// File size in bytes, when known cheaply at chunk-production time.
    /// Same shape and rationale as `mtime_ns`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub size_bytes: Option<u64>,
    /// Inode change time in nanoseconds since UNIX epoch, when the platform
    /// exposes one (unix `st_ctime`). Unlike `mtime_ns` this cannot be forged
    /// by userspace (`utimensat` / `set_times`), so the incremental index
    /// requires it to agree before trusting its read-free fast-path skip.
    /// `None` on platforms without a change time, which disables that skip.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ctime_ns: Option<u64>,
    /// For DECODE sub-chunks only: the `[start, end)` byte range of the freshly
    /// decoded text within `data`. A decode sub-chunk is a small window of
    /// already-scanned parent context with the decoded blob spliced in at this
    /// span; everything OUTSIDE the span was scanned (and any finding deduped)
    /// when the parent chunk was scanned, so the self-contained passes only need
    /// to rescan a focus window around this span instead of the whole splice.
    /// `None` for all non-decode chunks (whole-file, windowed, git-diff, …).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub decoded_span: Option<(usize, usize)>,
}

macro_rules! define_intern_table {
    (
        $(#[$intern_meta:meta])*
        pub fn $fn_intern:ident,
        $(#[$list_meta:meta])*
        pub fn $fn_list:ident,
        $(
            $(#[$item_meta:meta])*
            ($name:ident, $str_val:literal)
        ),* $(,)?
    ) => {
        $(
            $(#[$item_meta])*
            #[doc = concat!("Canonical `source_type` for `", $str_val, "` chunks.")]
            pub static $name: LazyLock<Arc<str>> = LazyLock::new(|| Arc::from($str_val));
        )*

        $(#[$list_meta])*
        pub fn $fn_list() -> &'static [&'static str] {
            &[
                $( $str_val ),*
            ]
        }

        $(#[$intern_meta])*
        pub fn $fn_intern(val: &str) -> Arc<str> {
            match val {
                $(
                    $str_val => Arc::clone(&$name),
                )*
                other => Arc::from(other),
            }
        }
    };
}

define_intern_table! {
    /// Intern a source type string reference into an `Arc<str>`.
    ///
    /// Returns a clone of a pre-allocated static `Arc<str>` for canonical source types,
    /// avoiding heap allocation and string duplication.
    pub fn intern_source_type,
    /// Pre-interned common source type names.
    pub fn common_source_types,
    (SOURCE_TYPE_AZURE_BLOB, "azure_blob"),
    (SOURCE_TYPE_BINARY, "binary"),
    (SOURCE_TYPE_BINARY_GHIDRA_DECOMPILED, "binary:ghidra:decompiled"),
    (SOURCE_TYPE_BINARY_GHIDRA_STRINGS, "binary:ghidra:strings"),
    (SOURCE_TYPE_BINARY_STRINGS, "binary:strings"),
    (SOURCE_TYPE_DOCKER, "docker"),
    (SOURCE_TYPE_FILESYSTEM, "filesystem"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE, "filesystem/archive"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_BINARY, "filesystem/archive-binary"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_BINARY_TEX_ORPHANED, "filesystem/archive-binary/tex-orphaned"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_BINARY_TEX_REFERENCED, "filesystem/archive-binary/tex-referenced"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_BINARY_TEX_ROOT, "filesystem/archive-binary/tex-root"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_ANDROID, "filesystem/archive/android"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_ANDROID_RESOURCE, "filesystem/archive/android-resource"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_ANDROID_XML, "filesystem/archive/android-xml"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_TEX_COMMENT_ORPHANED, "filesystem/archive/tex-comment/orphaned"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_TEX_COMMENT_REFERENCED, "filesystem/archive/tex-comment/referenced"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_TEX_COMMENT_ROOT, "filesystem/archive/tex-comment/root"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_TEX_ORPHANED, "filesystem/archive/tex-orphaned"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_TEX_REFERENCED, "filesystem/archive/tex-referenced"),
    (SOURCE_TYPE_FILESYSTEM_ARCHIVE_TEX_ROOT, "filesystem/archive/tex-root"),
    (SOURCE_TYPE_FILESYSTEM_COMPRESSED, "filesystem/compressed"),
    (SOURCE_TYPE_FILESYSTEM_COMPRESSED_BINARY, "filesystem/compressed-binary"),
    (SOURCE_TYPE_FILESYSTEM_PDF, "filesystem/pdf"),
    (SOURCE_TYPE_FILESYSTEM_WINDOWED, "filesystem/windowed"),
    (SOURCE_TYPE_FILESYSTEM_BINARY_STRINGS, "filesystem:binary-strings"),
    (SOURCE_TYPE_GCS, "gcs"),
    (SOURCE_TYPE_GIT, "git"),
    (SOURCE_TYPE_GIT_DIFF, "git-diff"),
    (SOURCE_TYPE_GIT_HISTORY, "git-history"),
    (SOURCE_TYPE_GIT_STAGED, "git-staged"),
    (SOURCE_TYPE_GIT_DIFF_SLASH, "git/diff"),
    (SOURCE_TYPE_GIT_HEAD, "git/head"),
    (SOURCE_TYPE_GIT_HISTORY_SLASH, "git/history"),
    (SOURCE_TYPE_GIT_STAGED_SLASH, "git/staged"),
    (SOURCE_TYPE_GIT_TAG, "git/tag"),
    (SOURCE_TYPE_GIT_UNREACHABLE, "git/unreachable"),
    (SOURCE_TYPE_GITHUB, "github"),
    (SOURCE_TYPE_S3, "s3"),
    (SOURCE_TYPE_SLACK, "slack"),
    (SOURCE_TYPE_STDIN, "stdin"),
    (SOURCE_TYPE_WEB, "web"),
    (SOURCE_TYPE_WEB_JS, "web:js"),
    (SOURCE_TYPE_WEB_SOURCEMAP, "web:sourcemap"),
    (SOURCE_TYPE_WEB_SOURCEMAP_RAW, "web:sourcemap:raw"),
    (SOURCE_TYPE_WEB_WASM, "web:wasm"),
    (SOURCE_TYPE_WIRE_HAR_REQUEST, "wire:har:request"),
    (SOURCE_TYPE_WIRE_HAR_RESPONSE, "wire:har:response"),
}

/// Produces chunks of text for the scanner to process.
/// Each implementation handles a different input source.
///
/// # Examples
///
/// ```rust
/// use keyhog_core::{Chunk, ChunkMetadata, Source, SourceError};
///
/// struct StaticSource;
///
/// impl Source for StaticSource {
///     fn name(&self) -> &str {
///         "static"
///     }
///
///     fn chunks(&self) -> Box<dyn Iterator<Item = Result<Chunk, SourceError>> + '_> {
///         Box::new(std::iter::once(Ok(Chunk {
///             data: "TOKEN=value".into(),
///             metadata: ChunkMetadata {
///                 source_type: "static".into(),
///                 ..Default::default()
///             },
///         })))
///     }
///
///     fn as_any(&self) -> &dyn std::any::Any {
///         self
///     }
/// }
///
/// let source = StaticSource;
/// assert_eq!(source.name(), "static");
/// ```
pub trait Source: Send + Sync {
    /// Human-readable source name used in warnings and telemetry.
    fn name(&self) -> &str;
    /// Yield all readable chunks from this source.
    fn chunks(&self) -> Box<dyn Iterator<Item = Result<Chunk, SourceError>> + '_>;
    /// Support downcasting to concrete types.
    fn as_any(&self) -> &dyn std::any::Any;

    /// Whether all chunks for one exact `(source_type, path)` identity are
    /// emitted contiguously. Dispatch may use this to split unrelated routing
    /// classes without cutting a future cross-chunk dependency.
    fn chunk_identities_are_contiguous(&self) -> bool {
        false
    }
}

/// Errors returned by input sources while enumerating or reading content.
///
/// # Examples
///
/// ```rust
/// use keyhog_core::SourceError;
///
/// let error = SourceError::Other("pass a readable file or directory".into());
/// assert!(error.to_string().contains("Fix"));
/// ```
#[derive(Debug, Error)]
pub enum SourceError {
    #[error(
        "failed to read source: {0}. Fix: check the path exists, is readable, and is not a broken symlink"
    )]
    Io(#[from] std::io::Error),
    #[error(
        "failed to access git source: {0}. Fix: run inside a valid git repository and verify the requested refs exist"
    )]
    Git(String),
    #[error(
        "source coverage gap ({kind}) in {adapter} surface {surface} at {target}: {detail}. Fix: grant read access or raise the relevant source limit, then rerun the affected surface"
    )]
    Coverage {
        /// Stable source adapter name.
        adapter: String,
        /// Independently selected surface that was incomplete.
        surface: String,
        /// Credential-free target identity.
        target: String,
        /// Typed coverage classification.
        kind: SourceCoverageGapKind,
        /// Response-free operator guidance.
        detail: String,
    },
    #[error("unknown source '{name}'. Fix: use a source name listed by `keyhog scan --help`")]
    UnknownSource {
        /// Unrecognized source identifier supplied by the caller.
        name: String,
    },
    #[error(
        "source '{source_name}' is unavailable because this KeyHog artifact was built without the '{feature}' feature. Fix: install an artifact that includes '{feature}' or choose an enabled source"
    )]
    FeatureUnavailable {
        /// Canonical source identifier.
        source_name: String,
        /// Cargo feature required to construct the source.
        feature: String,
    },
    #[error(
        "invalid configuration for source '{source_name}': {detail}. Fix: use the parameter format documented by `keyhog scan --help`"
    )]
    InvalidConfiguration {
        /// Canonical source identifier.
        source_name: String,
        /// Credential-free explanation of the invalid input shape.
        detail: String,
    },
    #[error(
        "source name '{name}' is no longer accepted; use '{replacement}'. Fix: update the source identifier and rerun the scan"
    )]
    DeprecatedSourceName {
        /// Retired source identifier.
        name: String,
        /// Canonical replacement identifier.
        replacement: String,
    },
    #[error(
        "failed to read source: {0}. Fix: adjust the source settings or input so KeyHog can read plain text safely"
    )]
    Other(String),
}