scryer-engine 0.3.0

//! The health of a project's index: how much is in it and what could not be indexed. Built
//! from SQL counts and the statistics each indexing run records, and rendered the same way by
//! `get_indexing_status` and `scryer doctor`.

use std::collections::BTreeMap;

use schemars::JsonSchema;
use scryer_db::ScryerDb;
use serde::{Deserialize, Serialize};

use crate::stats::IndexStats;

/// What is in a project's index, and what could not be put there.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct IndexHealth {
    pub file_count: usize,
    pub symbol_count: usize,
    #[serde(default)]
    pub reference_count: usize,
    #[serde(default)]
    pub edge_count: usize,
    /// Indexed files by language.
    #[serde(default)]
    pub files_by_language: BTreeMap<String, usize>,
    /// References by how they were resolved: exact, name or macro.
    #[serde(default)]
    pub references_by_via: BTreeMap<String, usize>,
    /// When the index last changed; absent if the project was indexed before this was recorded.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub last_indexed: Option<String>,
    /// Recorded by the last indexing run; absent for an index built before this was recorded
    /// (run `scryer index` again).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub stats: Option<IndexStats>,
}

/// One `COUNT(*)` over a table's rows for a project (project 0 holds Cargo dependencies).
async fn count(db: &ScryerDb, table: &'static str, project_id: u64) -> anyhow::Result<usize> {
    let mut guard = db.lock().await;
    let rows = toasty::sql::query(format!(
        "SELECT COUNT(*) FROM {table} WHERE project_id = ?1"
    ))
    .bind(project_id as i64)
    .exec(&mut *guard)
    .await?;
    Ok(rows
        .first()
        .and_then(first_field)
        .map_or(0, |n| usize::try_from(n).unwrap_or(0)))
}

/// `SELECT <key>, COUNT(*) ... GROUP BY <key>` as a map.
async fn count_by(
    db: &ScryerDb,
    table: &'static str,
    column: &'static str,
    project_id: u64,
) -> anyhow::Result<BTreeMap<String, usize>> {
    let mut guard = db.lock().await;
    let rows = toasty::sql::query(format!(
        "SELECT {column}, COUNT(*) FROM {table} WHERE project_id = ?1 GROUP BY {column}"
    ))
    .bind(project_id as i64)
    .exec(&mut *guard)
    .await?;
    let mut out = BTreeMap::new();
    for row in rows {
        let toasty::stmt::Value::Record(record) = row else {
            continue;
        };
        if let [key, n] = record.fields.as_slice()
            && let Some(key) = key.as_str()
            && let Some(n) = value_i64(n)
        {
            out.insert(key.to_string(), usize::try_from(n).unwrap_or(0));
        }
    }
    Ok(out)
}

fn value_i64(value: &toasty::stmt::Value) -> Option<i64> {
    match value {
        toasty::stmt::Value::I64(n) => Some(*n),
        toasty::stmt::Value::U64(n) => i64::try_from(*n).ok(),
        toasty::stmt::Value::I32(n) => Some(i64::from(*n)),
        _ => None,
    }
}

fn first_field(row: &toasty::stmt::Value) -> Option<i64> {
    match row {
        toasty::stmt::Value::Record(record) => record.fields.first().and_then(value_i64),
        other => value_i64(other),
    }
}

/// The statistics recorded by the last indexing run that changed `project_id`, with the time.
pub(crate) async fn load_index_stats(
    db: &ScryerDb,
    project_id: u64,
) -> anyhow::Result<Option<IndexStats>> {
    Ok(load_index_stats_row(db, project_id).await?.map(|(s, _)| s))
}

async fn load_index_stats_row(
    db: &ScryerDb,
    project_id: u64,
) -> anyhow::Result<Option<(IndexStats, String)>> {
    let mut guard = db.lock().await;
    let row =
        scryer_db::IndexStats::filter(scryer_db::IndexStats::fields().project_id().eq(project_id))
            .first()
            .exec(&mut *guard)
            .await?;
    Ok(row.and_then(|r| {
        serde_json::from_str(&r.stats_json)
            .ok()
            .map(|s| (s, r.updated_at))
    }))
}

/// Gather the health of `project_id`'s index. Counts are computed by the database; no rows
/// are loaded.
pub async fn index_health(db: &ScryerDb, project_id: u64) -> anyhow::Result<IndexHealth> {
    let stats = load_index_stats_row(db, project_id).await?;
    Ok(IndexHealth {
        file_count: count(db, "source_file", project_id).await?,
        symbol_count: count(db, "symbol", project_id).await?,
        reference_count: count(db, "symbol_reference", project_id).await?,
        edge_count: count(db, "code_graph_edge", project_id).await?,
        files_by_language: count_by(db, "source_file", "language", project_id).await?,
        references_by_via: count_by(db, "symbol_reference", "via", project_id).await?,
        last_indexed: stats.as_ref().map(|(_, at)| at.clone()),
        stats: stats.map(|(s, _)| s),
    })
}

impl IndexHealth {
    /// What is wrong with the index, one sentence each. Empty when nothing is.
    pub fn problems(&self) -> Vec<String> {
        let mut out = Vec::new();
        let Some(stats) = &self.stats else {
            if self.file_count > 0 {
                out.push(
                    "No health data was recorded for this index (it predates it); run `scryer index` to record it".to_string(),
                );
            }
            return out;
        };
        if stats.parse_failures.count > 0 {
            let first = stats
                .parse_failures
                .first
                .as_ref()
                .map(|f| {
                    format!(
                        " (first: {}: {})",
                        f.path,
                        f.error.lines().next().unwrap_or("")
                    )
                })
                .unwrap_or_default();
            out.push(format!(
                "{} file(s) failed to parse and are not indexed{first}",
                stats.parse_failures.count
            ));
        }
        if !stats.syntax_error_files.is_empty() {
            out.push(format!(
                "{} file(s) parsed with syntax errors (the code may be broken, or use syntax or macros the parser does not support), so code around the errors may be missing from the index (first: {})",
                stats.syntax_error_files.len(),
                stats.syntax_error_files[0]
            ));
        }
        if stats.skipped.too_large > 0 {
            out.push(format!(
                "{} file(s) over the {} MiB size cap are not indexed",
                stats.skipped.too_large,
                crate::scanner::MAX_SOURCE_FILE_BYTES / (1024 * 1024)
            ));
        }
        if stats.skipped.unreadable > 0 {
            out.push(format!(
                "{} file(s) could not be read and are not indexed",
                stats.skipped.unreadable
            ));
        }
        if stats.dependencies.is_problem() {
            out.push(format!("Dependencies: {}", stats.dependencies.describe()));
        }
        out
    }

    /// A human-readable report, as printed by `scryer doctor`.
    pub fn render(&self) -> String {
        use std::fmt::Write;
        let join = |map: &BTreeMap<String, usize>| {
            if map.is_empty() {
                "none".to_string()
            } else {
                map.iter()
                    .map(|(k, v)| format!("{k} {v}"))
                    .collect::<Vec<_>>()
                    .join(" | ")
            }
        };
        let mut out = String::new();
        let _ = writeln!(
            out,
            "Index:        {} files | {} symbols | {} references | {} call and reexport edges",
            self.file_count, self.symbol_count, self.reference_count, self.edge_count
        );
        let _ = writeln!(out, "Languages:    {}", join(&self.files_by_language));
        let _ = writeln!(out, "References:   {}", join(&self.references_by_via));
        if let Some(at) = &self.last_indexed {
            let _ = writeln!(out, "Last indexed: {at}");
        }
        if let Some(stats) = &self.stats {
            let _ = writeln!(
                out,
                "Unresolved:   {} references name nothing in the project or its dependencies (standard library and other external code, or unknown names)",
                stats.unresolved_references
            );
            let _ = writeln!(out, "Dependencies: {}", stats.dependencies.describe());
        }
        let problems = self.problems();
        if problems.is_empty() {
            let _ = writeln!(out, "Problems:     none");
        } else {
            let _ = writeln!(out, "Problems:");
            for problem in problems {
                let _ = writeln!(out, "  - {problem}");
            }
        }
        out
    }
}