use std::collections::HashMap;
use std::io::Write;
use rusqlite::params;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use super::*;
use crate::classify::tiers::llm::LlmClassifier;
use crate::classify::tiers::weighted_sum::WeightedSumConfig;
use crate::core::config::{
ClassificationConfig, Config, LlmFallbackScope, RepoMapConfig, RepoMapMode,
};
use crate::core::db::Database;
const RULES: &str = "extend_defaults: false
rules:
- id: sec
category: security
keywords: [\"security:\"]
priority: 120
confidence: 0.8
- id: flaky
category: qa
keywords: [\"flaky:\"]
priority: 110
confidence: 0.79
- id: feat
category: new_feature
keywords: [\"feat:\"]
priority: 100
confidence: 0.95
categories:
- name: internal_tooling
- name: platform_infrastructure
- name: bug_fix
";
const MAPPED: &str = "acme-tools";
const UNMAPPED: &str = "acme-billing";
const MONO: &str = "acme-mono";
const NOTHING: &str = "zzz qqq vvv";
fn rules_file() -> tempfile::NamedTempFile {
let mut f = tempfile::Builder::new()
.suffix(".yaml")
.tempfile()
.expect("tempfile");
f.write_all(RULES.as_bytes()).expect("write rules");
f
}
fn config(rules: &std::path::Path, map: &[(&str, &str)], repo_map: RepoMapConfig) -> Config {
Config {
classification: Some(ClassificationConfig {
rules_files: vec![rules.to_path_buf()],
repo_categories: map
.iter()
.map(|(r, c)| (r.to_string(), c.to_string()))
.collect::<HashMap<_, _>>(),
repo_map,
llm_fallback_scope: LlmFallbackScope::LowConfidence,
weighted_sum: WeightedSumConfig {
enabled: false,
..WeightedSumConfig::default()
},
..ClassificationConfig::default()
}),
..Config::default()
}
}
fn floor() -> RepoMapConfig {
RepoMapConfig {
mode: RepoMapMode::Floor,
..RepoMapConfig::default()
}
}
fn insert(db: &Database, sha: &str, repo: &str, message: &str) {
db.connection()
.execute(
"INSERT INTO commits \
(sha, author_name, author_email, timestamp, message, repository, is_merge) \
VALUES (?1, 'a', 'a@x', '2024-01-01T00:00:00Z', ?2, ?3, 0)",
params![sha, message, repo],
)
.expect("insert commit");
}
fn insert_with_paths(db: &Database, sha: &str, repo: &str, paths: &[&str]) {
insert(db, sha, repo, NOTHING);
let id = db.connection().last_insert_rowid();
for p in paths {
db.connection()
.execute(
"INSERT INTO files (commit_id, path, change_type) VALUES (?1, ?2, 'M')",
params![id, p],
)
.expect("insert file");
}
}
fn verdict(db: &Database, sha: &str) -> (String, String, f64) {
db.connection()
.query_row(
"SELECT cl.category, cl.method, cl.confidence FROM commits c \
JOIN classifications cl ON cl.id = c.classification_id WHERE c.sha = ?1",
[sha],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.expect("classified")
}
fn cat_method(db: &Database, sha: &str) -> (String, String) {
let (c, m, _) = verdict(db, sha);
(c, m)
}
fn pair(category: &str, method: &str) -> (String, String) {
(category.to_string(), method.to_string())
}
async fn run(pipeline: &ClassificationPipeline, db: &mut Database) -> Result<ClassificationStats> {
let engine = pipeline.build_rule_engine()?;
pipeline.run_with_engine(db, engine).await
}
async fn mock_llm(category: &str, confidence: f64) -> MockServer {
let content = serde_json::json!({
"category": category,
"subcategory": null,
"confidence": confidence,
"complexity": 2
})
.to_string();
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"choices": [{"message": {"content": content}}],
"usage": {"prompt_tokens": 120, "completion_tokens": 9}
})))
.mount(&server)
.await;
server
}
async fn run_with_llm(
pipeline: &ClassificationPipeline,
db: &mut Database,
server: &MockServer,
) -> ClassificationStats {
let mut engine = pipeline.build_rule_engine().expect("rule engine");
engine.attach_llm(
LlmClassifier::new("test-model", Some("sk-test".to_string()))
.with_endpoint(format!("{}/v1/chat/completions", server.uri())),
);
pipeline.run_with_engine(db, engine).await.expect("run")
}
#[tokio::test]
async fn floor_keeps_an_exception_at_the_threshold_and_floors_it_below() {
let rules = rules_file();
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
floor(),
));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-sec", MAPPED, "security: rotate the signing keys");
insert(&db, "sha-qa", MAPPED, "flaky: retry the login test");
insert(
&db,
"sha-qa-unmapped",
UNMAPPED,
"flaky: retry the login test",
);
run(&pipeline, &mut db).await.expect("run");
let (category, method, confidence) = verdict(&db, "sha-sec");
assert_eq!(pair(&category, &method), pair("security", "exact_rule"));
assert!((confidence - 0.8).abs() < 1e-9, "{confidence}");
let (category, method, confidence) = verdict(&db, "sha-qa");
assert_eq!(
pair(&category, &method),
pair("internal_tooling", "repo_map")
);
assert!((confidence - 1.0).abs() < 1e-9, "{confidence}");
assert_eq!(cat_method(&db, "sha-qa-unmapped"), pair("qa", "exact_rule"));
}
#[tokio::test]
async fn floor_overrides_a_non_exception_rule_and_llm_verdict() {
let rules = rules_file();
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
floor(),
));
let server = mock_llm("new_feature", 0.95).await;
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-feat", MAPPED, "feat: add the widget");
insert(&db, "sha-none", MAPPED, NOTHING);
let stats = run_with_llm(&pipeline, &mut db, &server).await;
assert_eq!(server.received_requests().await.expect("rec").len(), 1);
assert_eq!(stats.llm_usage.calls, 1);
for sha in ["sha-feat", "sha-none"] {
assert_eq!(
cat_method(&db, sha),
pair("internal_tooling", "repo_map"),
"{sha}"
);
}
}
#[tokio::test]
async fn floor_keeps_a_high_confidence_llm_exception() {
let rules = rules_file();
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
floor(),
));
let server = mock_llm("bug_fix", 0.9).await;
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-none", MAPPED, NOTHING);
run_with_llm(&pipeline, &mut db, &server).await;
assert_eq!(cat_method(&db, "sha-none"), pair("bug_fix", "llm_fallback"));
}
#[tokio::test]
async fn exceptions_and_threshold_are_config_values() {
let rules = rules_file();
let repo_map = RepoMapConfig {
mode: RepoMapMode::Floor,
exceptions: Some(vec!["new_feature".into(), "security".into()]),
min_confidence: 0.85,
};
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
repo_map,
));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-feat", MAPPED, "feat: add the widget");
insert(&db, "sha-sec", MAPPED, "security: rotate the signing keys");
insert(&db, "sha-qa", MAPPED, "flaky: retry the login test");
run(&pipeline, &mut db).await.expect("run");
assert_eq!(
cat_method(&db, "sha-feat"),
pair("new_feature", "exact_rule")
);
for sha in ["sha-sec", "sha-qa"] {
assert_eq!(
cat_method(&db, sha),
pair("internal_tooling", "repo_map"),
"{sha}"
);
}
}
#[tokio::test]
async fn an_absent_repo_map_block_leaves_output_unchanged() {
let rules = rules_file();
let seed = |db: &Database| {
insert(db, "sha-sec", MAPPED, "security: rotate the signing keys");
insert(db, "sha-none", MAPPED, NOTHING);
insert(
db,
"sha-unmapped",
UNMAPPED,
"security: rotate the signing keys",
);
};
let rows = |db: &Database| -> Vec<(String, String, f64)> {
["sha-sec", "sha-none", "sha-unmapped"]
.iter()
.map(|s| verdict(db, s))
.collect()
};
let map = [(MAPPED, "internal_tooling")];
let mut absent_cfg = config(rules.path(), &map, RepoMapConfig::default());
if let Some(c) = absent_cfg.classification.as_mut() {
c.repo_map = serde_yaml::from_str::<ClassificationConfig>("{}")
.expect("empty section")
.repo_map;
}
let mut absent = Database::open_in_memory().expect("db");
seed(&absent);
run(&ClassificationPipeline::new(absent_cfg), &mut absent)
.await
.expect("run");
let explicit_cfg = config(
rules.path(),
&map,
RepoMapConfig {
mode: RepoMapMode::Override,
..RepoMapConfig::default()
},
);
let mut explicit = Database::open_in_memory().expect("db");
seed(&explicit);
run(&ClassificationPipeline::new(explicit_cfg), &mut explicit)
.await
.expect("run");
assert_eq!(rows(&absent), rows(&explicit));
assert_eq!(
cat_method(&absent, "sha-sec"),
pair("internal_tooling", "repo_map")
);
assert_eq!(
cat_method(&absent, "sha-none"),
pair("internal_tooling", "repo_map")
);
assert_eq!(
cat_method(&absent, "sha-unmapped"),
pair("security", "exact_rule")
);
}
#[tokio::test]
async fn the_longest_matching_prefix_wins() {
let rules = rules_file();
let map = [
(MONO, "internal_tooling"),
("acme-mono:services", "platform_infrastructure"),
("acme-mono:services/qa-harness/", "qa"),
];
let pipeline =
ClassificationPipeline::new(config(rules.path(), &map, RepoMapConfig::default()));
let mut db = Database::open_in_memory().expect("db");
insert_with_paths(&db, "sha-qa", MONO, &["services/qa-harness/run.rs"]);
insert_with_paths(&db, "sha-api", MONO, &["services/api/main.rs"]);
insert_with_paths(&db, "sha-legacy", MONO, &["services-legacy/x.rs"]);
run(&pipeline, &mut db).await.expect("run");
assert_eq!(cat_method(&db, "sha-qa"), pair("qa", "repo_map"));
assert_eq!(
cat_method(&db, "sha-api"),
pair("platform_infrastructure", "repo_map")
);
assert_eq!(
cat_method(&db, "sha-legacy"),
pair("internal_tooling", "repo_map")
);
}
#[tokio::test]
async fn a_bare_repo_key_is_the_fallback() {
let rules = rules_file();
let map = [
(MONO, "internal_tooling"),
("acme-mono:services", "qa"),
("acme-platform:services", "qa"),
];
let pipeline =
ClassificationPipeline::new(config(rules.path(), &map, RepoMapConfig::default()));
let mut db = Database::open_in_memory().expect("db");
insert_with_paths(&db, "sha-docs", MONO, &["docs/readme.md"]);
insert_with_paths(&db, "sha-nopaths", MONO, &[]);
insert_with_paths(&db, "sha-svc", MONO, &["services/a.rs"]);
insert_with_paths(&db, "sha-other-docs", "acme-platform", &["docs/readme.md"]);
run(&pipeline, &mut db).await.expect("run");
for sha in ["sha-docs", "sha-nopaths"] {
assert_eq!(
cat_method(&db, sha),
pair("internal_tooling", "repo_map"),
"{sha}"
);
}
assert_eq!(cat_method(&db, "sha-svc"), pair("qa", "repo_map"));
let (category, method) = cat_method(&db, "sha-other-docs");
assert_ne!(method, "repo_map");
assert_eq!(category, "uncategorized");
}
#[tokio::test]
async fn a_commit_spanning_prefixes_takes_the_category_most_paths_resolve_to() {
let rules = rules_file();
let map = [
("r-major:services", "platform_infrastructure"),
("r-major:tests", "qa"),
("r-pool:a", "qa"),
("r-pool:b", "qa"),
("r-pool:c", "platform_infrastructure"),
("r-len:services", "qa"),
("r-len:tests", "platform_infrastructure"),
("r-alpha:bb", "qa"),
("r-alpha:aa", "platform_infrastructure"),
("r-unmapped:services", "qa"),
("r-tie:services", "qa"),
];
let pipeline =
ClassificationPipeline::new(config(rules.path(), &map, RepoMapConfig::default()));
let mut db = Database::open_in_memory().expect("db");
insert_with_paths(
&db,
"sha-major",
"r-major",
&["services/a.rs", "services/b.rs", "tests/c.rs"],
);
insert_with_paths(&db, "sha-pool", "r-pool", &["a/1", "b/1", "c/1"]);
insert_with_paths(&db, "sha-len", "r-len", &["services/a.rs", "tests/c.rs"]);
insert_with_paths(&db, "sha-alpha", "r-alpha", &["aa/1", "bb/1"]);
insert_with_paths(
&db,
"sha-unmapped",
"r-unmapped",
&["docs/x.md", "docs/y.md", "services/a.rs"],
);
insert_with_paths(&db, "sha-tie", "r-tie", &["docs/x.md", "services/a.rs"]);
run(&pipeline, &mut db).await.expect("run");
let platform = pair("platform_infrastructure", "repo_map");
let qa = pair("qa", "repo_map");
assert_eq!(cat_method(&db, "sha-major"), platform);
assert_eq!(cat_method(&db, "sha-pool"), qa);
assert_eq!(cat_method(&db, "sha-len"), qa);
assert_eq!(cat_method(&db, "sha-alpha"), platform);
let (category, method) = cat_method(&db, "sha-unmapped");
assert_ne!(method, "repo_map", "{category}");
assert_eq!(cat_method(&db, "sha-tie"), qa);
}
#[tokio::test]
#[tracing_test::traced_test]
async fn an_unmatched_key_warns_once_per_run() {
let rules = rules_file();
let map = [
(MAPPED, "internal_tooling"),
("ghost-repo", "qa"),
("acme-mono:nowhere", "qa"),
("acme-mono:services", "qa"),
];
let pipeline =
ClassificationPipeline::new(config(rules.path(), &map, RepoMapConfig::default()));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", MAPPED, NOTHING);
insert(&db, "sha-b", MAPPED, NOTHING);
insert_with_paths(&db, "sha-c", MONO, &["services/a.rs"]);
run(&pipeline, &mut db).await.expect("run");
logs_assert(|lines: &[&str]| {
let warned = |key: &str| {
lines
.iter()
.filter(|l| l.contains("WARN") && l.contains(&format!("'{key}'")))
.count()
};
for (key, want) in [
("ghost-repo", 1),
("acme-mono:nowhere", 1),
(MAPPED, 0),
("acme-mono:services", 0),
] {
if warned(key) != want {
return Err(format!("{key}: {} warnings, want {want}", warned(key)));
}
}
Ok(())
});
}
#[tokio::test]
async fn a_malformed_key_is_rejected() {
let rules = rules_file();
for key in ["acme-mono/api", "acme-mono:", ":api", "acme-mono:/"] {
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(key, "qa")],
RepoMapConfig::default(),
));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", MONO, NOTHING);
let msg = run(&pipeline, &mut db)
.await
.expect_err("malformed key")
.to_string();
assert!(msg.contains(key), "{key}: {msg}");
assert!(msg.contains("<repo>:<prefix>"), "{key}: {msg}");
}
}
#[tokio::test]
async fn an_out_of_range_min_confidence_is_rejected() {
let rules = rules_file();
let repo_map = RepoMapConfig {
min_confidence: 1.5,
..floor()
};
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
repo_map,
));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", MAPPED, NOTHING);
let msg = run(&pipeline, &mut db)
.await
.expect_err("out of range")
.to_string();
assert!(msg.contains("min_confidence"), "{msg}");
}
#[test]
fn a_prefix_key_gives_jev_only_its_repository_name() {
let rules = rules_file();
let cfg = config(
rules.path(),
&[("qmono-repo:services/api", "qa")],
RepoMapConfig::default(),
);
let names = super::pipeline_jev::known_names(&cfg);
assert!(
names.repos.iter().any(|n| n == "qmono-repo"),
"{:?}",
names.repos
);
for part in ["api", "services", "services/api", "qmono-repo:services/api"] {
assert!(
!names.repos.iter().any(|n| n == part),
"{part}: {:?}",
names.repos
);
}
}
#[tokio::test]
async fn floor_keeps_a_default_ruleset_bug_fix() {
let config = Config {
classification: Some(ClassificationConfig {
repo_categories: [(MAPPED.to_string(), "tooling".to_string())].into(),
repo_map: floor(),
..ClassificationConfig::default()
}),
..Config::default()
};
let pipeline = ClassificationPipeline::new(config);
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-fix", MAPPED, "fix: handle the empty token");
insert(&db, "sha-feat", MAPPED, "feat: add the export button");
run(&pipeline, &mut db).await.expect("run");
assert_eq!(cat_method(&db, "sha-fix").0, "bugfix");
assert_eq!(cat_method(&db, "sha-feat"), pair("tooling", "repo_map"));
}
#[tokio::test]
async fn a_user_written_unknown_exception_is_rejected() {
let rules = rules_file();
let repo_map: RepoMapConfig =
serde_yaml::from_str("mode: floor\nexceptions: [qa, bugfixes]\n").expect("block");
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
repo_map,
));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", MAPPED, NOTHING);
let msg = run(&pipeline, &mut db)
.await
.expect_err("unknown exception")
.to_string();
assert!(msg.contains("bugfixes"), "{msg}");
assert!(msg.contains("exceptions"), "{msg}");
}
#[tokio::test]
async fn a_bare_key_naming_a_configured_slash_repository_maps_it() {
let rules = rules_file();
let with_repo = |key: &str| {
let mut cfg = config(rules.path(), &[(key, "qa")], RepoMapConfig::default());
cfg.repositories = vec![crate::core::config::RepositoryConfig {
path: "/nonexistent/widget".into(),
name: Some("acme-org/widget".into()),
..Default::default()
}];
ClassificationPipeline::new(cfg)
};
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", "acme-org/widget", NOTHING);
run(&with_repo("acme-org/widget"), &mut db)
.await
.expect("configured name");
assert_eq!(cat_method(&db, "sha-a"), pair("qa", "repo_map"));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", "acme-org/widget", NOTHING);
let msg = run(&with_repo("acme-org/gadget"), &mut db)
.await
.expect_err("unconfigured slash key")
.to_string();
assert!(msg.contains("acme-org/gadget"), "{msg}");
assert!(msg.contains("repositories[].name"), "{msg}");
}
#[tokio::test]
async fn floor_mode_applies_to_a_prefix_key() {
let rules = rules_file();
let map = [
(MONO, "internal_tooling"),
("acme-mono:services", "platform_infrastructure"),
];
let pipeline = ClassificationPipeline::new(config(rules.path(), &map, floor()));
let mut db = Database::open_in_memory().expect("db");
let with = |sha: &str, msg: &str, path: &str| {
insert(&db, sha, MONO, msg);
let id = db.connection().last_insert_rowid();
db.connection()
.execute(
"INSERT INTO files (commit_id, path, change_type) VALUES (?1, ?2, 'M')",
params![id, path],
)
.expect("insert file");
};
with(
"sha-sec",
"security: rotate the signing keys",
"services/a.rs",
);
with("sha-feat", "feat: add the widget", "services/b.rs");
with("sha-qa", "flaky: retry the login test", "docs/x.md");
run(&pipeline, &mut db).await.expect("run");
assert_eq!(cat_method(&db, "sha-sec"), pair("security", "exact_rule"));
assert_eq!(
cat_method(&db, "sha-feat"),
pair("platform_infrastructure", "repo_map")
);
assert_eq!(
cat_method(&db, "sha-qa"),
pair("internal_tooling", "repo_map")
);
}
#[tokio::test]
async fn duplicate_keys_naming_one_prefix_are_rejected() {
let rules = rules_file();
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[("r:api", "qa"), ("r:api/", "internal_tooling")],
RepoMapConfig::default(),
));
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-a", "r", NOTHING);
let msg = run(&pipeline, &mut db)
.await
.expect_err("duplicate")
.to_string();
assert!(msg.contains("'r:api'") && msg.contains("'r:api/'"), "{msg}");
}
#[tokio::test]
async fn floor_keeps_the_llm_complexity_when_it_replaces_an_llm_verdict() {
let rules = rules_file();
let pipeline = ClassificationPipeline::new(config(
rules.path(),
&[(MAPPED, "internal_tooling")],
floor(),
));
let server = mock_llm("new_feature", 0.95).await;
let mut db = Database::open_in_memory().expect("db");
insert(&db, "sha-none", MAPPED, NOTHING);
run_with_llm(&pipeline, &mut db, &server).await;
assert_eq!(
cat_method(&db, "sha-none"),
pair("internal_tooling", "repo_map")
);
let complexity: Option<i64> = db
.connection()
.query_row(
"SELECT cl.complexity FROM commits c JOIN classifications cl \
ON cl.id = c.classification_id WHERE c.sha = 'sha-none'",
[],
|r| r.get(0),
)
.expect("complexity");
assert_eq!(complexity, Some(2));
}