use super::super::super::error::{MergeError, MergeErrorSubject};
use super::super::super::patch::ClausewitzPatch;
use super::super::super::patch_merge::PatchAddress;
use super::super::stale_vanilla::detect_stale_vanilla_targets;
use super::DepMisuseRemoveCount;
use crate::game::eu4::content::{MergeKeySource, MergePolicies};
use crate::game::eu4::script::ParsedScriptFile;
use crate::game::eu4::script::emit::{EmitOptions, emit_clausewitz_statements_with_options};
use crate::game::eu4::script::parser::{AstStatement, AstValue};
use crate::input::{InputScriptCache, ResolvedInputContributor};
use crate::merge::kernel::{DeltaOperation, NodeId, TreeMatcher};
use crate::merge::model::{SemanticDeltaPartition, SemanticSourceDelta};
use crate::merge::structured::{
DefinitionModuleAdapter, TransformModuleAdapter, TreePartitionAdapter, semantic_node_address,
};
use crate::merge::transform::tree::EntityTransform;
use crate::model::{DepMisuseFinding, GamePath, StaleVanillaTargetDescriptor};
use std::collections::{BTreeMap, HashMap, HashSet, btree_map::Entry};
const SEMANTIC_MISSING_PATH_NOTE: &str = "vanilla snapshot for this file does not contain the semantic parent; this remove-style change may be cross-version drift, dependency-targeted, or intentionally guarded";
const SEMANTIC_MISSING_KEY_NOTE: &str = "vanilla snapshot contains the semantic parent but not the target; this remove-style change may be cross-version drift, dependency-targeted, or intentionally guarded";
pub(super) fn vanilla_snippet_for_address(
vanilla: Option<&ParsedScriptFile>,
address: &PatchAddress,
emit_options: &EmitOptions,
) -> Option<String> {
let vanilla = vanilla?;
let statements = vanilla_statements_at_address(&vanilla.ast.statements, address);
Some(match statements {
Some(statements) if !statements.is_empty() => {
emit_clausewitz_statements_with_options(&statements, emit_options)
.unwrap_or_else(|err| format!("(failed to render vanilla snippet: {err})"))
}
_ => "(key not present in vanilla)".to_string(),
})
}
fn vanilla_statements_at_address(
statements: &[AstStatement],
address: &PatchAddress,
) -> Option<Vec<AstStatement>> {
let mut current = statements;
for segment in &address.path {
current = current.iter().find_map(|statement| match statement {
AstStatement::Assignment {
key,
value: AstValue::Block { items, .. },
..
} if key == segment => Some(items.as_slice()),
_ => None,
})?;
}
let Some(key) = vanilla_address_lookup_key(&address.key) else {
return Some(current.to_vec());
};
if key.is_empty() {
return Some(current.to_vec());
}
let matches = current
.iter()
.filter(|statement| {
matches!(statement, AstStatement::Assignment { key: statement_key, .. } if statement_key == key)
})
.cloned()
.collect::<Vec<_>>();
(!matches.is_empty()).then_some(matches)
}
fn vanilla_address_lookup_key(address_key: &str) -> Option<&str> {
if let Some(rest) = address_key.strip_prefix("__node__::") {
return rest.split("::").next();
}
if let Some(rest) = address_key.strip_prefix("__list_item__::") {
return rest.split("::").next();
}
if let Some(rest) = address_key.strip_prefix("__rename__::") {
return Some(rest);
}
if address_key.starts_with("__append_block_item__::")
|| address_key.starts_with("__remove_block_item__::")
{
return None;
}
Some(address_key)
}
pub(super) fn parse_vanilla_for_stale_detection(
file_path: &GamePath,
contributors: &[ResolvedInputContributor],
script_cache: &InputScriptCache,
) -> Result<Option<ParsedScriptFile>, MergeError> {
let Some(base) = contributors
.iter()
.find(|contributor| contributor.is_base_game)
else {
return Ok(None);
};
script_cache
.load(base)
.map(|parsed| Some((*parsed).clone()))
.map_err(|error| MergeError::Validation {
subject: Some(MergeErrorSubject::Game(file_path.to_owned())),
message: format!(
"failed to load verified vanilla AST for stale target detection: {error}"
),
})
}
pub(super) fn collect_stale_vanilla_targets(
file_path: &GamePath,
mod_patches: &[(String, usize, Vec<ClausewitzPatch>)],
vanilla: Option<&ParsedScriptFile>,
merge_key_source: MergeKeySource,
mod_versions: &HashMap<String, String>,
) -> Vec<StaleVanillaTargetDescriptor> {
mod_patches
.iter()
.flat_map(|(mod_id, _, patches)| {
let mod_version = mod_versions
.get(mod_id)
.map(String::as_str)
.unwrap_or("unknown");
detect_stale_vanilla_targets(
patches,
file_path,
mod_id,
mod_version,
vanilla,
merge_key_source,
)
})
.collect()
}
pub(super) fn collect_semantic_stale_vanilla_targets(
file_path: &GamePath,
source_deltas: &[SemanticSourceDelta],
vanilla: Option<&ParsedScriptFile>,
policies: &MergePolicies,
mod_versions: &HashMap<String, String>,
entity_transform: Option<&dyn EntityTransform>,
) -> Result<Vec<StaleVanillaTargetDescriptor>, String> {
let Some(vanilla) = vanilla else {
return Ok(Vec::new());
};
let transformed = entity_transform
.filter(|transform| transform.applies_to(file_path))
.map(|transform| TransformModuleAdapter { transform });
let adapter: &dyn TreePartitionAdapter = match &transformed {
Some(adapter) => adapter,
None => &DefinitionModuleAdapter,
};
let prepared = adapter.prepare(&vanilla.ast);
let mut vanilla_trees = BTreeMap::new();
let mut findings = Vec::new();
for source_delta in source_deltas {
let mod_version = mod_versions
.get(&source_delta.source.source_id)
.map(String::as_str)
.unwrap_or("unknown");
for partition in &source_delta.partitions {
let mut remove_targets = semantic_remove_targets(partition).peekable();
if remove_targets.peek().is_none() {
continue;
}
let vanilla_tree = match vanilla_trees.entry(&partition.partition) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
entry.insert(prepared.normalize(&partition.partition, policies).map_err(
|error| {
format!(
"failed to normalize vanilla partition {:?}: {error}",
partition.partition
)
},
)?)
}
};
let matching = TreeMatcher::default().match_trees(vanilla_tree, &partition.base_tree);
for (kind, target) in remove_targets {
if matching.get_from_right(target).is_some() {
continue;
}
let target_node = partition.base_tree.node(target).map_err(|error| {
format!("semantic remove target is outside its base tree: {error}")
})?;
let parent_exists = target_node
.parent
.is_none_or(|parent| matching.get_from_right(parent).is_some());
let address =
semantic_node_address(&partition.base_tree, target).map_err(|error| {
format!("failed to project semantic remove target: {error}")
})?;
if parent_exists && address.key.is_none() {
continue;
}
findings.push(StaleVanillaTargetDescriptor {
mod_id: source_delta.source.source_id.clone(),
mod_version: mod_version.to_string(),
file_path: file_path.to_owned(),
patch_kind: kind.to_string(),
target_path: address.path,
target_key: address.key,
note: Some(
if parent_exists {
SEMANTIC_MISSING_KEY_NOTE
} else {
SEMANTIC_MISSING_PATH_NOTE
}
.to_string(),
),
});
}
}
}
Ok(findings)
}
fn semantic_remove_targets(
partition: &SemanticDeltaPartition,
) -> impl Iterator<Item = (&'static str, NodeId)> + '_ {
partition
.delta
.operations
.iter()
.filter_map(|operation| match operation {
DeltaOperation::Delete { tombstone } => Some(("Delete", tombstone.deleted.node)),
DeltaOperation::Rename { base, .. } => Some(("Rename", base.node)),
DeltaOperation::Insert { .. }
| DeltaOperation::Update { .. }
| DeltaOperation::Move { .. } => None,
})
}
pub(super) fn collect_dep_misuse_remove_counts(
findings: &[DepMisuseFinding],
contributors: &[ResolvedInputContributor],
mod_patches: &[(String, usize, Vec<ClausewitzPatch>)],
) -> Vec<DepMisuseRemoveCount> {
if findings.is_empty() {
return Vec::new();
}
let contributor_mods = contributors
.iter()
.filter(|contributor| !contributor.is_base_game && !contributor.is_synthetic_base)
.map(|contributor| contributor.mod_id.as_str())
.collect::<HashSet<_>>();
let mut counts = Vec::new();
for finding in findings {
if !contributor_mods.contains(finding.mod_id.as_str())
|| !contributor_mods.contains(finding.suspicious_dep_id.as_str())
{
continue;
}
let count = mod_patches
.iter()
.filter(|(mod_id, _, _)| mod_id == &finding.mod_id)
.flat_map(|(_, _, patches)| patches)
.filter(|patch| is_remove_patch(patch))
.count();
if count == 0 {
continue;
}
counts.push(DepMisuseRemoveCount {
mod_id: finding.mod_id.clone(),
dep_id: finding.suspicious_dep_id.clone(),
count: count.min(u32::MAX as usize) as u32,
});
}
counts
}
pub(super) fn collect_semantic_dep_misuse_remove_counts(
findings: &[DepMisuseFinding],
contributors: &[ResolvedInputContributor],
source_deltas: &[SemanticSourceDelta],
) -> Vec<DepMisuseRemoveCount> {
if findings.is_empty() {
return Vec::new();
}
let contributor_mods = contributors
.iter()
.filter(|contributor| !contributor.is_base_game && !contributor.is_synthetic_base)
.map(|contributor| contributor.mod_id.as_str())
.collect::<HashSet<_>>();
findings
.iter()
.filter(|finding| {
contributor_mods.contains(finding.mod_id.as_str())
&& contributor_mods.contains(finding.suspicious_dep_id.as_str())
})
.filter_map(|finding| {
let count = source_deltas
.iter()
.filter(|delta| delta.source.source_id == finding.mod_id)
.map(semantic_remove_count)
.sum::<usize>();
(count != 0).then(|| DepMisuseRemoveCount {
mod_id: finding.mod_id.clone(),
dep_id: finding.suspicious_dep_id.clone(),
count: count.min(u32::MAX as usize) as u32,
})
})
.collect()
}
fn semantic_remove_count(source_delta: &SemanticSourceDelta) -> usize {
source_delta
.partitions
.iter()
.flat_map(|partition| {
partition
.delta
.operations
.iter()
.filter(|operation| match operation {
DeltaOperation::Delete { tombstone } => partition
.base_tree
.node(tombstone.deleted.node)
.is_ok_and(|node| {
node.kind.starts_with("clausewitz.assignment:")
|| node.kind == "clausewitz.assignment.entity"
|| node.kind == "clausewitz.item"
}),
DeltaOperation::Insert { .. }
| DeltaOperation::Update { .. }
| DeltaOperation::Move { .. }
| DeltaOperation::Rename { .. } => false,
})
})
.count()
}
fn is_remove_patch(patch: &ClausewitzPatch) -> bool {
matches!(
patch,
ClausewitzPatch::RemoveNode { .. }
| ClausewitzPatch::RemoveListItem { .. }
| ClausewitzPatch::RemoveBlockItem { .. }
)
}
pub(super) fn apply_dep_misuse_remove_counts(
findings: &mut [DepMisuseFinding],
counts: Vec<DepMisuseRemoveCount>,
) {
for count in counts {
if let Some(finding) = findings.iter_mut().find(|finding| {
finding.mod_id == count.mod_id && finding.suspicious_dep_id == count.dep_id
}) {
finding.evidence.false_remove_count = finding
.evidence
.false_remove_count
.saturating_add(count.count);
}
}
}
#[cfg(test)]
mod tests {
use crate::game::eu4::cultures::correspondence::CultureCorrespondence;
use std::fs;
use crate::game::eu4::content::ScriptFileKind;
use crate::game::eu4::script::parser::parse_clausewitz_content;
use crate::merge::kernel::{
ChildCardinality, NormalizedTree, RevisionDelta, RevisionId, TreeMatcher, TreeNode,
};
use super::*;
use crate::merge::model::{
SemanticDeltaPartition, SemanticMergeSource, SemanticPartitionId, SemanticSourceDelta,
};
use tempfile::TempDir;
#[test]
fn stale_detection_never_bypasses_a_missing_verified_ast() {
let temp = TempDir::new().expect("temp dir");
let relative = "events/test.txt";
fs::create_dir_all(temp.path().join("events")).expect("create events dir");
fs::write(
temp.path().join(relative),
"country_event = { id = test.1 }\n",
)
.expect("write vanilla script");
let contributor = ResolvedInputContributor {
mod_id: "__game__".to_string(),
root_path: temp.path().to_path_buf(),
relative_path: crate::model::GamePathBuf::parse(relative).expect("valid game path"),
precedence: 0,
is_base_game: true,
is_synthetic_base: false,
parse_ok_hint: Some(true),
mod_hash: None,
};
let error = parse_vanilla_for_stale_detection(
&contributor.relative_path,
std::slice::from_ref(&contributor),
&InputScriptCache::default(),
)
.expect_err("missing verified AST must fail closed");
assert!(matches!(error, MergeError::Validation { .. }));
assert!(error.to_string().contains("no semantic-snapshot input"));
}
#[test]
fn semantic_remove_count_counts_only_deleted_statement_roots() {
let base = normalized(vec![
assignment("removed", "yes"),
assignment("changed", "old"),
item("removed-item"),
]);
let revision = normalized(vec![
assignment("changed", "new"),
assignment("inserted", "yes"),
]);
let matching = TreeMatcher::default().match_trees(&base, &revision);
let delta = RevisionDelta::between(&base, RevisionId::LEFT, &revision, &matching);
assert!(
delta
.operations
.iter()
.any(|operation| matches!(operation, DeltaOperation::Update { .. }))
);
assert!(
delta
.operations
.iter()
.any(|operation| matches!(operation, DeltaOperation::Insert { .. }))
);
let source_delta = SemanticSourceDelta {
source: SemanticMergeSource {
source_id: "mod-a".to_string(),
precedence: 10,
},
partitions: vec![SemanticDeltaPartition {
partition: SemanticPartitionId::File,
base_tree: base,
revision_tree: revision,
delta,
}],
};
assert_eq!(semantic_remove_count(&source_delta), 2);
}
#[test]
fn semantic_remove_count_includes_deleted_culture_roots_but_not_renames() {
let parse = |source| {
parse_clausewitz_content(
&crate::model::GamePathBuf::parse("common/cultures/test.txt")
.expect("valid game path"),
source,
)
.ast
};
let base = parse(
"g = { removed_culture = { primary = AAA male_names = { Johann } } old_culture = { primary = BBB } }",
);
let revision = parse("g = { renamed_culture = { primary = BBB } }");
let correspondence = CultureCorrespondence::from_files(&base, &[&revision])
.unwrap()
.unwrap();
let adapter = TransformModuleAdapter {
transform: &correspondence,
};
let policies = MergePolicies::default();
let normalize = |file: &crate::game::eu4::script::parser::AstFile| {
adapter
.prepare(file)
.normalize(&SemanticPartitionId::File, &policies)
.unwrap()
};
let source = source_delta(normalize(&base), normalize(&revision));
let partition = &source.partitions[0];
assert!(partition.delta.operations.iter().any(|operation| matches!(operation, DeltaOperation::Rename { from, to, .. }
if from.value.as_deref() == Some("old_culture") && to.value.as_deref() == Some("renamed_culture"))));
let deleted = partition.delta.tombstones().collect::<Vec<_>>();
assert_eq!(deleted.len(), 1, "one deleted statement subtree");
assert_eq!(
partition
.base_tree
.node(deleted[0].deleted.node)
.unwrap()
.value
.as_deref(),
Some("removed_culture")
);
assert_eq!(semantic_remove_count(&source), 1);
}
#[test]
fn culture_stale_detection_uses_the_same_identity_as_the_source_delta() {
use crate::game::eu4::cultures::correspondence::CultureCorrespondence;
use crate::merge::structured::TransformModuleAdapter;
let path =
&crate::model::GamePathBuf::parse("common/cultures/test.txt").expect("valid game path");
let parsed = |source| {
let mut file = parsed_vanilla(source);
file.relative_path = path.clone();
file.ast.path = path.clone();
file
};
let vanilla = parsed("g = { old = { primary = AAA } }");
let parent = parsed("g = { old = { primary = AAA } dependency_old = { primary = BBB } }");
let revision =
parsed("g = { renamed = { primary = AAA } dependency_new = { primary = BBB } }");
let correspondence = CultureCorrespondence::from_files(&parent.ast, &[&revision.ast])
.unwrap()
.unwrap();
let adapter = TransformModuleAdapter {
transform: &correspondence,
};
let policies = MergePolicies::default();
let normalize = |file: &ParsedScriptFile| {
adapter
.prepare(&file.ast)
.normalize(&SemanticPartitionId::File, &policies)
.unwrap()
};
let delta = source_delta(normalize(&parent), normalize(&revision));
assert_eq!(
delta.partitions[0]
.delta
.operations
.iter()
.filter(|operation| matches!(operation, DeltaOperation::Rename { .. }))
.count(),
2
);
let findings = collect_semantic_stale_vanilla_targets(
path,
&[delta],
Some(&vanilla),
&policies,
&HashMap::new(),
Some(&correspondence),
)
.unwrap();
assert_eq!(
findings.len(),
1,
"the vanilla culture must not be reported as missing: {findings:?}"
);
assert_eq!(findings[0].target_key.as_deref(), Some("dependency_old"));
assert_eq!(findings[0].target_path, ["g"]);
assert_eq!(findings[0].patch_kind, "Rename");
}
#[test]
fn semantic_stale_detection_compares_remove_targets_with_original_vanilla() {
let present = source_delta(
normalized(vec![assignment("present", "yes")]),
normalized(vec![]),
);
let absent = source_delta(
normalized(vec![assignment("absent", "yes")]),
normalized(vec![]),
);
let vanilla = parsed_vanilla("present = yes\n");
let versions = HashMap::from([("mod-a".to_string(), "1.0.0".to_string())]);
let present_findings = collect_semantic_stale_vanilla_targets(
crate::model::GamePath::new("common/test.txt").expect("valid game path"),
&[present],
Some(&vanilla),
&MergePolicies::default(),
&versions,
None,
)
.expect("inspect present semantic target");
let absent_findings = collect_semantic_stale_vanilla_targets(
crate::model::GamePath::new("common/test.txt").expect("valid game path"),
&[absent],
Some(&vanilla),
&MergePolicies::default(),
&versions,
None,
)
.expect("inspect absent semantic target");
assert!(present_findings.is_empty());
assert_eq!(absent_findings.len(), 1);
assert_eq!(absent_findings[0].patch_kind, "Delete");
assert_eq!(absent_findings[0].target_key.as_deref(), Some("absent"));
}
fn source_delta(base: NormalizedTree, revision: NormalizedTree) -> SemanticSourceDelta {
let matching = TreeMatcher::default().match_trees(&base, &revision);
let delta = RevisionDelta::between(&base, RevisionId::LEFT, &revision, &matching);
SemanticSourceDelta {
source: SemanticMergeSource {
source_id: "mod-a".to_string(),
precedence: 10,
},
partitions: vec![SemanticDeltaPartition {
partition: SemanticPartitionId::File,
base_tree: base,
revision_tree: revision,
delta,
}],
}
}
fn parsed_vanilla(source: &str) -> ParsedScriptFile {
let path = crate::model::GamePathBuf::parse("common/test.txt").expect("valid game path");
let parsed = parse_clausewitz_content(&path, source);
assert!(parsed.diagnostics.is_empty(), "{:?}", parsed.diagnostics);
ParsedScriptFile {
mod_id: "__game__".to_string(),
path: None,
relative_path: path,
content_family: None,
file_kind: ScriptFileKind::new("other"),
module_name: "test".to_string(),
ast: parsed.ast,
source: source.to_string(),
parse_issues: Vec::new(),
parse_cache_hit: false,
}
}
fn normalized(children: Vec<TreeNode>) -> NormalizedTree {
NormalizedTree::from_root(TreeNode::branch("clausewitz.file", children))
.expect("valid normalized fixture")
}
fn assignment(key: &str, value: &str) -> TreeNode {
TreeNode::branch(
format!("clausewitz.assignment:{key}"),
vec![TreeNode::leaf("clausewitz.scalar.identifier", value)],
)
.with_parent_scoped_anchor("clausewitz.assignment.key", key)
.with_child_cardinality(ChildCardinality::ExactlyOne)
}
fn item(value: &str) -> TreeNode {
TreeNode::branch(
"clausewitz.item",
vec![TreeNode::leaf("clausewitz.scalar.identifier", value)],
)
.with_parent_scoped_anchor("clausewitz.item.scalar", value)
.with_child_cardinality(ChildCardinality::ExactlyOne)
}
}