use core::fmt;
use crate::clone_class::CloneClass;
use crate::discovery::{BuildVariant, Language};
use crate::engine::normalize::{self, LiteralNorm, NormAtom, Resolution};
use crate::engine::{EngineReport, InputFile};
use crate::frontend::Token;
use crate::semantic::{SOG_SCHEMA_VERSION, SemanticOperationGraph};
pub const FP_SCHEMA_VERSION: &str = "fp-schema-v1";
pub const HASH_ALGORITHM: &str = "blake3-128";
macro_rules! stable_id {
($(#[$doc:meta])* $name:ident) => {
$(#[$doc])*
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct $name([u8; 16]);
impl $name {
#[must_use]
pub const fn from_bytes(bytes: [u8; 16]) -> Self {
Self(bytes)
}
#[must_use]
pub const fn as_bytes(&self) -> &[u8; 16] {
&self.0
}
#[must_use]
pub fn to_hex(&self) -> String {
self.to_string()
}
}
impl fmt::Display for $name {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
for byte in self.0 {
write!(f, "{byte:02x}")?;
}
Ok(())
}
}
};
}
stable_id!(
UnitFingerprint
);
stable_id!(
FragmentFingerprint
);
stable_id!(
CloneGroupFingerprint
);
stable_id!(
GroupLineageId
);
stable_id!(
FindingId
);
stable_id!(
CrossVariantComparisonId
);
stable_id!(
CrossVariantGroupId
);
stable_id!(
CrossVariantMemberId
);
stable_id!(
CrossLanguageComparisonId
);
stable_id!(
CrossLanguageGroupId
);
stable_id!(
CrossLanguageMemberId
);
pub const CROSS_VARIANT_POLICY_VERSION: &str = "cross-variant-exact-v1";
pub const CROSS_LANGUAGE_POLICY_VERSION: &str = "cross-language-semantic-v1";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContentNorm {
Raw,
Normalized(LiteralNorm),
ResolvedNormalized(LiteralNorm),
}
impl ContentNorm {
#[must_use]
pub const fn label(self) -> &'static str {
match self {
Self::Raw => "raw",
Self::Normalized(LiteralNorm::Preserve) => "alpha-lit-preserve",
Self::Normalized(LiteralNorm::Category) => "alpha-lit-category",
Self::Normalized(LiteralNorm::Full) => "alpha-lit-full",
Self::ResolvedNormalized(LiteralNorm::Preserve) => "alpha-resolved-lit-preserve",
Self::ResolvedNormalized(LiteralNorm::Category) => "alpha-resolved-lit-category",
Self::ResolvedNormalized(LiteralNorm::Full) => "alpha-resolved-lit-full",
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct FileContext<'a> {
pub frontend_version: &'a str,
pub language: Language,
}
struct IdHasher {
hasher: blake3::Hasher,
}
impl IdHasher {
fn new(domain: &str) -> Self {
let mut this = Self {
hasher: blake3::Hasher::new(),
};
this.write_bytes(domain.as_bytes());
this
}
fn write_bytes(&mut self, bytes: &[u8]) {
let len = u32::try_from(bytes.len()).unwrap_or(u32::MAX);
self.hasher.update(&len.to_le_bytes());
self.hasher.update(bytes);
}
fn write_str(&mut self, text: &str) {
self.write_bytes(text.as_bytes());
}
fn write_u8(&mut self, value: u8) {
self.hasher.update(&[value]);
}
fn write_u32(&mut self, value: u32) {
self.hasher.update(&value.to_le_bytes());
}
fn write_context(&mut self, variant: &BuildVariant, file: &FileContext<'_>, norm: ContentNorm) {
self.write_str(FP_SCHEMA_VERSION);
self.write_str(HASH_ALGORITHM);
self.write_str(norm.label());
self.write_u32(variant.normalization_version);
self.write_str(file.frontend_version);
self.write_str(variant.mode.name());
self.write_str(file.language.name());
self.write_str(&variant.canonical());
}
fn write_content(
&mut self,
tokens: &[Token],
norm: ContentNorm,
resolution: Option<&Resolution>,
) {
match norm {
ContentNorm::Raw => {
for token in tokens {
self.write_u8(token.kind.tag());
self.write_bytes(token.text.as_bytes());
}
}
ContentNorm::Normalized(literals) | ContentNorm::ResolvedNormalized(literals) => {
let mut normalized = Vec::new();
normalize::normalize_resolved_into(
tokens,
literals,
matches!(norm, ContentNorm::ResolvedNormalized(_))
.then_some(resolution)
.flatten(),
&mut normalized,
);
for norm_token in normalized {
self.write_u8(norm_token.tag);
match norm_token.atom {
NormAtom::Renamed(n) => {
self.write_u8(1);
self.write_u32(n);
}
NormAtom::Text(text) => {
self.write_u8(2);
self.write_bytes(text.as_bytes());
}
NormAtom::Literal(class) => {
self.write_u8(3);
self.write_u8(class);
}
}
}
}
}
}
fn finish(self) -> [u8; 16] {
let digest = self.hasher.finalize();
let mut out = [0u8; 16];
out.copy_from_slice(&digest.as_bytes()[..16]);
out
}
}
#[must_use]
pub fn unit_fingerprint(
variant: &BuildVariant,
file: &FileContext<'_>,
tokens: &[Token],
norm: ContentNorm,
) -> UnitFingerprint {
let mut hasher = IdHasher::new("unit");
hasher.write_context(variant, file, norm);
hasher.write_content(tokens, norm, None);
UnitFingerprint(hasher.finish())
}
#[must_use]
pub fn fragment_fingerprint(
variant: &BuildVariant,
file: &FileContext<'_>,
kind: &str,
tokens: &[Token],
norm: ContentNorm,
) -> FragmentFingerprint {
let mut hasher = IdHasher::new("fragment");
hasher.write_context(variant, file, norm);
hasher.write_str(kind);
hasher.write_content(tokens, norm, None);
FragmentFingerprint(hasher.finish())
}
#[must_use]
pub fn resolved_fragment_fingerprint(
variant: &BuildVariant,
file: &FileContext<'_>,
kind: &str,
tokens: &[Token],
norm: ContentNorm,
resolution: Option<&Resolution>,
) -> FragmentFingerprint {
let mut hasher = IdHasher::new("fragment");
hasher.write_context(variant, file, norm);
hasher.write_str(kind);
hasher.write_content(tokens, norm, resolution);
FragmentFingerprint(hasher.finish())
}
#[must_use]
pub fn semantic_fragment_fingerprint(
variant: &BuildVariant,
graph: &SemanticOperationGraph,
) -> FragmentFingerprint {
let mut hasher = IdHasher::new("fragment-semantic");
hasher.write_context(
variant,
&FileContext {
frontend_version: SOG_SCHEMA_VERSION,
language: graph.language,
},
ContentNorm::Raw,
);
hasher.write_str(&graph.schema_version);
hasher.write_u32(u32::try_from(graph.nodes.len()).unwrap_or(u32::MAX));
for node in &graph.nodes {
hasher.write_str(node.kind.name());
match node.attributes.type_tag {
Some(tag) => {
hasher.write_u8(1);
hasher.write_str(tag.name());
}
None => hasher.write_u8(0),
}
hasher.write_u32(u32::try_from(node.attributes.api_names.len()).unwrap_or(u32::MAX));
for api_name in &node.attributes.api_names {
hasher.write_str(api_name);
}
match &node.attributes.resource_kind {
Some(resource_kind) => {
hasher.write_u8(1);
hasher.write_str(resource_kind);
}
None => hasher.write_u8(0),
}
match node.attributes.fallible_kind {
Some(kind) => {
hasher.write_u8(1);
hasher.write_str(kind.name());
}
None => hasher.write_u8(0),
}
match node.attributes.direct_propagation {
Some(kind) => {
hasher.write_u8(1);
hasher.write_str(kind.name());
}
None => hasher.write_u8(0),
}
match node.attributes.structure_fingerprint {
Some(fingerprint) => {
hasher.write_u8(1);
hasher.write_bytes(&fingerprint);
}
None => hasher.write_u8(0),
}
}
hasher.write_u32(u32::try_from(graph.edges.len()).unwrap_or(u32::MAX));
for edge in &graph.edges {
hasher.write_u32(edge.from);
hasher.write_u32(edge.to);
hasher.write_str(edge.kind.name());
}
FragmentFingerprint(hasher.finish())
}
#[must_use]
pub fn semantic_structure_fingerprint(
variant: &BuildVariant,
file: &FileContext<'_>,
tokens: &[Token],
) -> [u8; 16] {
let mut hasher = IdHasher::new("semantic-source-structure-v1");
hasher.write_context(variant, file, ContentNorm::Raw);
hasher.write_content(tokens, ContentNorm::Raw, None);
hasher.finish()
}
#[must_use]
pub fn semantic_occurrence_fingerprint(
content: FragmentFingerprint,
host: &UnitFingerprint,
occurrence_rank: u32,
) -> FragmentFingerprint {
let mut hasher = IdHasher::new("semantic-occurrence-v1");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_bytes(content.as_bytes());
hasher.write_bytes(host.as_bytes());
hasher.write_u32(occurrence_rank);
FragmentFingerprint(hasher.finish())
}
#[must_use]
pub fn clone_group_fingerprint(
variant: &BuildVariant,
clone_type: CloneClass,
members: &[FragmentFingerprint],
) -> CloneGroupFingerprint {
let mut distinct: Vec<[u8; 16]> = members.iter().map(|m| m.0).collect();
distinct.sort_unstable();
distinct.dedup();
let mut hasher = IdHasher::new("group");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(HASH_ALGORITHM);
hasher.write_str(variant.mode.name());
hasher.write_str(&variant.canonical());
hasher.write_str(clone_type.name());
hasher.write_u32(u32::try_from(distinct.len()).unwrap_or(u32::MAX));
for bytes in &distinct {
hasher.write_bytes(bytes);
}
CloneGroupFingerprint(hasher.finish())
}
#[must_use]
pub fn group_lineage_id(group: &CloneGroupFingerprint) -> GroupLineageId {
let mut hasher = IdHasher::new("group-lineage");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(HASH_ALGORITHM);
hasher.write_bytes(group.as_bytes());
GroupLineageId(hasher.finish())
}
#[must_use]
pub fn semantic_clone_group_fingerprint(
variant: &BuildVariant,
rule_id: &str,
rule_version: u32,
members: &[FragmentFingerprint],
) -> CloneGroupFingerprint {
let mut occurrences: Vec<[u8; 16]> = members.iter().map(|member| member.0).collect();
occurrences.sort_unstable();
let mut hasher = IdHasher::new("group-semantic");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(HASH_ALGORITHM);
hasher.write_str(variant.mode.name());
hasher.write_str(&variant.canonical());
hasher.write_str(CloneClass::RestrictedSemantic.name());
hasher.write_str(SOG_SCHEMA_VERSION);
hasher.write_str(rule_id);
hasher.write_u32(rule_version);
hasher.write_u32(u32::try_from(occurrences.len()).unwrap_or(u32::MAX));
for bytes in &occurrences {
hasher.write_bytes(bytes);
}
CloneGroupFingerprint(hasher.finish())
}
#[must_use]
pub fn structural_clone_group_fingerprint(
variant: &BuildVariant,
class: CloneClass,
canonical: &FragmentFingerprint,
members: &[FragmentFingerprint],
) -> CloneGroupFingerprint {
let mut distinct: Vec<[u8; 16]> = members.iter().map(|m| m.0).collect();
distinct.sort_unstable();
distinct.dedup();
let mut hasher = IdHasher::new("group-structural");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(HASH_ALGORITHM);
hasher.write_str(variant.mode.name());
hasher.write_str(&variant.canonical());
hasher.write_str(class.name());
hasher.write_bytes(&canonical.0);
hasher.write_u32(u32::try_from(distinct.len()).unwrap_or(u32::MAX));
for bytes in &distinct {
hasher.write_bytes(bytes);
}
CloneGroupFingerprint(hasher.finish())
}
#[must_use]
pub fn finding_id(
group: &CloneGroupFingerprint,
host: Option<&UnitFingerprint>,
rank_in_host: u32,
) -> FindingId {
let mut hasher = IdHasher::new("finding");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_bytes(&group.0);
match host {
Some(unit) => {
hasher.write_u8(1);
hasher.write_bytes(&unit.0);
}
None => hasher.write_u8(0),
}
hasher.write_u32(rank_in_host);
FindingId(hasher.finish())
}
#[must_use]
pub fn cross_variant_comparison_id(origins: &[String]) -> CrossVariantComparisonId {
let mut origins = origins.to_vec();
origins.sort_unstable();
origins.dedup();
let mut hasher = IdHasher::new("cross-variant-comparison");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(CROSS_VARIANT_POLICY_VERSION);
hasher.write_u32(u32::try_from(origins.len()).unwrap_or(u32::MAX));
for origin in origins {
hasher.write_str(&origin);
}
CrossVariantComparisonId(hasher.finish())
}
#[must_use]
pub fn cross_language_comparison_id(origins: &[String]) -> CrossLanguageComparisonId {
let mut origins = origins.to_vec();
origins.sort_unstable();
origins.dedup();
let mut hasher = IdHasher::new("cross-language-comparison");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(CROSS_LANGUAGE_POLICY_VERSION);
hasher.write_u32(u32::try_from(origins.len()).unwrap_or(u32::MAX));
for origin in origins {
hasher.write_str(&origin);
}
CrossLanguageComparisonId(hasher.finish())
}
#[must_use]
pub fn cross_language_group_id(
comparison: &CrossLanguageComparisonId,
rule_id: &str,
rule_version: u32,
members: &[FragmentFingerprint],
) -> CrossLanguageGroupId {
let mut members = members.to_vec();
members.sort_unstable();
let mut hasher = IdHasher::new("cross-language-group");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(CROSS_LANGUAGE_POLICY_VERSION);
hasher.write_bytes(comparison.as_bytes());
hasher.write_str(rule_id);
hasher.write_u32(rule_version);
hasher.write_u32(u32::try_from(members.len()).unwrap_or(u32::MAX));
for member in members {
hasher.write_bytes(member.as_bytes());
}
CrossLanguageGroupId(hasher.finish())
}
#[must_use]
pub fn cross_variant_group_id(
comparison: &CrossVariantComparisonId,
class: CloneClass,
language: Language,
content: &[u8; 16],
) -> CrossVariantGroupId {
let mut hasher = IdHasher::new("cross-variant-group");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(CROSS_VARIANT_POLICY_VERSION);
hasher.write_bytes(comparison.as_bytes());
hasher.write_str(class.name());
hasher.write_str(language.name());
hasher.write_bytes(content);
CrossVariantGroupId(hasher.finish())
}
#[must_use]
pub fn cross_variant_member_id(
group: &CrossVariantGroupId,
origin_variant: &str,
language: Language,
occurrence_rank: u32,
) -> CrossVariantMemberId {
let mut hasher = IdHasher::new("cross-variant-member");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(CROSS_VARIANT_POLICY_VERSION);
hasher.write_bytes(group.as_bytes());
hasher.write_str(origin_variant);
hasher.write_str(language.name());
hasher.write_u32(occurrence_rank);
CrossVariantMemberId(hasher.finish())
}
#[must_use]
pub fn cross_language_member_id(
group: &CrossLanguageGroupId,
origin_variant: &str,
occurrence: &FragmentFingerprint,
) -> CrossLanguageMemberId {
let mut hasher = IdHasher::new("cross-language-member");
hasher.write_str(FP_SCHEMA_VERSION);
hasher.write_str(CROSS_LANGUAGE_POLICY_VERSION);
hasher.write_bytes(group.as_bytes());
hasher.write_str(origin_variant);
hasher.write_bytes(occurrence.as_bytes());
CrossLanguageMemberId(hasher.finish())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemberIds {
pub content: FragmentFingerprint,
pub finding: FindingId,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GroupIds {
pub fingerprint: CloneGroupFingerprint,
pub members: Vec<MemberIds>,
}
#[must_use]
pub fn report_ids(
files: &[InputFile<'_>],
contexts: &[FileContext<'_>],
variant: &BuildVariant,
report: &EngineReport,
literals: LiteralNorm,
) -> Vec<GroupIds> {
report
.groups
.iter()
.map(|group| {
let norm = match group.clone_type {
CloneClass::Type1 => ContentNorm::Raw,
CloneClass::Type2 | CloneClass::Type3 | CloneClass::RestrictedSemantic => {
ContentNorm::Normalized(literals)
}
};
let member_fps: Vec<FragmentFingerprint> = group
.members
.iter()
.map(|member| {
let file = &files[member.file];
let start = member.token_start.min(file.tokens.len());
let end = member.token_end.min(file.tokens.len()).max(start);
let tokens = &file.tokens[start..end];
fragment_fingerprint(variant, &contexts[member.file], "member", tokens, norm)
})
.collect();
let fingerprint = clone_group_fingerprint(variant, group.clone_type, &member_fps);
let hosts: Vec<Option<UnitFingerprint>> = group
.members
.iter()
.map(|member| {
member.unit.map(|unit_idx| {
let unit = &files[member.file].units[unit_idx];
let file = &files[member.file];
let start = unit.token_start.min(file.tokens.len());
let end = unit.token_end.min(file.tokens.len()).max(start);
let tokens = &file.tokens[start..end];
unit_fingerprint(variant, &contexts[member.file], tokens, ContentNorm::Raw)
})
})
.collect();
let members = member_fps
.iter()
.zip(hosts.iter())
.enumerate()
.map(|(i, (content, host))| {
let rank = hosts[..i].iter().filter(|h| *h == host).count();
MemberIds {
content: *content,
finding: finding_id(
&fingerprint,
host.as_ref(),
u32::try_from(rank).unwrap_or(u32::MAX),
),
}
})
.collect();
GroupIds {
fingerprint,
members,
}
})
.collect()
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::unwrap_used)]
mod tests;