use crate::types::{Address, FileResult, HashAlgos};
use std::collections::HashMap;
#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct MergedResult {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub size: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mimetype: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub lang: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub published: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_seen: Option<String>,
#[serde(default, skip_serializing_if = "HashAlgos::is_empty")]
pub hashes: HashAlgos,
pub providers: Vec<String>,
pub addresses: Vec<Address>,
pub score: f64,
}
#[derive(Debug, Clone, Default)]
pub struct MergeOptions {
pub provider_weights: HashMap<String, f64>,
}
pub fn merge<I>(results: I, options: &MergeOptions) -> Vec<MergedResult>
where
I: IntoIterator<Item = FileResult>,
{
let mut clusters: Vec<MergedResult> = Vec::new();
let mut index: HashMap<String, usize> = HashMap::new();
for result in results {
let keys = dedup_keys(&result);
let position = match keys.iter().find_map(|key| index.get(key).copied()) {
Some(position) => position,
None => {
clusters.push(MergedResult::default());
clusters.len() - 1
}
};
absorb(&mut clusters[position], result);
for key in keys {
index.insert(key, position);
}
}
for cluster in &mut clusters {
cluster.providers.sort();
cluster.providers.dedup();
cluster.score = score(cluster, &options.provider_weights);
}
clusters.sort_by(|a, b| {
b.score
.total_cmp(&a.score)
.then_with(|| a.title.cmp(&b.title))
});
clusters
}
fn dedup_keys(result: &FileResult) -> Vec<String> {
let mut keys = Vec::new();
if let Some(hashes) = &result.hashes {
for algorithm in ["sha256", "sha1", "infohash", "md5", "cid"] {
if let Some(value) = hashes.get(algorithm) {
keys.push(format!("{algorithm}:{value}"));
}
}
}
if keys.is_empty() {
if let (Some(title), Some(size)) = (&result.title, result.size) {
keys.push(format!("name:{}:{size}", normalize(title)));
}
}
keys
}
fn normalize(title: &str) -> String {
title
.trim()
.to_lowercase()
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
fn absorb(cluster: &mut MergedResult, result: FileResult) {
if cluster.title.is_none() {
cluster.title = result.title;
}
if cluster.size.is_none() {
cluster.size = result.size;
}
if cluster.mimetype.is_none() {
cluster.mimetype = result.mimetype;
}
if cluster.lang.is_none() {
cluster.lang = result.lang;
}
if cluster.published.is_none() {
cluster.published = result.published;
}
if cluster.last_seen.is_none() {
cluster.last_seen = result.last_seen;
}
if let Some(hashes) = result.hashes {
for (algorithm, value) in hashes {
cluster.hashes.entry(algorithm).or_insert(value);
}
}
if !cluster.providers.contains(&result.provider) {
cluster.providers.push(result.provider);
}
for address in result.addresses {
if !cluster
.addresses
.iter()
.any(|existing| existing.uri == address.uri)
{
cluster.addresses.push(address);
}
}
}
fn score(cluster: &MergedResult, weights: &HashMap<String, f64>) -> f64 {
let mut score: f64 = cluster
.providers
.iter()
.map(|provider| weights.get(provider).copied().unwrap_or(1.0))
.sum();
if !cluster.hashes.is_empty() {
score += 0.5;
}
if cluster.mimetype.is_some() {
score += 0.25;
}
if cluster.size.is_some() {
score += 0.25;
}
if cluster.published.is_some() {
score += 0.1;
}
score
}