use std::convert::identity;
use crate::{
models::{mastodon::MastodonEntry, Domain, FilterOpt, List, Search, SearchState},
sources::manager::SourceManagerHandle,
};
const MASTODON_CSV_HEADER: &str =
"domain,severity,public_comment,reject_media,reject_reports,obfuscate";
pub async fn fetch_all(opts: FilterOpt, handles: Vec<SourceManagerHandle>) -> anyhow::Result<List> {
let mut list = List::empty();
for handle in handles {
match handle.fetch().await {
Ok(ls) => list.merge(ls),
Err(err) => {
tracing::error!("failed to fetch list -- {err}");
}
}
}
if opts.dedup.is_some_and(identity) {
list.domains.sort();
list.domains.dedup();
}
Ok(list)
}
pub async fn fetch_all_as_mastodon_csv(
opts: FilterOpt,
handles: Vec<SourceManagerHandle>,
) -> anyhow::Result<String> {
let list = fetch_all(opts, handles).await?;
let entries: Vec<MastodonEntry> = list.domains.into_iter().map(Domain::into).collect();
let mut csv = String::from(MASTODON_CSV_HEADER);
for entry in entries {
let line = format!("\n{}", entry.as_csv_entry());
csv.push_str(&line);
}
Ok(csv)
}
pub async fn search_domain(
query: Search,
handles: Vec<SourceManagerHandle>,
) -> anyhow::Result<List> {
let list = fetch_all(query.clone().into(), handles).await?;
let state = SearchState::new(&list);
let mut domains: Vec<Domain> = vec![];
let filters = Search::criteria();
for domain in list.domains {
if filters.iter().all(|filter| filter(&query, &domain, &state)) {
domains.push(domain);
}
}
Ok(domains.into())
}
#[cfg(test)]
mod tests {
use async_trait::async_trait;
use crate::{
models::{Domain, FilterOpt, Level, List, Search},
sources::{manager, Source},
};
const LIFETIME: u64 = 600;
const DOMAIN_A: &str = "domain.a";
const DOMAIN_B: &str = "domain.b";
const DOMAIN_C: &str = "domain.c";
const SOURCE_A: &str = "source.a";
const SOURCE_B: &str = "source.b";
const SOURCE_C: &str = "source.c";
fn domain(d: &str, s: &str) -> Domain {
Domain {
domain: d.into(),
level: Level::Severe,
notes: None,
reason: None,
source: s.into(),
}
}
fn with_level(mut domain: Domain, level: Level) -> Domain {
domain.level = level;
domain
}
fn simple_domain_collection() -> Vec<Domain> {
vec![
domain(DOMAIN_A, SOURCE_A),
domain(DOMAIN_B, SOURCE_B),
with_level(domain(DOMAIN_C, SOURCE_C), Level::Low),
]
}
fn duplicate_entries() -> Vec<Domain> {
let mut col = simple_domain_collection();
let mut dup = simple_domain_collection();
col.append(&mut dup);
col
}
fn duplicate_domains() -> Vec<Domain> {
let mut col = simple_domain_collection();
col.push(domain(DOMAIN_A, SOURCE_B));
col
}
fn duplicated_sources() -> Vec<Domain> {
let mut col = simple_domain_collection();
col.push(domain(DOMAIN_B, SOURCE_C));
col
}
#[derive(Debug)]
enum TestSource {
Simple,
DuplicateEntry,
DuplicateDomains,
DuplicateSources,
}
#[derive(Debug, thiserror::Error)]
#[error("oops")]
struct SomeError;
#[async_trait]
impl Source for TestSource {
type Error = SomeError;
async fn fetch(&self) -> Result<List, Self::Error> {
Ok(List {
domains: match self {
TestSource::Simple => simple_domain_collection(),
TestSource::DuplicateDomains => duplicate_domains(),
TestSource::DuplicateSources => duplicated_sources(),
TestSource::DuplicateEntry => duplicate_entries(),
},
})
}
fn lifetime(&self) -> u64 {
LIFETIME
}
}
#[tokio::test]
async fn can_list_all_entries() -> anyhow::Result<()> {
let handle = manager::spawn(TestSource::Simple).await?;
let list = super::fetch_all(Default::default(), vec![handle]).await?;
let expected = vec![
domain(DOMAIN_A, SOURCE_A),
domain(DOMAIN_B, SOURCE_B),
with_level(domain(DOMAIN_C, SOURCE_C), Level::Low),
];
assert_eq!(3, list.domains.len());
for domain in expected {
assert!(list.domains.contains(&domain));
}
Ok(())
}
#[tokio::test]
async fn can_dedup_entries() -> anyhow::Result<()> {
let handle = manager::spawn(TestSource::DuplicateEntry).await?;
let list = super::fetch_all(FilterOpt { dedup: Some(true) }, vec![handle]).await?;
let expected = vec![
domain(DOMAIN_A, SOURCE_A),
domain(DOMAIN_B, SOURCE_B),
with_level(domain(DOMAIN_C, SOURCE_C), Level::Low),
];
assert_eq!(3, list.domains.len());
for domain in expected {
assert!(list.domains.contains(&domain));
}
Ok(())
}
#[tokio::test]
async fn can_filter_by_domain() -> anyhow::Result<()> {
let handle = manager::spawn(TestSource::DuplicateDomains).await?;
let search = Search {
domain: Some(DOMAIN_A.into()),
..Default::default()
};
let list = super::search_domain(search, vec![handle.clone()]).await?;
let expected = vec![domain(DOMAIN_A, SOURCE_A), domain(DOMAIN_A, SOURCE_B)];
assert_eq!(expected, list.domains);
let search = Search {
domain: Some("made.up.domain".into()),
..Default::default()
};
let list = super::search_domain(search, vec![handle]).await?;
assert!(list.domains.is_empty());
Ok(())
}
#[tokio::test]
async fn can_filter_by_source() -> anyhow::Result<()> {
let handle = manager::spawn(TestSource::DuplicateSources).await?;
let search = Search {
source: Some(SOURCE_C.into()),
..Default::default()
};
let list = super::search_domain(search, vec![handle.clone()]).await?;
let expected = vec![
with_level(domain(DOMAIN_C, SOURCE_C), Level::Low),
domain(DOMAIN_B, SOURCE_C),
];
assert_eq!(2, list.domains.len());
assert!(list.domains.contains(&expected[0]));
assert!(list.domains.contains(&expected[1]));
let search = Search {
source: Some("made.up.source".into()),
..Default::default()
};
let list = super::search_domain(search, vec![handle]).await?;
assert!(list.domains.is_empty());
Ok(())
}
#[tokio::test]
async fn can_filter_by_threshold() -> anyhow::Result<()> {
let handle = manager::spawn(TestSource::DuplicateSources).await?;
let search = Search {
threshold: Some(2),
..Default::default()
};
let list = super::search_domain(search, vec![handle.clone()]).await?;
let expected = vec![domain(DOMAIN_B, SOURCE_B), domain(DOMAIN_B, SOURCE_C)];
assert_eq!(expected, list.domains);
let search = Search {
threshold: Some(3),
..Default::default()
};
let list = super::search_domain(search, vec![handle]).await?;
assert!(list.domains.is_empty());
Ok(())
}
#[tokio::test]
async fn can_filter_by_level() -> anyhow::Result<()> {
let handle = manager::spawn(TestSource::Simple).await?;
let search = Search {
level: Some(Level::Severe),
..Default::default()
};
let list = super::search_domain(search, vec![handle.clone()]).await?;
assert_eq!(2, list.domains.len());
assert!(list.domains.contains(&domain(DOMAIN_A, SOURCE_A)));
assert!(list.domains.contains(&domain(DOMAIN_B, SOURCE_B)));
let search = Search {
level: Some(Level::Low),
..Default::default()
};
let list = super::search_domain(search, vec![handle]).await?;
assert_eq!(1, list.domains.len());
assert!(list
.domains
.contains(&with_level(domain(DOMAIN_C, SOURCE_C), Level::Low)));
Ok(())
}
}