use crate::{
ColumnKind, ColumnShape, ConfidenceFilter, ConfidenceOperand, ScalarFilter, Score, width,
};
use super::{DetectedLanguage, DetectedLanguageFilter};
use crate::primitives::lang::{Language, LanguageId, LanguageIdFilter, ScriptSubtag};
fn tag(sent: &str) -> LanguageId {
LanguageId::new(sent).unwrap_or_else(|refused| panic!("`{sent}` is a language tag: {refused}"))
}
fn sure(value: f32) -> Score {
Score::new(value).expect("a confidence in the unit interval")
}
fn threshold(value: f64) -> ConfidenceOperand {
ConfidenceOperand::number(value).expect("a threshold in [0, 1]")
}
fn detected(sent: &str, confidence: f32) -> DetectedLanguage {
DetectedLanguage::new(tag(sent), sure(confidence))
}
#[test]
fn a_bare_code_and_a_scripted_one_are_the_same_carrier() {
let whisper = detected("zh", 0.94);
assert_eq!(whisper.language().language().as_str(), "zh");
assert_eq!(whisper.language().script(), None);
let finer = detected("zh-Hans", 0.94);
assert_eq!(
finer.language().script().map(ScriptSubtag::as_str),
Some("Hans")
);
assert_ne!(whisper, finer);
assert_eq!(whisper.confidence(), finer.confidence());
}
#[test]
fn the_confidence_is_part_of_the_identity() {
assert_ne!(detected("de", 0.9), detected("de", 0.2));
assert_eq!(detected("de", 0.9), detected("de", 0.9));
assert_eq!(detected("ger", 0.9), detected("deu", 0.9));
let mut distinct = std::collections::HashSet::new();
distinct.insert(detected("ger", 0.9));
distinct.insert(detected("deu", 0.9));
distinct.insert(detected("de", 0.2));
assert_eq!(distinct.len(), 2);
}
#[test]
fn a_detection_spans_the_identitys_four_columns_and_one_more() {
let segments = <DetectedLanguage as ColumnKind>::SEGMENTS;
assert_eq!(segments.len(), 2, "one composite and one plain");
assert_eq!(width(segments), 5, "and five columns between them");
let ColumnShape::Composite {
field,
segments: inner,
} = segments[0]
else {
panic!("the identity is a composite");
};
assert_eq!(field, None, "FLATTENED — it lends no segment of its own");
assert_eq!(inner, <LanguageId as ColumnKind>::SEGMENTS);
let ColumnShape::Plain(confidence) = segments[1] else {
panic!("the confidence is one plain column");
};
assert_eq!(confidence.stored().as_str(), "confidence");
}
#[cfg(feature = "mongo")]
#[test]
fn a_detection_carries_as_the_identitys_keys_and_one_more() {
use crate::MongoCarrier;
use bson::Bson;
let Bson::Document(carried) = detected("zh-Hans", 0.94).to_bson() else {
panic!("a detection carries as a document");
};
let keys: Vec<&str> = carried.keys().map(String::as_str).collect();
assert_eq!(
keys,
["language", "script", "region", "rest", "confidence"],
"the identity's four keys at THIS level, then the confidence",
);
assert_eq!(carried.get("language"), Some(&Bson::from("zh")));
assert_eq!(carried.get("script"), Some(&Bson::from("Hans")));
assert_eq!(
carried.get("confidence"),
Some(&Bson::Double(f64::from(0.94_f32))),
"the width's own row — BSON's only binary float, which an `f32` widens into exactly",
);
let Bson::Document(alone) = tag("zh-Hans").to_bson() else {
panic!("an identity carries as a sub-document");
};
for seat in alone.keys() {
assert_eq!(
carried.get(seat),
alone.get(seat),
"`{seat}` says the same thing in both",
);
}
assert_eq!(
carried.len(),
alone.len() + 1,
"and the detection adds exactly one key to it",
);
}
#[cfg(feature = "mongo")]
#[test]
fn a_detection_reads_back_the_document_it_carried_as() {
use crate::{MongoCarrier, query::mongo::MongoExtract};
for (sent, confidence) in [
("zh", 0.94),
("zh-Hans", 0.94),
("de-DE", 0.5),
("und", 0.0),
] {
let value = detected(sent, confidence);
let written = value.to_bson();
assert_eq!(
DetectedLanguage::extract(Some(&written)),
Ok(value),
"`{sent}`"
);
}
}
#[cfg(feature = "mongo")]
#[test]
fn a_detection_read_refuses_absence_and_the_wrong_type() {
use crate::query::mongo::{MongoExtract, MongoTypeError};
use bson::Bson;
assert_eq!(DetectedLanguage::extract(None), Err(MongoTypeError::Absent));
assert_eq!(
DetectedLanguage::extract(Some(&Bson::Int32(1))),
Err(MongoTypeError::wrong_type("object", "int"))
);
}
#[cfg(feature = "mongo")]
#[test]
fn an_out_of_domain_confidence_is_refused_by_scores_own_rule() {
use crate::query::mongo::{MongoExtract, MongoTypeError};
use bson::doc;
let document = doc! {
"language": "zh", "script": bson::Bson::Null, "region": bson::Bson::Null,
"rest": bson::Bson::Null, "confidence": 2.0
};
assert!(matches!(
DetectedLanguage::extract(Some(&document.into())),
Err(MongoTypeError::Range(_))
));
}
#[test]
fn the_filter_holds_a_whole_identity_filter_and_a_score_filter() {
let empty = DetectedLanguageFilter::new();
assert!(empty.is_empty());
assert!(empty.language().is_empty());
assert_eq!(empty.confidence(), None);
let chinese = DetectedLanguageFilter::new().with_language(
LanguageIdFilter::new()
.with_language(ScalarFilter::new().with_eq(Language::new("zh").expect("a language"))),
);
assert!(!chinese.is_empty());
assert!(!chinese.language().is_empty());
let confident =
DetectedLanguageFilter::new().with_confidence(ConfidenceFilter::new().with_gte(threshold(0.8)));
assert!(!confident.is_empty());
assert_eq!(
confident.confidence().and_then(ConfidenceFilter::gte),
Some(&threshold(0.8))
);
let hollow = DetectedLanguageFilter::new()
.with_language(LanguageIdFilter::new())
.with_confidence(ConfidenceFilter::new());
assert!(hollow.is_empty());
}
#[test]
fn a_detection_knows_nothing_about_its_rank() {
let candidates = [
detected("zh", 0.94),
detected("ja", 0.04),
detected("ko", 0.02),
];
assert_eq!(candidates[0].confidence(), sure(0.94));
assert!(
candidates
.windows(2)
.all(|pair| pair[0].confidence() >= pair[1].confidence())
);
let mut sorted = candidates.clone();
sorted.sort();
assert_eq!(sorted[0].language().language().as_str(), "ja");
}
#[test]
fn the_rendering_is_a_rendering_and_not_a_round_trip() {
let detection = detected("zh-Hans", 0.94);
assert_eq!(detection.to_string(), "zh-Hans (0.94)");
assert_eq!(detection.language().to_string(), "zh-Hans");
assert_eq!(
LanguageId::new(&detection.language().to_string()).expect("canonical"),
*detection.language()
);
assert_eq!(
format!("{detection:?}"),
r#"DetectedLanguage { language: "zh-Hans", confidence: 0.94 }"#
);
}
#[test]
fn the_constructor_is_total_so_an_upstream_conversion_cannot_fail() {
let language = tag("zh");
let confidence = sure(0.94);
let detection = DetectedLanguage::new(language.clone(), confidence);
assert_eq!(detection.language(), &language);
assert_eq!(detection.confidence(), confidence);
}