use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::OnceLock;
use aho_corasick::{AhoCorasick, AhoCorasickBuilder};
use regex::{Regex, RegexBuilder};
use serde::{Deserialize, Serialize};
#[cfg(test)]
const TROPES_MD_FILE: &str = include_str!("tropes/tropes.md");
const BUILT_IN_RULES_JSON: &str = include_str!("tropes/rules.json");
static BUILT_IN_RULES: OnceLock<Result<Vec<BuiltInTropeRule>, String>> = OnceLock::new();
static PHRASE_SCANNER: OnceLock<Result<PhraseScanner, String>> = OnceLock::new();
static REGEX_SCANNER: OnceLock<Result<RegexScanner, String>> = OnceLock::new();
/// Built-in scanning preset.
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TropePreset {
/// Fewer findings, biased toward stronger evidence.
Lenient,
/// Balanced default for article and essay diagnostics.
#[default]
Balanced,
/// More findings, useful for editing passes.
Strict,
}
impl TropePreset {
fn looks_like_false_range(self, text: &str) -> bool {
let lower = text.to_ascii_lowercase();
let Some(from) = lower.find("from ") else {
return false;
};
let Some(to_relative) = lower[from + 5..].find(" to ") else {
return false;
};
if !lower[..from].trim().is_empty() {
return false;
}
let to = from + 5 + to_relative;
let left = &lower[from + 5..to];
let right = &lower[to + 4..];
let total_words = words(left).len() + words(right).len();
if total_words < 2 || lower.chars().any(|ch| ch.is_ascii_digit()) {
return false;
}
let abstract_markers = [
"innovation",
"implementation",
"transformation",
"singularity",
"cosmic",
"problem-solving",
"scientific",
"artistic",
"technological",
"cultural",
"discovery",
];
let marker_match = abstract_markers.iter().any(|marker| lower.contains(marker));
let list_like = left.contains(',') || right.contains(',') || left.contains(" and ") || right.contains(" and ");
match self {
TropePreset::Lenient => marker_match && list_like && total_words >= 6,
TropePreset::Balanced => marker_match,
TropePreset::Strict => marker_match || list_like || total_words >= 8,
}
}
fn document_occurrence_threshold(self, balanced: usize, lenient: usize, strict: usize) -> usize {
match self {
TropePreset::Lenient => lenient,
TropePreset::Balanced => balanced,
TropePreset::Strict => strict,
}
}
fn em_dash_threshold(self, word_count: usize) -> usize {
match self {
TropePreset::Lenient => 5.max(word_count / 100),
TropePreset::Balanced => 3.max(word_count / 150),
TropePreset::Strict => 2.max(word_count / 300),
}
}
}
/// Broad category for a trope rule or detector.
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename_all = "snake_case")]
pub enum TropeCategory {
/// Overused words, pompous substitutions, or vague vocabulary.
WordChoice,
/// Sentence-level rhetorical structures.
SentenceStructure,
/// Paragraph-level repetition or layout patterns.
ParagraphStructure,
/// Tone, attribution, and register issues.
Tone,
/// Markdown, punctuation, or visual formatting tells.
Formatting,
/// Document-level composition patterns.
Composition,
}
/// Coarse trope signal for a report.
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TropeSignal {
/// Little or no evidence found.
#[default]
Low,
/// Some evidence found.
Medium,
/// Strong evidence found.
High,
/// Very strong evidence found.
VeryHigh,
}
impl From<u8> for TropeSignal {
fn from(score: u8) -> Self {
match score {
0..=19 => TropeSignal::Low,
20..=44 => TropeSignal::Medium,
45..=74 => TropeSignal::High,
_ => TropeSignal::VeryHigh,
}
}
}
/// Severity of one finding.
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename_all = "snake_case")]
pub enum Severity {
/// Mild signal or low-confidence wording issue.
#[default]
Low,
/// Clear signal worth review.
Medium,
/// Strong signal likely worth editing.
High,
}
impl Severity {
fn weight(self) -> usize {
match self {
Severity::Low => 1,
Severity::Medium => 3,
Severity::High => 6,
}
}
}
/// Type of built-in rule.
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TropeRuleKind {
/// Exact word or phrase rule.
Phrase,
/// Regular expression rule.
Regex,
/// Hand-written structural detector rule.
Structure,
/// Whole-document detector rule.
Document,
}
/// Implementation slice a built-in rule belongs to.
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TropeRuleScope {
/// First scanner slice.
Seed,
/// Catalog rule or detector covered by later scanner phases.
Catalog,
}
/// Built-in trope rule loaded from the bundled catalog data.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct BuiltInTropeRule {
/// Stable rule identifier.
pub id: String,
/// Broad trope category.
pub category: TropeCategory,
/// Scanner implementation kind.
pub kind: TropeRuleKind,
/// Rule severity.
pub severity: Severity,
/// Whether this rule is part of the first scanner slice.
pub scope: TropeRuleScope,
/// Exact phrases, regex strings, or detector examples.
pub patterns: Vec<String>,
/// Human-readable explanation.
pub message: String,
/// Optional editing suggestion.
pub suggestion: Option<String>,
/// Source provenance in the vendored Markdown catalog.
pub source: RuleSource,
}
impl BuiltInTropeRule {
fn scan_em_dash_addiction(&self, i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let spans = literal_spans(i.text, "—")
.into_iter()
.chain(literal_spans(i.text, "--"))
.collect::<Vec<_>>();
let minimum = opts.preset.em_dash_threshold(i.text.split_whitespace().count());
if spans.len() >= minimum {
for span in spans {
self.push_finding(i, span.start_byte, span.end_byte, opts, findings);
}
}
}
fn scan_repeated_document_phrases(&self, i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let spans = self
.patterns
.iter()
.flat_map(|pattern| literal_spans(i.text, pattern))
.collect::<Vec<_>>();
let minimum = opts.preset.document_occurrence_threshold(2, 3, 1);
if spans.len() >= minimum {
for span in spans {
self.push_finding(i, span.start_byte, span.end_byte, opts, findings);
}
}
}
fn scan_dead_metaphor(&self, i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let minimum = opts.preset.document_occurrence_threshold(3, 4, 2);
for pattern in &self.patterns {
let spans = literal_spans(i.text, pattern);
if spans.len() >= minimum {
for span in spans {
self.push_finding(i, span.start_byte, span.end_byte, opts, findings);
}
}
}
}
fn scan_historical_analogy_stacking(
&self, i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
) {
let spans = self
.patterns
.iter()
.flat_map(|pattern| literal_spans(i.text, pattern))
.collect::<Vec<_>>();
let minimum = opts.preset.document_occurrence_threshold(2, 3, 2);
if spans.len() >= minimum {
for span in spans {
self.push_finding(i, span.start_byte, span.end_byte, opts, findings);
}
}
}
fn scan_one_point_dilution(&self, i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let sentences = split_paragraphs(i.text)
.into_iter()
.flat_map(|paragraph| split_sentences(i.text, paragraph))
.collect::<Vec<_>>();
let minimum_distance = match opts.preset {
TropePreset::Lenient => 3,
TropePreset::Balanced | TropePreset::Strict => 1,
};
for (left_index, left) in sentences.iter().enumerate() {
let left_words = content_word_set((*left).text(i.text));
if left_words.len() < 4 {
continue;
}
for right in sentences.iter().skip(left_index + minimum_distance) {
let right_words = content_word_set((*right).text(i.text));
if right_words.len() < 4 || left_words == right_words {
continue;
}
let overlap = left_words.intersection(&right_words).count();
let union = left_words.union(&right_words).count();
if union > 0 && overlap * 100 >= union * 60 {
self.push_finding(i, right.start_byte, right.end_byte, opts, findings);
break;
}
}
}
}
fn scan_content_duplication(&self, i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let sentences = split_paragraphs(i.text)
.into_iter()
.flat_map(|paragraph| split_sentences(i.text, paragraph))
.collect::<Vec<_>>();
let mut seen = BTreeMap::new();
for sentence in sentences {
let normalized = normalize_for_comparison(sentence.text(i.text));
if words(&normalized).len() < 4 {
continue;
}
if seen.insert(normalized, sentence).is_some() {
self.push_finding(i, sentence.start_byte, sentence.end_byte, opts, findings);
}
}
}
fn finding_for_rule(
&self, i: &TextIndex<'_>, start_byte: usize, end_byte: usize, opts: &TropeOptions,
) -> Option<TropeFinding> {
Some(TropeFinding {
rule_id: self.id.clone(),
category: self.category,
severity: self.severity,
range: i.range(start_byte, end_byte)?,
matched_text: i.matched_text(start_byte, end_byte)?.to_string(),
message: self.message.clone(),
suggestion: opts.include_suggestions.then(|| self.suggestion.clone()).flatten(),
source: Some(self.source.clone()),
})
}
fn push_finding(
&self, i: &TextIndex<'_>, start_byte: usize, end_byte: usize, opts: &TropeOptions,
findings: &mut Vec<TropeFinding>,
) {
if let Some(finding) = self.finding_for_rule(i, start_byte, end_byte, opts) {
findings.push(finding);
}
}
}
/// Options for trope diagnostics.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct TropeOptions {
/// Built-in rule preset used for scanning.
#[serde(default)]
pub preset: TropePreset,
/// Lowest severity included in the report.
#[serde(default)]
pub min_severity: Severity,
/// Include rule suggestions when the matched rule provides one.
#[serde(default = "default_include_suggestions")]
pub include_suggestions: bool,
}
impl Default for TropeOptions {
fn default() -> Self {
Self { preset: TropePreset::default(), min_severity: Severity::default(), include_suggestions: true }
}
}
impl TropeOptions {
fn effective_min_severity(&self) -> Severity {
match self.preset {
TropePreset::Lenient => self.min_severity.max(Severity::Medium),
TropePreset::Balanced | TropePreset::Strict => self.min_severity,
}
}
}
/// Complete trope diagnostics report.
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct TropeReport {
/// Heuristic density score from 0 to 100. This is not an authorship
/// probability.
pub score: u8,
/// Coarse evidence bucket derived from score and finding severity.
pub signal: TropeSignal,
/// Individual matched rules and detectors.
pub findings: Vec<TropeFinding>,
/// Aggregate counts and density measurements.
pub summary: TropeSummary,
}
/// One trope finding in the input text.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct TropeFinding {
/// Stable rule identifier.
pub rule_id: String,
/// Broad trope category for grouping and summaries.
pub category: TropeCategory,
/// Finding severity.
pub severity: Severity,
/// Location of the match in the original input.
pub range: TextRange,
/// Exact text matched from the original input.
pub matched_text: String,
/// Human-readable explanation of the finding.
pub message: String,
/// Optional editing suggestion.
pub suggestion: Option<String>,
/// Built-in rule provenance, when available.
pub source: Option<RuleSource>,
}
/// Aggregate report measurements.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct TropeSummary {
/// Number of whitespace-delimited words in the input.
pub word_count: usize,
/// Number of findings included in the report.
pub finding_count: usize,
/// Findings per 1,000 words.
pub findings_per_1000_words: f32,
/// Findings grouped by category.
pub category_counts: BTreeMap<TropeCategory, usize>,
/// Findings grouped by severity.
pub severity_counts: BTreeMap<Severity, usize>,
/// Highest-signal categories, ordered by count.
pub top_categories: Vec<TropeCategory>,
/// Rule ids that appeared more than once.
pub repeated_rules: Vec<String>,
}
impl TropeSummary {
fn from_findings(text: &str, findings: &[TropeFinding]) -> Self {
let word_count = text.split_whitespace().count();
let finding_count = findings.len();
let findings_per_1000_words =
if word_count == 0 { 0.0 } else { finding_count as f32 * 1000.0 / word_count as f32 };
let mut category_counts = BTreeMap::new();
let mut severity_counts = BTreeMap::new();
let mut rule_counts = BTreeMap::new();
for finding in findings {
*category_counts.entry(finding.category).or_insert(0) += 1;
*severity_counts.entry(finding.severity).or_insert(0) += 1;
*rule_counts.entry(finding.rule_id.as_str()).or_insert(0) += 1;
}
let mut top_categories = category_counts
.iter()
.map(|(category, count)| (*category, *count))
.collect::<Vec<_>>();
top_categories.sort_by(|(left_category, left_count), (right_category, right_count)| {
right_count
.cmp(left_count)
.then_with(|| left_category.cmp(right_category))
});
Self {
word_count,
finding_count,
findings_per_1000_words,
category_counts,
severity_counts,
top_categories: top_categories.into_iter().map(|(category, _)| category).collect(),
repeated_rules: rule_counts
.into_iter()
.filter(|(_, count)| *count > 1)
.map(|(rule_id, _)| rule_id.to_string())
.collect(),
}
}
fn score(&self, findings: &[TropeFinding]) -> u8 {
if self.finding_count == 0 {
return 0;
}
let adjusted_words = self.word_count.max(200) as f32;
let adjusted_density = self.finding_count as f32 * 1000.0 / adjusted_words;
let mut rule_severities: BTreeMap<&str, Severity> = BTreeMap::new();
for finding in findings {
rule_severities
.entry(finding.rule_id.as_str())
.and_modify(|severity| *severity = (*severity).max(finding.severity))
.or_insert(finding.severity);
}
let severity_points = rule_severities
.values()
.map(|severity| severity.weight())
.sum::<usize>() as f32;
(adjusted_density * 2.0 + severity_points * 4.0)
.round()
.clamp(0.0, 100.0) as u8
}
}
impl Default for TropeSummary {
fn default() -> Self {
Self {
word_count: 0,
finding_count: 0,
findings_per_1000_words: 0.0,
category_counts: BTreeMap::new(),
severity_counts: BTreeMap::new(),
top_categories: Vec::new(),
repeated_rules: Vec::new(),
}
}
}
/// Location of a match in the original input.
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct TextRange {
/// Start byte offset.
pub start_byte: usize,
/// End byte offset.
pub end_byte: usize,
/// Start UTF-16 code-unit offset.
pub start_utf16: usize,
/// End UTF-16 code-unit offset.
pub end_utf16: usize,
/// 1-based start line.
pub start_line: usize,
/// 1-based start column.
pub start_column: usize,
/// 1-based exclusive end line.
pub end_line: usize,
/// 1-based exclusive end column.
pub end_column: usize,
}
/// Byte, UTF-16, and line/column index for an input string.
pub struct TextIndex<'a> {
text: &'a str,
line_starts: Vec<usize>,
utf16_offsets: Vec<(usize, usize)>,
}
impl<'a> TextIndex<'a> {
pub fn new(text: &'a str) -> Self {
let mut line_starts = vec![0];
let mut utf16_offsets = vec![(0, 0)];
let mut utf16_offset = 0;
for (byte, ch) in text.char_indices() {
utf16_offset += ch.len_utf16();
utf16_offsets.push((byte + ch.len_utf8(), utf16_offset));
if ch == '\n' {
line_starts.push(byte + ch.len_utf8());
}
}
Self { text, line_starts, utf16_offsets }
}
pub fn range(&self, start_byte: usize, end_byte: usize) -> Option<TextRange> {
if start_byte > end_byte {
return None;
}
let (start_line, start_column) = self.position(start_byte)?;
let (end_line, end_column) = self.position(end_byte)?;
Some(TextRange {
start_byte,
end_byte,
start_utf16: self.utf16_offset(start_byte)?,
end_utf16: self.utf16_offset(end_byte)?,
start_line,
start_column,
end_line,
end_column,
})
}
pub fn matched_text(&self, start_byte: usize, end_byte: usize) -> Option<&'a str> {
self.range(start_byte, end_byte)?;
self.text.get(start_byte..end_byte)
}
fn position(&self, byte: usize) -> Option<(usize, usize)> {
if byte > self.text.len() || !self.text.is_char_boundary(byte) {
return None;
}
let line_index = match self.line_starts.binary_search(&byte) {
Ok(index) => index,
Err(index) => index.checked_sub(1)?,
};
let line_start = self.line_starts[line_index];
let column = self.text.get(line_start..byte)?.chars().count() + 1;
Some((line_index + 1, column))
}
fn utf16_offset(&self, byte: usize) -> Option<usize> {
self.utf16_offsets
.binary_search_by_key(&byte, |(indexed_byte, _)| *indexed_byte)
.ok()
.map(|index| self.utf16_offsets[index].1)
}
}
/// Provenance for a built-in trope rule.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct RuleSource {
/// Repository-relative source file.
pub file: String,
/// Heading in the source file that produced the rule.
pub heading: String,
}
struct PhraseScanner {
matcher: AhoCorasick,
pattern_rules: Vec<usize>,
}
struct RegexScanner {
rules: Vec<RegexRule>,
}
struct RegexRule {
rule_index: usize,
patterns: Vec<Regex>,
}
#[derive(Clone, Copy)]
struct TextSpan {
start_byte: usize,
end_byte: usize,
}
impl TextSpan {
fn normalize(self, text: &str) -> String {
self.text(text).to_ascii_lowercase()
}
fn text(self, text: &str) -> &str {
text.get(self.start_byte..self.end_byte).unwrap_or("")
}
}
/// Analyze text for AI-like writing tropes.
pub fn analyze_tropes(text: &str, opts: &TropeOptions) -> TropeReport {
let index = TextIndex::new(text);
let mut findings = Vec::new();
scan_phrase_rules(text, &index, opts, &mut findings);
scan_regex_rules(text, &index, opts, &mut findings);
scan_structure_rules(text, &index, opts, &mut findings);
scan_document_rules(&index, opts, &mut findings);
deduplicate_findings(&mut findings);
sort_findings(&mut findings);
let summary = TropeSummary::from_findings(text, &findings);
let score = summary.score(&findings);
let signal = TropeSignal::from(score);
TropeReport { score, signal, summary, findings }
}
fn built_in_trope_rules() -> Result<&'static [BuiltInTropeRule], &'static str> {
match BUILT_IN_RULES.get_or_init(|| serde_json::from_str(BUILT_IN_RULES_JSON).map_err(|error| error.to_string())) {
Ok(rules) => Ok(rules.as_slice()),
Err(error) => Err(error.as_str()),
}
}
fn phrase_scanner() -> Result<&'static PhraseScanner, &'static str> {
match PHRASE_SCANNER.get_or_init(|| {
let rules = built_in_trope_rules().map_err(str::to_string)?;
let mut patterns = Vec::new();
let mut pattern_rules = Vec::new();
for (rule_index, rule) in rules.iter().enumerate() {
if rule.kind == TropeRuleKind::Phrase {
for pattern in &rule.patterns {
patterns.push(pattern.clone());
pattern_rules.push(rule_index);
}
}
}
let matcher = AhoCorasickBuilder::new()
.ascii_case_insensitive(true)
.build(patterns)
.map_err(|error| error.to_string())?;
Ok(PhraseScanner { matcher, pattern_rules })
}) {
Ok(scanner) => Ok(scanner),
Err(error) => Err(error.as_str()),
}
}
fn regex_scanner() -> Result<&'static RegexScanner, &'static str> {
let result = REGEX_SCANNER.get_or_init(|| {
let rules = built_in_trope_rules().map_err(str::to_string)?;
let mut regex_rules = Vec::new();
for (rule_index, rule) in rules.iter().enumerate() {
if rule.kind == TropeRuleKind::Regex {
let mut patterns = Vec::new();
for pattern in &rule.patterns {
patterns.push(
RegexBuilder::new(pattern)
.case_insensitive(true)
.multi_line(true)
.build()
.map_err(|error| error.to_string())?,
);
}
regex_rules.push(RegexRule { rule_index, patterns });
}
}
Ok(RegexScanner { rules: regex_rules })
});
match result {
Ok(scanner) => Ok(scanner),
Err(error) => Err(error.as_str()),
}
}
fn scan_phrase_rules(text: &str, index: &TextIndex<'_>, options: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let Ok(rules) = built_in_trope_rules() else {
return;
};
let Ok(scanner) = phrase_scanner() else {
return;
};
for mat in scanner.matcher.find_overlapping_iter(text) {
let Some(&rule_index) = scanner.pattern_rules.get(mat.pattern().as_usize()) else {
continue;
};
let rule = &rules[rule_index];
if rule.severity < options.effective_min_severity() || !has_word_boundaries(text, mat.start(), mat.end()) {
continue;
}
if let Some(finding) = rule.finding_for_rule(index, mat.start(), mat.end(), options) {
findings.push(finding);
}
}
}
fn scan_regex_rules(text: &str, index: &TextIndex<'_>, options: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let Ok(rules) = built_in_trope_rules() else {
return;
};
let Ok(scanner) = regex_scanner() else {
return;
};
for regex_rule in &scanner.rules {
let rule = &rules[regex_rule.rule_index];
if rule.severity < options.effective_min_severity() {
continue;
}
for pattern in ®ex_rule.patterns {
for mat in pattern.find_iter(text) {
if let Some(finding) = rule.finding_for_rule(index, mat.start(), mat.end(), options) {
findings.push(finding);
}
}
}
}
}
fn scan_structure_rules(text: &str, index: &TextIndex<'_>, options: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let Ok(rules) = built_in_trope_rules() else {
return;
};
for paragraph in split_paragraphs(text) {
let sentences = split_sentences(text, paragraph);
scan_not_not_just(rules, index, options, findings, &sentences);
scan_negative_parallelism(rules, index, options, findings, &sentences);
scan_self_posed_questions(rules, index, options, findings, &sentences);
scan_fragment_stacks(rules, index, options, findings, &sentences);
scan_repeated_sentence_openers(rules, index, options, findings, &sentences);
scan_tricolon_abuse(rules, index, options, findings, paragraph, &sentences);
scan_false_ranges(rules, index, options, findings, &sentences);
scan_listicle_in_a_trench_coat(rules, index, options, findings, &sentences);
scan_despite_its_challenges(rules, index, options, findings, &sentences);
}
}
fn scan_document_rules(index: &TextIndex<'_>, options: &TropeOptions, findings: &mut Vec<TropeFinding>) {
let Ok(rules) = built_in_trope_rules() else {
return;
};
let minimum_severity = options.effective_min_severity();
for rule in rules.iter().filter(|rule| rule.kind == TropeRuleKind::Document) {
if rule.severity < minimum_severity {
continue;
}
match rule.id.as_str() {
"em-dash-addiction" => rule.scan_em_dash_addiction(index, options, findings),
"fractal-summaries" => rule.scan_repeated_document_phrases(index, options, findings),
"dead-metaphor" => rule.scan_dead_metaphor(index, options, findings),
"historical-analogy-stacking" => rule.scan_historical_analogy_stacking(index, options, findings),
"one-point-dilution" => rule.scan_one_point_dilution(index, options, findings),
"content-duplication" => rule.scan_content_duplication(index, options, findings),
_ => {}
}
}
}
fn scan_not_not_just(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "not-not-just") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
for sentence in sentences {
let text = (*sentence).normalize(i.text);
if starts_with_not(&text) && text.matches("not ").count() >= 2 && text.contains(", but ") {
rule.push_finding(i, sentence.start_byte, sentence.end_byte, opts, findings);
}
}
let window_size = match opts.preset {
TropePreset::Lenient => 3,
TropePreset::Balanced | TropePreset::Strict => 3,
};
for window in sentences.windows(window_size) {
let first = window[0].normalize(i.text);
let second = window[1].normalize(i.text);
let third = window[2].normalize(i.text);
if starts_with_not(&first)
&& starts_with_not(&second)
&& !starts_with_not(&third)
&& words(&first).len() <= 8
&& words(&second).len() <= 8
{
rule.push_finding(i, window[0].start_byte, window[2].end_byte, opts, findings);
}
}
}
fn scan_tricolon_abuse(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
paragraph: TextSpan, sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "tricolon-abuse") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
let minimum_items = match opts.preset {
TropePreset::Lenient => 5,
TropePreset::Balanced => 4,
TropePreset::Strict => 3,
};
for sentence in sentences {
let sentence_text = (*sentence).text(i.text);
if is_non_prose_line(sentence_text) {
continue;
}
if compact_list_item_count(sentence_text) >= minimum_items {
rule.push_finding(i, sentence.start_byte, sentence.end_byte, opts, findings);
}
}
let mut semicolon_openers = BTreeMap::new();
for sentence in sentences {
let text = (*sentence).text(i.text);
let mut parts = text.split(';');
let Some(left) = parts.next() else { continue };
let Some(right) = parts.next() else { continue };
if parts.next().is_some() || words(left).len() < 3 || words(right).len() < 3 {
continue;
}
if let Some(opener) = sentence_opener(left) {
*semicolon_openers.entry(opener).or_insert(0) += 1;
}
}
let semicolon_sentences = semicolon_openers.values().filter(|count| **count >= 2).sum::<usize>();
let minimum_semicolon_sentences = match opts.preset {
TropePreset::Lenient => 3,
TropePreset::Balanced | TropePreset::Strict => 2,
};
if semicolon_sentences >= minimum_semicolon_sentences {
rule.push_finding(i, paragraph.start_byte, paragraph.end_byte, opts, findings);
}
}
fn scan_false_ranges(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "false-ranges") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
for sentence in sentences {
let sentence_text = (*sentence).text(i.text);
if !opts.preset.looks_like_false_range(sentence_text) {
continue;
}
let lower = sentence_text.to_ascii_lowercase();
let start = lower.find("from ").unwrap_or(0);
rule.push_finding(i, sentence.start_byte + start, sentence.end_byte, opts, findings);
}
}
fn scan_listicle_in_a_trench_coat(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "listicle-in-a-trench-coat") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
let minimum_items = match opts.preset {
TropePreset::Lenient => 4,
TropePreset::Balanced => 3,
TropePreset::Strict => 2,
};
for (start, window) in sentences.windows(minimum_items).enumerate() {
let mut matches = true;
for (index, sentence) in window.iter().enumerate() {
let Some(number) = ordinal_sentence_number((*sentence).text(i.text)) else {
matches = false;
break;
};
if number != index + 1 {
matches = false;
break;
}
}
if matches {
rule.push_finding(
i,
sentences[start].start_byte,
window.last().map_or(sentences[start].end_byte, |span| span.end_byte),
opts,
findings,
);
}
}
}
fn scan_despite_its_challenges(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "despite-its-challenges") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
for sentence in sentences {
let text = (*sentence).normalize(i.text);
if text.contains("despite its")
|| text.contains("despite these challenges")
|| text.contains("faces challenges")
{
rule.push_finding(i, sentence.start_byte, sentence.end_byte, opts, findings);
}
}
}
fn literal_spans(text: &str, pattern: &str) -> Vec<TextSpan> {
let Ok(regex) = RegexBuilder::new(®ex::escape(pattern))
.case_insensitive(true)
.build()
else {
return Vec::new();
};
regex
.find_iter(text)
.filter(|mat| has_word_boundaries(text, mat.start(), mat.end()))
.map(|mat| TextSpan { start_byte: mat.start(), end_byte: mat.end() })
.collect()
}
fn split_paragraphs(text: &str) -> Vec<TextSpan> {
let mut paragraphs = Vec::new();
let mut start = None;
let mut cursor = 0;
for line in text.split_inclusive('\n') {
let line_start = cursor;
let line_end = cursor + line.len();
cursor = line_end;
if line.trim().is_empty() {
if let Some(paragraph_start) = start.take() {
push_trimmed_span(text, paragraph_start, line_start, &mut paragraphs);
}
} else if start.is_none() {
start = Some(line_start);
}
}
if let Some(paragraph_start) = start {
push_trimmed_span(text, paragraph_start, text.len(), &mut paragraphs);
} else if text.is_empty() {
paragraphs.push(TextSpan { start_byte: 0, end_byte: 0 });
}
paragraphs
}
fn split_sentences(text: &str, span: TextSpan) -> Vec<TextSpan> {
let mut sentences = Vec::new();
let mut start = None;
let Some(slice) = text.get(span.start_byte..span.end_byte) else {
return sentences;
};
for (relative_byte, ch) in slice.char_indices() {
let byte = span.start_byte + relative_byte;
if start.is_none() && !ch.is_whitespace() {
start = Some(byte);
}
if is_sentence_terminator(text, byte, ch)
&& let Some(sentence_start) = start.take()
{
push_trimmed_span(text, sentence_start, byte + ch.len_utf8(), &mut sentences);
}
}
if let Some(sentence_start) = start {
push_trimmed_span(text, sentence_start, span.end_byte, &mut sentences);
}
sentences
}
fn push_trimmed_span(text: &str, start_byte: usize, end_byte: usize, spans: &mut Vec<TextSpan>) {
let Some(slice) = text.get(start_byte..end_byte) else {
return;
};
let trimmed_start = slice
.char_indices()
.find_map(|(byte, ch)| (!ch.is_whitespace()).then_some(start_byte + byte));
let Some(start) = trimmed_start else {
return;
};
let end = slice
.char_indices()
.rev()
.find_map(|(byte, ch)| (!ch.is_whitespace()).then_some(start_byte + byte + ch.len_utf8()))
.unwrap_or(start);
if start <= end {
spans.push(TextSpan { start_byte: start, end_byte: end });
}
}
fn scan_negative_parallelism(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "negative-parallelism") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
for pair in sentences.windows(2) {
let first = pair[0].normalize(i.text);
let second = pair[1].normalize(i.text);
if contains_negation(&first)
&& starts_with_reframe(&second)
&& is_sentence_level_negation(&first)
&& !first.contains(" not in ")
{
rule.push_finding(i, pair[0].start_byte, pair[1].end_byte, opts, findings);
}
}
}
fn scan_self_posed_questions(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "self-posed-question") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
for pair in sentences.windows(2) {
let question = pair[0].text(i.text);
let answer = pair[1].text(i.text);
let question_words = words(question);
if question.trim_end().ends_with('?')
&& question_words.len() <= 6
&& question_words.first().is_some_and(|word| word == "the")
&& !answer.trim_end().ends_with('?')
{
rule.push_finding(i, pair[0].start_byte, pair[1].end_byte, opts, findings);
}
}
}
fn scan_fragment_stacks(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "short-punchy-fragments") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
let window_size = match opts.preset {
TropePreset::Lenient => 4,
TropePreset::Balanced => 3,
TropePreset::Strict => 2,
};
for window in sentences.windows(window_size) {
let candidate = TextSpan { start_byte: window[0].start_byte, end_byte: window[window.len() - 1].end_byte };
if candidate.text(i.text).contains('\n') {
continue;
}
let fragments = window
.iter()
.filter(|sentence| looks_like_punchy_fragment((**sentence).text(i.text)))
.count();
let minimum_fragments = match opts.preset {
TropePreset::Lenient => 3,
TropePreset::Balanced | TropePreset::Strict => 2,
};
if fragments >= minimum_fragments {
rule.push_finding(i, candidate.start_byte, candidate.end_byte, opts, findings);
}
}
}
fn scan_repeated_sentence_openers(
rules: &[BuiltInTropeRule], i: &TextIndex<'_>, opts: &TropeOptions, findings: &mut Vec<TropeFinding>,
sentences: &[TextSpan],
) {
let Some(rule) = rule_by_id(rules, "anaphora-abuse") else {
return;
};
if rule.severity < opts.effective_min_severity() {
return;
}
let window_size = match opts.preset {
TropePreset::Lenient => 4,
TropePreset::Balanced => 3,
TropePreset::Strict => 2,
};
for window in sentences.windows(window_size) {
let openers = window
.iter()
.filter_map(|span| sentence_opener((*span).text(i.text)))
.collect::<Vec<_>>();
if openers.len() == window_size && openers.windows(2).all(|pair| pair[0] == pair[1]) {
rule.push_finding(
i,
window[0].start_byte,
window[window.len() - 1].end_byte,
opts,
findings,
);
}
}
}
fn rule_by_id<'a>(rules: &'a [BuiltInTropeRule], id: &str) -> Option<&'a BuiltInTropeRule> {
rules.iter().find(|rule| rule.id == id)
}
fn starts_with_not(text: &str) -> bool {
text.strip_prefix("not ").is_some()
}
fn is_sentence_level_negation(text: &str) -> bool {
starts_with_not(text)
|| text.starts_with("it's not ")
|| text.starts_with("it is not ")
|| text.starts_with("this is not ")
|| text.starts_with("the question isn't ")
|| text.starts_with("the question is not ")
}
fn compact_list_item_count(text: &str) -> usize {
let trimmed = text.trim().trim_end_matches(['.', '!', '?']);
let parts = trimmed.split(',').map(str::trim).collect::<Vec<_>>();
if parts.len() < 3 || parts.iter().any(|part| words(part).len() > 3) || words(trimmed).len() > 16 {
return 0;
}
parts.len()
}
fn ordinal_sentence_number(text: &str) -> Option<usize> {
let lower = text.trim_start().to_ascii_lowercase();
["the first", "the second", "the third", "the fourth"]
.iter()
.position(|prefix| lower.starts_with(prefix))
.map(|index| index + 1)
}
fn content_word_set(text: &str) -> BTreeSet<String> {
let stop_words = [
"a", "an", "and", "are", "as", "be", "by", "for", "from", "in", "is", "it", "of", "on", "or", "that", "the",
"this", "to", "was", "were", "with",
];
words(text)
.into_iter()
.filter(|word| !stop_words.contains(&word.as_str()))
.collect()
}
fn normalize_for_comparison(text: &str) -> String {
text.split_whitespace()
.map(|word| word.trim_matches(|ch: char| !is_word_char(ch)))
.filter(|word| !word.is_empty())
.map(str::to_ascii_lowercase)
.collect::<Vec<_>>()
.join(" ")
}
fn is_sentence_terminator(text: &str, byte: usize, ch: char) -> bool {
if !matches!(ch, '.' | '?' | '!') || is_non_prose_line(line_at(text, byte)) {
return false;
}
if ch == '.' {
let next_byte = byte + ch.len_utf8();
if previous_char(text, byte) == Some('.') || next_char(text, next_byte) == Some('.') {
return false;
}
if previous_char(text, byte).is_some_and(is_word_char) && next_char(text, next_byte).is_some_and(is_word_char) {
return false;
}
}
true
}
fn line_at(text: &str, byte: usize) -> &str {
let start = text
.get(..byte)
.and_then(|prefix| prefix.rfind('\n'))
.map_or(0, |index| index + 1);
let end = text
.get(byte..)
.and_then(|suffix| suffix.find('\n'))
.map_or(text.len(), |index| byte + index);
text.get(start..end).unwrap_or("")
}
fn is_non_prose_line(line: &str) -> bool {
let trimmed = line.trim_start();
if trimmed.is_empty() {
return false;
}
if trimmed.starts_with(['-', '*', '+', '↑', '→'])
|| trimmed.starts_with('[')
|| trimmed.starts_with('$')
|| trimmed.starts_with("//")
|| starts_with_numbered_item(trimmed)
{
return true;
}
[
"::", "=>", "->", " = ", " == ", " === ", " != ", " !== ", " && ", " || ", "{", "}",
]
.iter()
.any(|marker| line.contains(marker))
}
fn starts_with_numbered_item(line: &str) -> bool {
let digits = line.bytes().take_while(u8::is_ascii_digit).count();
digits > 0
&& line
.as_bytes()
.get(digits)
.is_some_and(|byte| matches!(byte, b'.' | b')'))
}
fn looks_like_punchy_fragment(text: &str) -> bool {
let fragment_words = words(text);
if fragment_words.is_empty() || fragment_words.len() > 3 {
return false;
}
let first = fragment_words[0].as_str();
matches!(
first,
"as" | "at"
| "after"
| "before"
| "by"
| "for"
| "from"
| "in"
| "into"
| "of"
| "on"
| "over"
| "through"
| "to"
| "under"
| "with"
| "without"
) || (fragment_words.len() == 1 && first.ends_with("ly"))
}
fn contains_negation(text: &str) -> bool {
text.contains(" not ") || text.contains("n't ") || text.contains(" isn't ") || text.contains(" aren't ")
}
fn starts_with_reframe(text: &str) -> bool {
[
"it is ",
"it's ",
"this is ",
"that is ",
"they are ",
"they're ",
"the question is ",
]
.iter()
.any(|prefix| text.starts_with(prefix))
}
fn sentence_opener(text: &str) -> Option<String> {
let opener = words(text).into_iter().take(2).collect::<Vec<_>>();
(opener.len() == 2).then(|| opener.join(" "))
}
fn words(text: &str) -> Vec<String> {
text.split_whitespace()
.filter_map(|word| {
let trimmed = word.trim_matches(|ch: char| !is_word_char(ch));
(!trimmed.is_empty()).then(|| trimmed.to_ascii_lowercase())
})
.collect()
}
fn has_word_boundaries(text: &str, start_byte: usize, end_byte: usize) -> bool {
let Some(matched) = text.get(start_byte..end_byte) else {
return false;
};
let starts_with_word = matched.chars().next().is_some_and(is_word_char);
let ends_with_word = matched.chars().next_back().is_some_and(is_word_char);
if starts_with_word && previous_char(text, start_byte).is_some_and(is_word_char) {
return false;
}
if ends_with_word && next_char(text, end_byte).is_some_and(is_word_char) {
return false;
}
true
}
fn previous_char(text: &str, byte: usize) -> Option<char> {
text.get(..byte)?.chars().next_back()
}
fn next_char(text: &str, byte: usize) -> Option<char> {
text.get(byte..)?.chars().next()
}
fn is_word_char(ch: char) -> bool {
ch == '_' || ch.is_alphanumeric()
}
fn deduplicate_findings(findings: &mut Vec<TropeFinding>) {
let mut seen = BTreeSet::new();
findings.retain(|finding| {
seen.insert((
finding.range.start_byte,
finding.range.end_byte,
finding.rule_id.clone(),
finding.matched_text.clone(),
))
});
}
fn sort_findings(findings: &mut [TropeFinding]) {
findings.sort_by(|left, right| {
left.range
.start_byte
.cmp(&right.range.start_byte)
.then_with(|| left.range.end_byte.cmp(&right.range.end_byte))
.then_with(|| compare_severity_desc(left.severity, right.severity))
.then_with(|| left.rule_id.cmp(&right.rule_id))
});
}
fn compare_severity_desc(left: Severity, right: Severity) -> Ordering {
right.cmp(&left)
}
fn default_include_suggestions() -> bool {
true
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeSet;
const SEED_RULE_IDS: [&str; 6] = [
"delve-and-friends",
"serves-as-dodge",
"worth-noting",
"teacher-voice",
"signposted-conclusion",
"bold-first-bullets",
];
#[test]
fn default_options_are_useful() {
let options = TropeOptions::default();
assert_eq!(options.preset, TropePreset::Balanced);
assert_eq!(options.min_severity, Severity::Low);
assert!(options.include_suggestions);
}
#[test]
fn analyze_tropes_returns_serializable_empty_report() {
let report = analyze_tropes("Plain article text.", &TropeOptions::default());
assert_eq!(report.score, 0);
assert_eq!(report.signal, TropeSignal::Low);
assert!(report.findings.is_empty());
assert_eq!(report.summary.word_count, 3);
let value = serde_json::to_value(report).unwrap();
assert_eq!(value["score"], 0);
assert_eq!(value["signal"], "low");
assert_eq!(value["summary"]["finding_count"], 0);
}
#[test]
fn text_index_maps_ascii_offsets() {
let index = TextIndex::new("Hello world");
assert_eq!(
index.range(6, 11),
Some(TextRange {
start_byte: 6,
end_byte: 11,
start_utf16: 6,
end_utf16: 11,
start_line: 1,
start_column: 7,
end_line: 1,
end_column: 12,
})
);
assert_eq!(index.matched_text(6, 11), Some("world"));
}
#[test]
fn text_index_maps_multiline_offsets() {
let index = TextIndex::new("first line\nsecond line\nthird");
assert_eq!(
index.range(11, 17),
Some(TextRange {
start_byte: 11,
end_byte: 17,
start_utf16: 11,
end_utf16: 17,
start_line: 2,
start_column: 1,
end_line: 2,
end_column: 7,
})
);
assert_eq!(index.matched_text(11, 17), Some("second"));
}
#[test]
fn text_index_maps_unicode_offsets_before_and_inside_match() {
let text = "Hi 💡 cafe\u{301} déjà";
let start = text.find("cafe\u{301}").unwrap();
let end = start + "cafe\u{301}".len();
let index = TextIndex::new(text);
assert_eq!(
index.range(start, end),
Some(TextRange {
start_byte: 8,
end_byte: 14,
start_utf16: 6,
end_utf16: 11,
start_line: 1,
start_column: 6,
end_line: 1,
end_column: 11,
})
);
assert_eq!(index.matched_text(start, end), Some("cafe\u{301}"));
}
#[test]
fn text_index_handles_empty_input() {
let index = TextIndex::new("");
assert_eq!(
index.range(0, 0),
Some(TextRange {
start_byte: 0,
end_byte: 0,
start_utf16: 0,
end_utf16: 0,
start_line: 1,
start_column: 1,
end_line: 1,
end_column: 1,
})
);
assert_eq!(index.matched_text(0, 0), Some(""));
assert_eq!(index.range(1, 1), None);
}
#[test]
fn text_index_rejects_invalid_ranges() {
let index = TextIndex::new("aé");
assert_eq!(index.range(3, 1), None);
assert_eq!(index.range(1, 2), None);
assert_eq!(index.range(0, 4), None);
assert_eq!(index.matched_text(1, 2), None);
}
#[test]
fn vendored_tropes_markdown_is_clean() {
assert!(TROPES_MD_FILE.starts_with("# AI Writing Tropes to Avoid\n\n## Word Choice"));
assert!(!TROPES_MD_FILE.contains("\n---\n"));
assert!(!TROPES_MD_FILE.contains("Add this file to your AI assistant"));
assert!(!TROPES_MD_FILE.contains("Source:"));
}
#[test]
fn built_in_rules_load_once_and_cover_markdown_catalog() {
let first = built_in_trope_rules().unwrap();
let second = built_in_trope_rules().unwrap();
assert!(std::ptr::eq(first.as_ptr(), second.as_ptr()));
let markdown_headings = markdown_trope_headings();
let rule_headings = first
.iter()
.map(|rule| rule.source.heading.as_str())
.collect::<BTreeSet<_>>();
assert_eq!(rule_headings, markdown_headings);
}
#[test]
fn built_in_rules_include_required_metadata() {
let rules = built_in_trope_rules().unwrap();
let mut ids = BTreeSet::new();
for rule in rules {
assert!(ids.insert(rule.id.as_str()), "duplicate rule id: {}", rule.id);
assert!(!rule.message.trim().is_empty(), "missing message for {}", rule.id);
assert!(!rule.patterns.is_empty(), "missing patterns for {}", rule.id);
assert_eq!(rule.source.file, "tropes.md");
assert!(
!rule.source.heading.trim().is_empty(),
"missing source heading for {}",
rule.id
);
}
}
#[test]
fn built_in_rules_identify_seed_slice() {
let seed_ids = built_in_trope_rules()
.unwrap()
.iter()
.filter(|rule| rule.scope == TropeRuleScope::Seed)
.map(|rule| rule.id.as_str())
.collect::<BTreeSet<_>>();
assert_eq!(seed_ids, SEED_RULE_IDS.into_iter().collect::<BTreeSet<_>>());
}
fn markdown_trope_headings() -> BTreeSet<&'static str> {
TROPES_MD_FILE
.lines()
.filter_map(|line| line.strip_prefix("### "))
.collect()
}
#[test]
fn phrase_scanner_matches_case_insensitively() {
let report = analyze_tropes("Delve into the details.", &TropeOptions::default());
assert_eq!(report.findings.len(), 1);
assert_eq!(report.findings[0].rule_id, "delve-and-friends");
assert_eq!(report.findings[0].matched_text, "Delve");
}
#[test]
fn phrase_scanner_enforces_word_boundaries() {
let report = analyze_tropes(
"The model can undelve and overleverageable prose.",
&TropeOptions::default(),
);
assert!(report.findings.is_empty());
}
#[test]
fn phrase_scanner_matches_multi_word_phrases() {
let report = analyze_tropes("It is worth noting that this has limits.", &TropeOptions::default());
assert_eq!(report.findings.len(), 1);
assert_eq!(report.findings[0].rule_id, "worth-noting");
assert_eq!(report.findings[0].range.start_byte, 0);
assert_eq!(report.findings[0].matched_text, "It is worth noting");
}
#[test]
fn scanner_preserves_overlapping_findings_for_different_rules() {
let report = analyze_tropes("This will fundamentally reshape the field.", &TropeOptions::default());
let rule_ids = report
.findings
.iter()
.map(|finding| finding.rule_id.as_str())
.collect::<BTreeSet<_>>();
assert!(rule_ids.contains("quietly-magic-adverbs"));
assert!(rule_ids.contains("grandiose-stakes-inflation"));
}
#[test]
fn scanner_deduplicates_identical_findings() {
let report = analyze_tropes("Importantly, this matters.", &TropeOptions::default());
assert_eq!(report.findings.len(), 1);
assert_eq!(report.findings[0].rule_id, "worth-noting");
}
#[test]
fn regex_scanner_reports_bold_first_bullets() {
let report = analyze_tropes(
"- **Security**: Environment-based configuration.",
&TropeOptions::default(),
);
assert_eq!(report.findings.len(), 1);
assert_eq!(report.findings[0].rule_id, "bold-first-bullets");
assert_eq!(report.findings[0].matched_text, "- **Security**:");
}
#[test]
fn scanner_sorts_findings_by_range_severity_and_rule_id() {
let report = analyze_tropes("Fundamentally reshape. In summary, delve.", &TropeOptions::default());
let ordered = report
.findings
.iter()
.map(|finding| finding.rule_id.as_str())
.collect::<Vec<_>>();
assert_eq!(
ordered,
vec![
"quietly-magic-adverbs",
"grandiose-stakes-inflation",
"signposted-conclusion",
"delve-and-friends",
]
);
}
#[test]
fn options_filter_severity_and_suggestions() {
let report = analyze_tropes(
"It is worth noting that we should delve.",
&TropeOptions { min_severity: Severity::Medium, include_suggestions: false, ..Default::default() },
);
assert_eq!(report.findings.len(), 1);
assert_eq!(report.findings[0].rule_id, "delve-and-friends");
assert_eq!(report.findings[0].suggestion, None);
}
#[test]
fn paragraph_splitter_separates_blank_lines() {
let paragraphs = split_paragraphs("One sentence.\n\nSecond paragraph.\nStill second.");
let text = "One sentence.\n\nSecond paragraph.\nStill second.";
assert_eq!(paragraphs.len(), 2);
assert_eq!(paragraphs[0].text(text,), "One sentence.");
assert_eq!(paragraphs[1].text(text,), "Second paragraph.\nStill second.");
}
#[test]
fn sentence_splitter_splits_simple_punctuation() {
let text = "One. Two? Three!";
let sentences = split_sentences(text, TextSpan { start_byte: 0, end_byte: text.len() });
let sentence_text = sentences.iter().map(|span| (*span).text(text)).collect::<Vec<_>>();
assert_eq!(sentence_text, vec!["One.", "Two?", "Three!"]);
}
#[test]
fn structural_detector_matches_negative_parallelism() {
let report = analyze_tropes("It is not bold. It is backwards.", &TropeOptions::default());
assert!(
report
.findings
.iter()
.any(|finding| finding.rule_id == "negative-parallelism")
);
}
#[test]
fn structural_detector_matches_self_posed_question() {
let report = analyze_tropes("The result? Devastating.", &TropeOptions::default());
assert!(
report
.findings
.iter()
.any(|finding| finding.rule_id == "self-posed-question")
);
}
#[test]
fn structural_detector_ignores_normal_questions() {
let report = analyze_tropes("Did it work? Yes.", &TropeOptions::default());
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "self-posed-question")
);
}
#[test]
fn structural_detector_matches_fragment_stack() {
let report = analyze_tropes("He shipped it. Openly. In production.", &TropeOptions::default());
assert!(
report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn structural_detector_ignores_short_lines_lists_and_code() {
let report = analyze_tropes(
"Heading.\nOne.\nTwo.\n\n- First item.\n- Second item.\n- Third item.\n\nconst target = {\nmessage: \"hello\",\n};\nconsole.log(target.message);",
&TropeOptions::default(),
);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn structural_detector_does_not_split_versions_identifiers_or_ellipses() {
let report = analyze_tropes(
"Rust 1.83.0. The API is stable. Use std::mem::replace. \"...\" Then continue.",
&TropeOptions::default(),
);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn unicode_apostrophes_are_normal_typography() {
let report = analyze_tropes(
"The author's work is clear. It’s useful. The “quoted” phrase uses normal typography.",
&TropeOptions::default(),
);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "unicode-decoration")
);
}
#[test]
fn extracted_rust_release_notes_do_not_overreport_code_and_api_lists() {
let report = analyze_tropes(
"The Rust team is happy to announce a new version of Rust, 1.83.0. Rust is a programming language empowering everyone to build reliable and efficient software.\nNew const capabilities\nstatic S: i32 = 25;\nconst C: &i32 = &S;\nStabilized APIs\nBufRead::skip_until\nControlFlow::break_value\nOption::get_or_insert_default\nMany people came together to create Rust 1.83.0.",
&TropeOptions::default(),
);
assert_ne!(report.signal, TropeSignal::VeryHigh);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn extracted_mdn_proxy_reference_does_not_overreport_code() {
let report = analyze_tropes(
"The Proxy object allows you to create an object that can be used in place of the original object.\nconst target = {\nmessage1: \"hello\",\nmessage2: \"everyone\",\n};\nconst proxy = new Proxy(target, handler);\nconsole.log(proxy.message1); // hello\nThe Reflect methods provide the reflective semantics for invoking the corresponding object internal methods.",
&TropeOptions::default(),
);
assert_ne!(report.signal, TropeSignal::VeryHigh);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn extracted_wikipedia_mozilla_page_does_not_overreport_references() {
let text = format!(
"Mozilla is a community that develops free software. It’s best known for Firefox and related projects.\nMozilla uses a framework for one project. Another framework supports a second project. A third framework supports a third project. The framework changes over time. This framework is documented. That framework is maintained. The framework is public. One more framework is discussed.\n↑ \"Mozilla Foundation\". Retrieved 2024-03-26.\n↑ \"Mozilla Festival\". wiki.mozilla.org. Retrieved August 30, 2020.\n↑ Roth, Emma (2024-03-22). \"Mozilla's product chief sues the Firefox maker\". Retrieved 2024-07-07.\n{}",
"neutral context ".repeat(600)
);
let report = analyze_tropes(&text, &TropeOptions::default());
assert_ne!(report.signal, TropeSignal::VeryHigh);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn extracted_makers_schedule_does_not_treat_ellipsis_as_fragments() {
let report = analyze_tropes(
"\"...\" Charles Dickens. There are two types of schedule, which I’ll call the manager's schedule and the maker's schedule. The manager's schedule is for bosses.\nWhen you're operating on the maker's schedule, meetings are a disaster. A single meeting can blow a whole afternoon by breaking it into two pieces.",
&TropeOptions::default(),
);
assert_ne!(report.signal, TropeSignal::VeryHigh);
assert!(
!report
.findings
.iter()
.any(|finding| finding.rule_id == "short-punchy-fragments")
);
}
#[test]
fn structural_detector_matches_repeated_sentence_openers() {
let report = analyze_tropes(
"They could expose APIs. They could offer scopes. They could provide access.",
&TropeOptions::default(),
);
assert!(
report
.findings
.iter()
.any(|finding| finding.rule_id == "anaphora-abuse")
);
}
#[test]
fn structural_detector_matches_remaining_catalog_rules() {
let report = analyze_tropes(
"Not a bug. Not a feature. A design flaw.\n\
Identity, payments, compute, distribution.\n\
From innovation to implementation.\n\
The first wall is absent. The second wall is missing. The third wall is missing.\n\
Despite these challenges, the project continues.",
&TropeOptions::default(),
);
for rule_id in [
"not-not-just",
"tricolon-abuse",
"false-ranges",
"listicle-in-a-trench-coat",
"despite-its-challenges",
] {
assert!(has_rule(&report, rule_id), "missing structural finding: {rule_id}");
}
}
#[test]
fn document_detectors_match_remaining_catalog_rules() {
let report = analyze_tropes(
"The problem — and this is the part nobody talks about — is systemic.\n\
It should work — but it does not — for users.\n\
The ecosystem grows. The ecosystem changes. The ecosystem expands.\n\
In this section we preview the point. As we've seen in this section, the point returns.\n\
Take Spotify as an example. Or consider Uber as another example.\n\
The platform gives teams a shared place to work.\n\
The platform gives teams a common place to work.\n\
This sentence appears in both sections. This sentence appears in both sections.",
&TropeOptions::default(),
);
for rule_id in [
"em-dash-addiction",
"fractal-summaries",
"dead-metaphor",
"historical-analogy-stacking",
"one-point-dilution",
"content-duplication",
] {
assert!(has_rule(&report, rule_id), "missing document finding: {rule_id}");
}
}
#[test]
fn document_detectors_require_repeated_evidence() {
let report = analyze_tropes(
"A single ecosystem can describe a software system. One dash — is fine.\n\
This section introduces the topic.",
&TropeOptions::default(),
);
for rule_id in [
"em-dash-addiction",
"fractal-summaries",
"dead-metaphor",
"historical-analogy-stacking",
"one-point-dilution",
"content-duplication",
] {
assert!(!has_rule(&report, rule_id), "unexpected document finding: {rule_id}");
}
}
#[test]
fn presets_change_evidence_thresholds() {
let low_signal_text = "In summary, done.";
let lenient = analyze_tropes(
low_signal_text,
&TropeOptions { preset: TropePreset::Lenient, ..Default::default() },
);
let balanced = analyze_tropes(low_signal_text, &TropeOptions::default());
let strict = analyze_tropes(
low_signal_text,
&TropeOptions { preset: TropePreset::Strict, ..Default::default() },
);
assert!(!has_rule(&lenient, "signposted-conclusion"));
assert!(has_rule(&balanced, "signposted-conclusion"));
assert!(has_rule(&strict, "signposted-conclusion"));
let two_item_listicle = "The first wall is absent. The second wall is missing.";
let lenient = analyze_tropes(
two_item_listicle,
&TropeOptions { preset: TropePreset::Lenient, ..Default::default() },
);
let balanced = analyze_tropes(two_item_listicle, &TropeOptions::default());
let strict = analyze_tropes(
two_item_listicle,
&TropeOptions { preset: TropePreset::Strict, ..Default::default() },
);
assert!(!has_rule(&lenient, "listicle-in-a-trench-coat"));
assert!(!has_rule(&balanced, "listicle-in-a-trench-coat"));
assert!(has_rule(&strict, "listicle-in-a-trench-coat"));
}
fn has_rule(report: &TropeReport, rule_id: &str) -> bool {
report.findings.iter().any(|finding| finding.rule_id == rule_id)
}
#[test]
fn scoring_empty_text_is_low() {
let report = analyze_tropes("", &TropeOptions::default());
assert_eq!(report.score, 0);
assert_eq!(report.signal, TropeSignal::Low);
assert_eq!(report.summary.finding_count, 0);
assert_eq!(report.summary.findings_per_1000_words, 0.0);
}
#[test]
fn scoring_one_mild_finding_stays_low() {
let report = analyze_tropes("In summary, done.", &TropeOptions::default());
assert_eq!(report.summary.finding_count, 1);
assert!(report.score < 20);
assert_eq!(report.signal, TropeSignal::Low);
}
#[test]
fn scoring_dense_repeated_findings_increases_signal() {
let report = analyze_tropes(
"In summary, delve. In summary, delve. In summary, delve.",
&TropeOptions::default(),
);
assert!(report.score >= 45);
assert!(matches!(report.signal, TropeSignal::High | TropeSignal::VeryHigh));
assert!(report.summary.repeated_rules.contains(&"delve-and-friends".to_string()));
assert!(
report
.summary
.repeated_rules
.contains(&"signposted-conclusion".to_string())
);
}
#[test]
fn scoring_never_exceeds_100() {
let text = "In summary, delve. ".repeat(100);
let report = analyze_tropes(&text, &TropeOptions::default());
assert_eq!(report.score, 100);
assert_eq!(report.signal, TropeSignal::VeryHigh);
}
}