use serde_json::Value;
use super::document::child_pointer;
use super::finding::{Finding, Rule};
use super::rules::{QuestionView, RequestView};
const LONG_ARRAY_ITEMS: usize = 20;
const ESCAPE_WORDS: [&str; 21] = [
"other",
"others",
"else",
"none",
"neither",
"nobody",
"noone",
"unknown",
"unclear",
"unsure",
"uncertain",
"ambiguous",
"undetermined",
"indeterminate",
"unspecified",
"unstated",
"unanswerable",
"inapplicable",
"irrelevant",
"idk",
"unmatched",
];
const ESCAPE_PHRASES: [&str; 33] = [
"n_a",
"no_one",
"no_match",
"no_fit",
"no_answer",
"off_topic",
"does_not_apply",
"doesn_t_apply",
"not_stated",
"not_specified",
"not_mentioned",
"not_given",
"not_provided",
"not_listed",
"not_applicable",
"not_relevant",
"not_sure",
"not_clear",
"not_known",
"not_determined",
"cannot_tell",
"can_t_tell",
"cant_tell",
"cannot_say",
"can_t_say",
"cannot_determine",
"can_t_determine",
"unable_to_tell",
"unable_to_determine",
"don_t_know",
"dont_know",
"do_not_know",
"hard_to_say",
];
const LACKING_PHRASES: [&str; 5] = ["not_enough", "insufficient", "no", "missing", "lacking"];
const INFORMATION_WORDS: [&str; 7] = [
"info",
"information",
"data",
"context",
"detail",
"details",
"evidence",
];
const YES_NO_PAIRS: [[&str; 2]; 3] = [["yes", "no"], ["true", "false"], ["y", "n"]];
pub(super) fn check(request: &RequestView<'_>, findings: &mut Vec<Finding>) {
for question in &request.questions {
let criteria_pointer = child_pointer(&question.pointer, "criteria");
match (question.kind, question.fields.get("criteria")) {
(Some("choice"), Some(Value::Object(options))) if !options.is_empty() => {
let names: Vec<String> = options.keys().map(|name| normalise(name)).collect();
check_escape_option(question, &criteria_pointer, &names, findings);
check_yes_no(question, &criteria_pointer, &names, findings);
let described = options
.iter()
.map(|(name, description)| (name.clone(), description));
check_duplicate_descriptions(
question,
&criteria_pointer,
"options",
described,
findings,
);
}
(Some("score"), Some(Value::Array(levels))) => {
let described = levels
.iter()
.enumerate()
.map(|(index, level)| (index.to_string(), level));
check_duplicate_descriptions(
question,
&criteria_pointer,
"levels",
described,
findings,
);
}
_ => {}
}
if let Some(state) = request.state {
check_indexed_paths(question, state, findings);
}
}
}
fn normalise(name: &str) -> String {
let mut normalised = String::with_capacity(name.len());
for character in name.trim().chars() {
if character.is_alphanumeric() {
normalised.extend(character.to_lowercase());
} else if !normalised.ends_with('_') {
normalised.push('_');
}
}
normalised.trim_matches('_').to_owned()
}
fn is_escape_option(normalised: &str) -> bool {
if normalised == "na"
|| normalised
.split('_')
.any(|word| ESCAPE_WORDS.contains(&word))
{
return true;
}
let padded = format!("_{normalised}_");
let has_phrase = |phrase: &str| padded.contains(&format!("_{phrase}_"));
ESCAPE_PHRASES.iter().any(|phrase| has_phrase(phrase))
|| LACKING_PHRASES.iter().any(|lacking| {
INFORMATION_WORDS
.iter()
.any(|information| has_phrase(&format!("{lacking}_{information}")))
})
}
fn check_escape_option(
question: &QuestionView<'_>,
pointer: &str,
names: &[String],
findings: &mut Vec<Finding>,
) {
if names.iter().any(|name| is_escape_option(name)) {
return;
}
findings.push(
Finding::warning(
Rule::ChoiceNoEscapeOption,
pointer,
format!(
"choice `{}` has no way out; the model must pick one of its options even when none fits",
question.id
),
)
.in_question(question.id)
.suggest("add an option such as `other`, `none_of_the_above` or `not_stated`"),
);
}
fn check_yes_no(
question: &QuestionView<'_>,
pointer: &str,
names: &[String],
findings: &mut Vec<Finding>,
) {
let [first, second] = names else { return };
let is_pair = YES_NO_PAIRS
.iter()
.any(|[yes, no]| (first == yes && second == no) || (first == no && second == yes));
if !is_pair {
return;
}
findings.push(
Finding::warning(
Rule::ChoiceYesNo,
pointer,
format!("choice `{}` is a yes/no question", question.id),
)
.in_question(question.id)
.suggest(
"use a noul: it returns the probability of yes, which a threshold can be tuned against",
),
);
}
fn check_duplicate_descriptions<'a>(
question: &QuestionView<'_>,
pointer: &str,
what: &str,
described: impl Iterator<Item = (String, &'a Value)>,
findings: &mut Vec<Finding>,
) {
let mut seen: Vec<(String, &Value)> = Vec::new();
for (name, description) in described {
if description.is_null()
|| description
.as_str()
.is_some_and(|text| text.trim().is_empty())
{
continue;
}
if let Some((earlier, _)) = seen
.iter()
.find(|(_, other)| same_description(other, description))
{
findings.push(
Finding::warning(
Rule::DuplicateDescription,
child_pointer(pointer, &name),
format!(
"{what} `{earlier}` and `{name}` of `{}` have the same description",
question.id
),
)
.in_question(question.id)
.suggest("say what tells them apart, or merge them"),
);
} else {
seen.push((name, description));
}
}
}
fn same_description(left: &Value, right: &Value) -> bool {
match (left.as_str(), right.as_str()) {
(Some(left), Some(right)) => left.trim().eq_ignore_ascii_case(right.trim()),
_ => left == right,
}
}
fn check_indexed_paths(question: &QuestionView<'_>, state: &Value, findings: &mut Vec<Finding>) {
let mut reported: Vec<String> = Vec::new();
for field in ["instructions", "criteria"] {
let Some(content) = question.fields.get(field) else {
continue;
};
let field_pointer = child_pointer(&question.pointer, field);
visit_strings(content, &field_pointer, &mut |text, pointer| {
for path in backticked(text) {
let Some(length) = longest_indexed_array(state, path) else {
continue;
};
if length <= LONG_ARRAY_ITEMS || reported.iter().any(|seen| seen == path) {
continue;
}
reported.push(path.to_owned());
findings.push(
Finding::warning(
Rule::PathIndexLongArray,
pointer,
format!(
"`{path}` indexes into an array of {length} items; the model miscounts positions in arrays longer than {LONG_ARRAY_ITEMS}"
),
)
.in_question(question.id)
.suggest("put the value itself in the instructions, or key the items by a short id instead of a position"),
);
}
});
}
}
fn visit_strings(value: &Value, pointer: &str, visit: &mut impl FnMut(&str, &str)) {
match value {
Value::String(text) => visit(text, pointer),
Value::Array(items) => {
for (index, item) in items.iter().enumerate() {
visit_strings(item, &child_pointer(pointer, &index.to_string()), visit);
}
}
Value::Object(map) => {
for (key, item) in map {
visit_strings(item, &child_pointer(pointer, key), visit);
}
}
_ => {}
}
}
fn backticked(text: &str) -> impl Iterator<Item = &str> {
text.split('`')
.skip(1)
.step_by(2)
.map(str::trim)
.filter(|segment| !segment.is_empty())
}
enum Step<'a> {
Key(&'a str),
Index(usize),
}
fn parse_path(path: &str) -> Option<Vec<Step<'_>>> {
let mut steps = Vec::new();
let mut has_index = false;
for part in path.split('.') {
let (key, mut rest) = part
.split_once('[')
.map_or((part, ""), |(key, rest)| (key, rest));
if !key.is_empty() {
if !key
.chars()
.all(|c| c.is_alphanumeric() || c == '_' || c == '-')
{
return None;
}
steps.push(Step::Key(key));
} else if rest.is_empty() {
return None;
}
while !rest.is_empty() {
let (index, after) = rest.split_once(']')?;
steps.push(Step::Index(index.trim().parse().ok()?));
has_index = true;
rest = match after.strip_prefix('[') {
Some(next) => next,
None if after.is_empty() => "",
None => return None,
};
}
}
has_index.then_some(steps)
}
fn longest_indexed_array(state: &Value, path: &str) -> Option<usize> {
let steps = parse_path(path)?;
let skip = usize::from(
matches!(steps.first(), Some(Step::Key("state"))) && state.get("state").is_none(),
);
let mut current = state;
let mut longest = None;
for step in steps.iter().skip(skip) {
current = match step {
Step::Key(key) => current.get(key)?,
Step::Index(index) => {
let items = current.as_array()?;
longest = longest.max(Some(items.len()));
match items.get(*index) {
Some(item) => item,
None => return longest,
}
}
};
}
longest
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::{backticked, is_escape_option, longest_indexed_array, normalise};
#[test]
fn option_names_are_compared_without_case_or_punctuation() {
assert_eq!(normalise(" None of the Above "), "none_of_the_above");
assert_eq!(normalise("N/A"), "n_a");
assert_eq!(normalise("not-stated"), "not_stated");
assert_eq!(normalise("__Other__"), "other");
}
#[test]
fn recognises_the_usual_ways_out() {
for name in [
"other",
"none_of_the_above",
"not_stated",
"N/A",
"na",
"unknown",
"other_reason",
"none_of_these_teams",
"billing_or_other",
"something-else",
"Not enough information",
"not_enough_data",
"insufficient_information",
"insufficient context",
"no_info",
"missing-details",
"nobody_not_stated",
"Nobody",
"no one",
"UNKNOWN_SENDER",
"unclear",
"cannot tell",
"can't determine",
"don't know",
"does not apply",
"Not applicable",
"not_listed",
"off-topic",
"irrelevant",
"no_match",
"neither",
"IDK",
] {
assert!(is_escape_option(&normalise(name)), "{name}");
}
for name in [
"billing",
"not_urgent",
"another",
"nonessential",
"no",
"otherwise",
"known_issue",
"insufficient_funds",
"not_enough_stock",
"missing_item",
"na_east",
"banana",
"enough_information",
"detailed",
"unknowns_resolved",
] {
assert!(!is_escape_option(&normalise(name)), "{name}");
}
}
#[test]
fn finds_backticked_segments() {
let found: Vec<&str> =
backticked("Compare `a[0]` with ` b.c[12] ` and `` and plain text").collect();
assert_eq!(found, ["a[0]", "b.c[12]"]);
assert_eq!(backticked("no code here").count(), 0);
assert_eq!(
backticked("an `unclosed segment").collect::<Vec<_>>(),
["unclosed segment"]
);
}
#[test]
fn follows_indexed_paths_through_the_state() {
let state = json!({
"items": (0..30).map(|n| json!({ "tags": ["a", "b"], "n": n })).collect::<Vec<_>>(),
"few": [1, 2, 3],
"grid": [[1, 2], [3, 4]]
});
assert_eq!(longest_indexed_array(&state, "items[3]"), Some(30));
assert_eq!(longest_indexed_array(&state, "state.items[3].n"), Some(30));
assert_eq!(
longest_indexed_array(&state, "items[3].tags[1]"),
Some(30),
"the longest one counts"
);
assert_eq!(
longest_indexed_array(&state, "items[99]"),
Some(30),
"past the end still indexes it"
);
assert_eq!(longest_indexed_array(&state, "few[0]"), Some(3));
assert_eq!(longest_indexed_array(&state, "grid[1][0]"), Some(2));
}
#[test]
fn ignores_backticks_that_are_not_indexed_paths_into_the_state() {
let state = json!({ "items": [1, 2, 3], "name": "x" });
for not_a_path in [
"items",
"name",
"missing[0]",
"items[x]",
"items[0",
"name[0]",
"f(x)[0]",
"a b[0]",
"[",
"",
] {
assert_eq!(
longest_indexed_array(&state, not_a_path),
None,
"{not_a_path:?}"
);
}
assert_eq!(
longest_indexed_array(&json!("plain text state"), "items[0]"),
None
);
assert_eq!(
longest_indexed_array(&json!([[1, 2, 3]]), "[0][1]"),
Some(3),
"a root array"
);
}
}