use std::collections::BTreeMap;
use std::io::{IsTerminal, Read};
use std::path::Path;
use serde::Deserialize;
use stegobench_core::exit;
pub const MAX_SCORES: usize = 20_000_000;
pub const MAX_INPUT_BYTES: u64 = 256 * 1024 * 1024;
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Input {
pub scores: Vec<Option<f64>>,
pub labels: Vec<bool>,
}
#[derive(Debug, thiserror::Error)]
pub enum MetricsError {
#[error("cannot read {path}: {source}")]
Read {
path: String,
#[source]
source: std::io::Error,
},
#[error(
"{path} holds more than {} MiB of JSON, which is more than this reads \
in one call. Split the run into arms and score each one: a single set \
of scores that large is not one measurement",
MAX_INPUT_BYTES / (1024 * 1024)
)]
TooLarge { path: String },
#[error(
"nothing was given to read. This reads a JSON object of scores and \
labels, either from a file you name or piped in on standard \
input.\n stegobench metrics scores.json\n cat scores.json | \
stegobench metrics -\nIt should look like {{\"scores\": [0.91, \
0.02], \"labels\": [true, false]}}, with null for an item the \
detector could not score"
)]
NothingToRead,
#[error(
"{count} scores is more than the {MAX_SCORES} this reads in one call. \
Split the run into arms and score each one"
)]
TooManyScores { count: usize },
#[error(
"{path} is not a JSON object of scores and labels: {source}\nIt should \
look like {{\"scores\": [0.91, 0.02], \"labels\": [true, false]}}, \
with null for an item the detector could not score"
)]
NotJson {
path: String,
#[source]
source: serde_json::Error,
},
#[error(
"a false-alarm budget of {text:?} is not a rate. Give a fraction \
between 0 and 1, such as 0.01 for one per cent"
)]
BudgetNotARate { text: String },
#[error(
"--at {text} was given twice. One budget is one measurement, and \
reporting it twice would put the same figure in the output under two \
names"
)]
DuplicateBudget { text: String },
#[error(
"{scores} score(s) against {labels} label(s): the two were built from \
different record sets, so any ranking over them would describe images \
nobody named"
)]
LengthMismatch { scores: usize, labels: usize },
#[error(
"nothing to measure: no scores were given. An empty set reads as a \
detector that caught nothing rather than one that was never run"
)]
Empty,
#[error(
"{count} of the {total} answer(s) are not numbers. A score written \
`null`, or one that is not a number, cannot be ordered, and a ranking \
that quietly skipped it would be measured on a subset nobody named"
)]
NotANumber { count: usize, total: usize },
#[error(
"{clean} clean and {stego} stego image(s), and a ranking needs both. \
There is nothing to tell apart when every answer carries the same \
label"
)]
OneSided { clean: usize, stego: usize },
#[error(
"{count} answer(s) were usable and the ranking over them still could \
not be formed{at}. This is a bug in stegobench rather than anything \
you did; please report it with this message"
)]
Unrankable { count: usize, at: String },
}
impl MetricsError {
pub fn exit_code(&self) -> i32 {
match self {
MetricsError::Read { .. } => exit::FAILURE,
MetricsError::NotJson { .. }
| MetricsError::BudgetNotARate { .. }
| MetricsError::DuplicateBudget { .. }
| MetricsError::LengthMismatch { .. } => exit::USAGE,
MetricsError::NothingToRead => exit::USAGE,
MetricsError::TooLarge { .. }
| MetricsError::TooManyScores { .. }
| MetricsError::Empty
| MetricsError::NotANumber { .. }
| MetricsError::OneSided { .. } => exit::PREFLIGHT_REFUSED,
MetricsError::Unrankable { .. } => exit::FAILURE,
}
}
pub fn reason(&self) -> &'static str {
match self {
MetricsError::Read { .. } => "read-failed",
MetricsError::NothingToRead => "nothing-to-read",
MetricsError::TooLarge { .. } => "input-too-large",
MetricsError::TooManyScores { .. } => "too-many-scores",
MetricsError::NotJson { .. } => "not-json",
MetricsError::BudgetNotARate { .. } => "budget-not-a-rate",
MetricsError::DuplicateBudget { .. } => "duplicate-budget",
MetricsError::LengthMismatch { .. } => "length-mismatch",
MetricsError::Empty => "empty",
MetricsError::NotANumber { .. } => "not-a-number",
MetricsError::OneSided { .. } => "one-sided",
MetricsError::Unrankable { .. } => "unrankable",
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Report {
pub auc: f64,
pub auc_ci95: Option<[f64; 2]>,
pub tpr_at_fpr: BTreeMap<String, f64>,
pub achieved_fpr: BTreeMap<String, f64>,
pub threshold_at_fpr: BTreeMap<String, f64>,
pub n_clean: usize,
pub n_stego: usize,
pub findings: Vec<stegobench_metrics::Finding>,
}
impl Report {
pub fn warnings(&self) -> Vec<String> {
self.findings
.iter()
.map(|f| match *f {
stegobench_metrics::Finding::EveryScoreIdentical { count } => format!(
"every one of the {count} answers was identical, so this AUC is \
0.5 by construction and not by measurement: nothing separated \
these images"
),
stegobench_metrics::Finding::ScoresRunBackwards { auc, inverted } => format!(
"this scored {auc:.4}, which is below the 0.5 a coin flip gets. \
That usually means the scores run the wrong way round rather \
than that the detector cannot see anything: at {inverted:.4} \
they separate these images about as well inverted as they \
would upright. Check the adapter's sign before reading this as \
a measurement"
),
})
.collect()
}
pub fn to_json(&self) -> serde_json::Value {
serde_json::json!({
"ok": true,
"warnings": self.warnings(),
"auc": self.auc,
"auc_ci95": self.auc_ci95,
"tpr_at_fpr": self.tpr_at_fpr,
"achieved_fpr": self.achieved_fpr,
"threshold_at_fpr": self.threshold_at_fpr,
"fpr_resolution": stegobench_metrics::fpr_resolution(self.n_clean),
"n_clean": self.n_clean,
"n_stego": self.n_stego,
"n": self.n_clean + self.n_stego,
})
}
pub fn human(&self) -> String {
let mut text = format!(
"{} answer(s): {} clean, {} stego\nAUC {:.4}{}",
self.n_clean + self.n_stego,
self.n_clean,
self.n_stego,
self.auc,
match self.auc_ci95 {
Some([lo, hi]) => format!("\n95% interval [{lo:.4}, {hi:.4}]"),
None if self.n_clean < 2 || self.n_stego < 2 => {
"\n95% interval not estimated: one class has fewer \
than two members"
.to_string()
}
None => "\n95% interval not estimated: these answers \
separate with nothing left over, so the estimator \
has no width to report"
.to_string(),
}
);
for (budget, tpr) in &self.tpr_at_fpr {
text.push_str(&format!("\nTPR at {budget:<7} {tpr:.4}"));
if let Some(threshold) = self.threshold_at_fpr.get(budget) {
text.push_str(&format!(" flag at score >= {threshold}"));
}
if let Some(got) = self.achieved_fpr.get(budget) {
if let Ok(asked) = budget.parse::<f64>() {
if *got + 1e-12 < asked {
text.push_str(&format!(
" (actually at a false-alarm rate of {got}: {} clean \
image(s) cannot express {budget})",
self.n_clean
));
}
}
}
}
text
}
}
pub fn run(file: Option<&Path>, at: &[String]) -> Result<Report, MetricsError> {
let budgets = parse_budgets(at)?;
let (label, text) = read_input(file)?;
let input: Input = serde_json::from_str(&text).map_err(|source| MetricsError::NotJson {
path: label,
source,
})?;
let report = compute(input, &budgets)?;
for warning in report.warnings() {
eprintln!("{warning}");
}
Ok(report)
}
fn parse_budgets(at: &[String]) -> Result<Vec<(String, f64)>, MetricsError> {
let mut out: Vec<(String, f64)> = Vec::new();
for text in at {
if out.iter().any(|(seen, _)| seen == text) {
return Err(MetricsError::DuplicateBudget { text: text.clone() });
}
let value: f64 = text
.parse()
.map_err(|_| MetricsError::BudgetNotARate { text: text.clone() })?;
if !(0.0..=1.0).contains(&value) {
return Err(MetricsError::BudgetNotARate { text: text.clone() });
}
out.push((text.clone(), value));
}
Ok(out)
}
fn nothing_to_read(file: Option<&Path>, stdin_is_a_terminal: bool) -> Option<MetricsError> {
(file.is_none() && stdin_is_a_terminal).then_some(MetricsError::NothingToRead)
}
fn read_input(file: Option<&Path>) -> Result<(String, String), MetricsError> {
if let Some(e) = nothing_to_read(file, std::io::stdin().is_terminal()) {
return Err(e);
}
let (label, mut reader): (String, Box<dyn Read>) = match file {
None => (
"standard input".to_string(),
Box::new(std::io::stdin().lock()),
),
Some(path) => {
let handle = std::fs::File::open(path).map_err(|source| MetricsError::Read {
path: path.display().to_string(),
source,
})?;
if handle.metadata().map(|m| m.len()).unwrap_or(0) > MAX_INPUT_BYTES {
return Err(MetricsError::TooLarge {
path: path.display().to_string(),
});
}
(path.display().to_string(), Box::new(handle))
}
};
let mut text = String::new();
let read = reader
.by_ref()
.take(MAX_INPUT_BYTES + 1)
.read_to_string(&mut text)
.map_err(|source| MetricsError::Read {
path: label.clone(),
source,
})?;
if read as u64 > MAX_INPUT_BYTES {
return Err(MetricsError::TooLarge { path: label });
}
Ok((label, text))
}
fn unrankable(scores: &[f64], labels: &[bool]) -> Option<MetricsError> {
if scores.len() != labels.len() {
return Some(MetricsError::LengthMismatch {
scores: scores.len(),
labels: labels.len(),
});
}
if scores.is_empty() {
return Some(MetricsError::Empty);
}
let nan = scores.iter().filter(|s| s.is_nan()).count();
if nan > 0 {
return Some(MetricsError::NotANumber {
count: nan,
total: scores.len(),
});
}
let stego = labels.iter().filter(|l| **l).count();
let clean = labels.len() - stego;
if clean == 0 || stego == 0 {
return Some(MetricsError::OneSided { clean, stego });
}
None
}
pub fn why_unrankable(scores: &[f64], labels: &[bool], at: &str) -> MetricsError {
unrankable(scores, labels).unwrap_or_else(|| MetricsError::Unrankable {
count: scores.len(),
at: at.to_string(),
})
}
pub fn compute(input: Input, budgets: &[(String, f64)]) -> Result<Report, MetricsError> {
if input.scores.len() != input.labels.len() {
return Err(MetricsError::LengthMismatch {
scores: input.scores.len(),
labels: input.labels.len(),
});
}
if input.scores.len() > MAX_SCORES {
return Err(MetricsError::TooManyScores {
count: input.scores.len(),
});
}
let scores: Vec<f64> = input
.scores
.into_iter()
.map(|s| s.unwrap_or(f64::NAN))
.collect();
let labels = input.labels;
if let Some(e) = unrankable(&scores, &labels) {
return Err(e);
}
let n_stego = labels.iter().filter(|l| **l).count();
let n_clean = labels.len() - n_stego;
let auc_ci95 = stegobench_metrics::roc_auc_interval(&scores, &labels, stegobench_metrics::Z_95)
.map(|ci| [ci.low, ci.high]);
let auc = stegobench_metrics::roc_auc(&scores, &labels)
.ok_or_else(|| why_unrankable(&scores, &labels, ""))?;
let mut tpr_at_fpr = BTreeMap::new();
let mut achieved_fpr = BTreeMap::new();
let mut threshold_at_fpr = BTreeMap::new();
for (text, value) in budgets {
let tpr = stegobench_metrics::tpr_at_fpr(&scores, &labels, *value).ok_or_else(|| {
why_unrankable(
&scores,
&labels,
&format!(" at a false-alarm budget of {text}"),
)
})?;
tpr_at_fpr.insert(text.clone(), tpr);
if let Some(point) = stegobench_metrics::operating_point(&scores, &labels, *value) {
achieved_fpr.insert(text.clone(), point.achieved_fpr);
if let Some(threshold) = point.threshold {
threshold_at_fpr.insert(text.clone(), threshold);
}
}
}
Ok(Report {
auc,
auc_ci95,
tpr_at_fpr,
achieved_fpr,
threshold_at_fpr,
n_clean,
n_stego,
findings: stegobench_metrics::findings(&scores, &labels),
})
}
#[cfg(test)]
mod tests {
use super::*;
fn input(scores: &[f64], labels: &[bool]) -> Input {
Input {
scores: scores.iter().map(|s| Some(*s)).collect(),
labels: labels.to_vec(),
}
}
fn budgets(at: &[&str]) -> Vec<(String, f64)> {
parse_budgets(&at.iter().map(|s| s.to_string()).collect::<Vec<_>>()).expect("rates")
}
#[test]
fn it_reports_the_same_numbers_the_crate_does() {
let scores = [0.9, 0.4, 0.5, 0.1];
let labels = [true, true, false, false];
let report = compute(input(&scores, &labels), &budgets(&["0.0", "0.5"])).expect("ranked");
assert_eq!(
report.auc,
stegobench_metrics::roc_auc(&scores, &labels).unwrap()
);
assert_eq!(report.tpr_at_fpr["0.0"], 0.5);
assert_eq!(report.tpr_at_fpr["0.5"], 1.0);
assert_eq!((report.n_clean, report.n_stego), (2, 2));
}
#[test]
fn a_budget_answers_under_the_spelling_it_was_asked_in() {
let report = compute(
input(&[0.9, 0.4, 0.5, 0.1], &[true, true, false, false]),
&budgets(&["0.5", "0.50"]),
)
.expect("ranked");
assert_eq!(report.tpr_at_fpr["0.5"], 1.0);
assert_eq!(report.tpr_at_fpr["0.50"], 1.0);
}
#[test]
fn a_budget_that_is_not_a_rate_is_refused_by_name() {
for text in ["-0.1", "1.5", "NaN", "one per cent", ""] {
let e = parse_budgets(&[text.to_string()]).expect_err("refused");
assert_eq!(e.reason(), "budget-not-a-rate", "{text}");
assert_eq!(e.exit_code(), exit::USAGE);
assert!(e.to_string().contains("between 0 and 1"), "{e}");
}
assert_eq!(budgets(&["0", "1"]).len(), 2);
}
#[test]
fn the_same_budget_twice_is_refused_rather_than_reported_twice() {
let e = parse_budgets(&["0.01".into(), "0.01".into()]).expect_err("refused");
assert_eq!(e.reason(), "duplicate-budget");
assert_eq!(e.exit_code(), exit::USAGE);
}
#[test]
fn a_length_mismatch_is_refused_rather_than_truncated() {
let e = compute(
Input {
scores: vec![Some(0.1), Some(0.2), Some(0.3)],
labels: vec![true, false],
},
&budgets(&["0.01"]),
)
.expect_err("refused");
assert_eq!(e.reason(), "length-mismatch");
assert_eq!(e.exit_code(), exit::USAGE);
assert!(
e.to_string().contains('3') && e.to_string().contains('2'),
"{e}"
);
}
#[test]
fn an_empty_set_is_refused_rather_than_scored_zero() {
let e = compute(
Input {
scores: vec![],
labels: vec![],
},
&budgets(&["0.01"]),
)
.expect_err("refused");
assert_eq!(e.reason(), "empty");
assert_eq!(e.exit_code(), exit::PREFLIGHT_REFUSED);
}
#[test]
fn a_score_the_detector_could_not_produce_is_refused_by_count() {
let e = compute(
Input {
scores: vec![Some(0.9), None, Some(0.2), None],
labels: vec![true, true, false, false],
},
&budgets(&["0.01"]),
)
.expect_err("refused");
assert_eq!(e.reason(), "not-a-number");
assert_eq!(e.exit_code(), exit::PREFLIGHT_REFUSED);
assert!(e.to_string().contains("2 of the 4"), "{e}");
}
#[test]
fn a_nan_reaching_compute_directly_is_refused_too() {
let e = compute(
input(&[0.9, f64::NAN, 0.2, 0.1], &[true, true, false, false]),
&budgets(&["0.01"]),
)
.expect_err("refused");
assert_eq!(e.reason(), "not-a-number");
}
#[test]
fn one_class_is_refused_with_both_counts() {
let e =
compute(input(&[0.1, 0.2], &[true, true]), &budgets(&["0.01"])).expect_err("refused");
assert_eq!(e.reason(), "one-sided");
assert_eq!(e.exit_code(), exit::PREFLIGHT_REFUSED);
assert!(e.to_string().contains("0 clean and 2 stego"), "{e}");
}
#[test]
fn more_scores_than_the_cap_are_refused_before_they_are_ranked() {
let e = compute(
Input {
scores: vec![Some(0.0); MAX_SCORES + 1],
labels: vec![true; MAX_SCORES + 1],
},
&budgets(&["0.01"]),
)
.expect_err("refused");
assert_eq!(e.reason(), "too-many-scores");
assert_eq!(e.exit_code(), exit::PREFLIGHT_REFUSED);
}
#[test]
fn a_file_that_is_not_scores_and_labels_says_what_one_looks_like() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("not-scores.json");
std::fs::write(&path, r#"{"auc": 0.5}"#).expect("write");
let e = run(Some(&path), &["0.01".to_string()]).expect_err("refused");
assert_eq!(e.reason(), "not-json");
assert_eq!(e.exit_code(), exit::USAGE);
assert!(e.to_string().contains("\"scores\""), "{e}");
}
#[test]
fn a_file_that_is_not_there_names_itself() {
let e = run(
Some(Path::new("/nonexistent/scores.json")),
&["0.01".into()],
)
.expect_err("refused");
assert_eq!(e.reason(), "read-failed");
assert_eq!(e.exit_code(), exit::FAILURE);
assert!(e.to_string().contains("scores.json"), "{e}");
}
#[test]
fn a_file_over_the_byte_cap_is_refused_before_it_is_parsed() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("huge.json");
let handle = std::fs::File::create(&path).expect("create");
handle.set_len(MAX_INPUT_BYTES + 1).expect("set_len");
drop(handle);
let e = run(Some(&path), &["0.01".into()]).expect_err("refused");
assert_eq!(e.reason(), "input-too-large");
assert_eq!(e.exit_code(), exit::PREFLIGHT_REFUSED);
}
#[test]
fn a_whole_run_from_a_file_produces_the_numbers_and_the_counts() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("scores.json");
std::fs::write(
&path,
r#"{"scores": [0.9, 0.4, 0.5, 0.1], "labels": [true, true, false, false]}"#,
)
.expect("write");
let report = run(Some(&path), &["0.0".into(), "0.5".into()]).expect("ranked");
assert_eq!(report.tpr_at_fpr["0.0"], 0.5);
assert_eq!(report.n_clean, 2);
let json = report.to_json();
assert_eq!(json["n"], 4);
assert_eq!(json["ok"], true);
assert!(report.human().contains("AUC"), "{}", report.human());
}
#[test]
fn an_unknown_field_in_the_input_is_refused() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("extra.json");
std::fs::write(
&path,
r#"{"scores": [0.9, 0.1], "labels": [true, false], "at": [0.01]}"#,
)
.expect("write");
let e = run(Some(&path), &["0.01".into()]).expect_err("refused");
assert_eq!(e.reason(), "not-json");
}
#[test]
fn an_operating_point_exactly_on_the_budget_is_inside_it() {
let mut scores = Vec::new();
let mut labels = Vec::new();
scores.push(1.0);
labels.push(false);
for _ in 0..40 {
scores.push(0.9);
labels.push(true);
}
for _ in 0..99 {
scores.push(0.5);
labels.push(false);
}
for _ in 0..60 {
scores.push(0.1);
labels.push(true);
}
let report = compute(input(&scores, &labels), &budgets(&["0.01"])).expect("ranked");
assert_eq!(report.tpr_at_fpr["0.01"], 0.4);
assert_eq!((report.n_clean, report.n_stego), (100, 100));
}
#[test]
fn every_reason_is_a_distinct_word() {
let reasons = [
MetricsError::Read {
path: "x".into(),
source: std::io::Error::other("x"),
},
MetricsError::TooLarge { path: "x".into() },
MetricsError::TooManyScores { count: 1 },
MetricsError::BudgetNotARate { text: "x".into() },
MetricsError::DuplicateBudget { text: "x".into() },
MetricsError::LengthMismatch {
scores: 1,
labels: 2,
},
MetricsError::Empty,
MetricsError::NotANumber { count: 1, total: 2 },
MetricsError::OneSided { clean: 0, stego: 2 },
MetricsError::Unrankable {
count: 2,
at: String::new(),
},
]
.iter()
.map(|e| e.reason())
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(reasons.len(), 10);
}
#[test]
fn an_inverted_detector_is_warned_about_the_way_score_warns_about_it() {
let report = compute(
input(&[0.1, 0.2, 0.8, 0.9], &[true, true, false, false]),
&[],
)
.expect("ranked");
assert_eq!(report.auc, 0.0);
let said = report.warnings();
assert_eq!(said.len(), 1, "{said:?}");
assert!(said[0].contains("wrong way round"), "{said:?}");
assert!(said[0].contains("0.0000"), "{said:?}");
assert!(said[0].contains("1.0000"), "{said:?}");
assert_eq!(
report.to_json()["warnings"],
serde_json::json!(said),
"the warning is missing from the machine-readable output"
);
}
#[test]
fn one_answer_to_everything_is_warned_about_rather_than_read_as_chance() {
let report = compute(
input(&[0.5, 0.5, 0.5, 0.5], &[true, true, false, false]),
&[],
)
.expect("ranked");
assert_eq!(report.auc, 0.5);
let said = report.warnings();
assert_eq!(said.len(), 1, "{said:?}");
assert!(said[0].contains("identical"), "{said:?}");
assert!(said[0].contains("by construction"), "{said:?}");
assert!(said[0].contains('4'), "{said:?}");
}
#[test]
fn a_detector_that_separates_the_images_is_not_warned_about() {
let report = compute(
input(&[0.9, 0.8, 0.2, 0.1], &[true, true, false, false]),
&budgets(&["0.5"]),
)
.expect("ranked");
assert!(report.warnings().is_empty(), "{:?}", report.warnings());
assert_eq!(report.to_json()["warnings"], serde_json::json!([]));
assert!(!report.human().contains("way round"), "{}", report.human());
}
#[test]
fn a_number_is_rendered_the_way_score_renders_it() {
let input = Input {
scores: vec![Some(0.9), Some(0.1), Some(0.8), Some(0.2)],
labels: vec![true, false, true, false],
};
let report = compute(input, &[("0.50".to_string(), 0.5)]).expect("rankable");
let text = report.human();
assert!(text.contains("AUC 1.0000"), "{text}");
assert!(text.contains("TPR at 0.50 1.0000"), "{text}");
assert!(text.contains("95% interval not estimated"), "{text}");
assert!(text.contains("separate with nothing left over"), "{text}");
assert!(!text.contains("fewer than two members"), "{text}");
let spread = compute(
Input {
scores: vec![
Some(0.9),
Some(0.5),
Some(0.4),
Some(0.2),
Some(0.8),
Some(0.7),
],
labels: vec![true, false, true, false, true, false],
},
&[],
)
.expect("rankable");
assert!(
spread.human().contains("95% interval ["),
"{}",
spread.human()
);
assert_eq!(report.to_json()["auc"], serde_json::json!(1.0));
}
}
#[cfg(test)]
mod nothing_to_read_at_all {
use super::*;
#[test]
fn a_terminal_with_no_file_is_refused_rather_than_waited_on() {
let e = nothing_to_read(None, true).expect("refused");
assert_eq!(e.exit_code(), exit::USAGE);
let said = e.to_string();
assert!(said.contains("metrics scores.json"), "{said}");
assert!(said.contains("standard input"), "{said}");
assert!(said.contains("| stegobench metrics"), "{said}");
assert_eq!(e.reason(), "nothing-to-read");
}
#[test]
fn a_pipe_is_still_waited_on() {
assert!(nothing_to_read(None, false).is_none());
}
#[test]
fn naming_a_file_never_consults_the_terminal() {
let path = std::path::PathBuf::from("scores.json");
assert!(nothing_to_read(Some(&path), true).is_none());
assert!(nothing_to_read(Some(&path), false).is_none());
}
}