llm_verify/
report.rs

1// SPDX-License-Identifier: Apache-2.0
2//! The data model every probe writes into and every output format reads from.
3
4use crate::i18n::Lang;
5use crate::protocol::Protocol;
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8use std::collections::BTreeMap;
9
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
11#[serde(rename_all = "lowercase")]
12pub enum Status {
13    Pass,
14    Warn,
15    Fail,
16    /// The endpoint does not offer what this probe needs. Not a defect.
17    Skip,
18    /// The probe itself could not run (network, timeout). Coverage gap.
19    Error,
20}
21
22impl Status {
23    pub fn symbol(&self) -> &'static str {
24        match self {
25            Self::Pass => "✓",
26            Self::Warn => "!",
27            Self::Fail => "✗",
28            Self::Skip => "–",
29            Self::Error => "?",
30        }
31    }
32
33    pub fn label(&self, lang: Lang) -> &'static str {
34        match self {
35            Self::Pass => ts!(lang, "pass", "通过"),
36            Self::Warn => ts!(lang, "warn", "警告"),
37            Self::Fail => ts!(lang, "fail", "失败"),
38            Self::Skip => ts!(lang, "skip", "跳过"),
39            Self::Error => ts!(lang, "error", "错误"),
40        }
41    }
42
43    pub fn css(&self) -> &'static str {
44        match self {
45            Self::Pass => "pass",
46            Self::Warn => "warn",
47            Self::Fail => "fail",
48            Self::Skip => "skip",
49            Self::Error => "err",
50        }
51    }
52
53    /// Whether this outcome counts toward the weighted score at all.
54    pub fn scored(&self) -> bool {
55        matches!(self, Self::Pass | Self::Warn | Self::Fail)
56    }
57
58    pub fn credit(&self) -> f64 {
59        match self {
60            Self::Pass => 1.0,
61            Self::Warn => 0.5,
62            _ => 0.0,
63        }
64    }
65}
66
67#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
68#[serde(rename_all = "lowercase")]
69pub enum Group {
70    Contract,
71    Stream,
72    Billing,
73    Channel,
74    Perf,
75    Identity,
76    Consistency,
77}
78
79impl Group {
80    pub const ALL: [Group; 7] = [
81        Group::Contract,
82        Group::Stream,
83        Group::Billing,
84        Group::Channel,
85        Group::Perf,
86        Group::Identity,
87        Group::Consistency,
88    ];
89
90    pub fn label(&self, lang: Lang) -> &'static str {
91        match self {
92            Self::Contract => ts!(lang, "Protocol contract", "协议契约"),
93            Self::Stream => ts!(lang, "Streaming", "流式传输"),
94            Self::Billing => ts!(lang, "Metering & billing", "计量计费"),
95            Self::Channel => ts!(lang, "Channel provenance", "渠道溯源"),
96            Self::Perf => ts!(lang, "Performance", "性能速度"),
97            Self::Identity => ts!(lang, "Model identity", "模型身份"),
98            Self::Consistency => ts!(lang, "Cross-request consistency", "跨请求一致性"),
99        }
100    }
101
102    pub fn blurb(&self, lang: Lang) -> &'static str {
103        match self {
104            Self::Contract => ts!(
105                lang,
106                "Is this a genuine API channel?",
107                "这是不是一条正牌 API 通道"
108            ),
109            Self::Stream => ts!(
110                lang,
111                "Does streaming follow the protocol, or arrive empty?",
112                "流式响应是否符合协议，有没有空 body"
113            ),
114            Self::Billing => ts!(
115                lang,
116                "Are the token counts honest, or are you overcharged?",
117                "计量数字可信吗，有没有多收钱"
118            ),
119            Self::Channel => ts!(
120                lang,
121                "What relays sit on this path?",
122                "这条链路上有哪些中转"
123            ),
124            Self::Perf => ts!(
125                lang,
126                "First-token latency, throughput and jitter",
127                "首字延迟、吞吐与抖动"
128            ),
129            Self::Identity => ts!(
130                lang,
131                "Is the model behind this the one that was sold?",
132                "背后跑的是不是它声称的那个模型"
133            ),
134            Self::Consistency => ts!(
135                lang,
136                "Does the endpoint behave the same way every time?",
137                "多次请求的行为是否一致"
138            ),
139        }
140    }
141
142    pub fn key(&self) -> &'static str {
143        match self {
144            Self::Contract => "contract",
145            Self::Stream => "stream",
146            Self::Billing => "billing",
147            Self::Channel => "channel",
148            Self::Perf => "perf",
149            Self::Identity => "identity",
150            Self::Consistency => "consistency",
151        }
152    }
153}
154
155#[derive(Debug, Clone, Serialize, Deserialize)]
156pub struct ProbeResult {
157    pub id: String,
158    pub label: String,
159    pub group: Group,
160    pub status: Status,
161    /// Relative importance inside the weighted score.
162    pub weight: u32,
163    /// Neutral probes gather evidence for the verdict but never move the score.
164    pub neutral: bool,
165    pub summary: String,
166    #[serde(skip_serializing_if = "Vec::is_empty")]
167    pub findings: Vec<String>,
168    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
169    pub metrics: BTreeMap<String, Value>,
170    pub duration_ms: u64,
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub evidence: Option<String>,
173}
174
175impl ProbeResult {
176    pub fn new(id: &str, label: &str, group: Group) -> Self {
177        Self {
178            id: id.to_string(),
179            label: label.to_string(),
180            group,
181            status: Status::Pass,
182            weight: 1,
183            neutral: false,
184            summary: String::new(),
185            findings: Vec::new(),
186            metrics: BTreeMap::new(),
187            duration_ms: 0,
188            evidence: None,
189        }
190    }
191
192    pub fn weight(mut self, w: u32) -> Self {
193        self.weight = w;
194        self
195    }
196
197    pub fn neutral(mut self) -> Self {
198        self.neutral = true;
199        self
200    }
201
202    pub fn pass(mut self, summary: impl Into<String>) -> Self {
203        self.status = Status::Pass;
204        self.summary = summary.into();
205        self
206    }
207
208    pub fn warn(mut self, summary: impl Into<String>) -> Self {
209        self.status = Status::Warn;
210        self.summary = summary.into();
211        self
212    }
213
214    pub fn fail(mut self, summary: impl Into<String>) -> Self {
215        self.status = Status::Fail;
216        self.summary = summary.into();
217        self
218    }
219
220    pub fn skip(mut self, summary: impl Into<String>) -> Self {
221        self.status = Status::Skip;
222        self.summary = summary.into();
223        self
224    }
225
226    pub fn error(mut self, summary: impl Into<String>) -> Self {
227        self.status = Status::Error;
228        self.summary = summary.into();
229        self
230    }
231
232    pub fn finding(mut self, f: impl Into<String>) -> Self {
233        self.findings.push(f.into());
234        self
235    }
236
237    pub fn metric(mut self, k: &str, v: impl Into<Value>) -> Self {
238        self.metrics.insert(k.to_string(), v.into());
239        self
240    }
241
242    pub fn evidence(mut self, e: impl Into<String>) -> Self {
243        let e = e.into();
244        if !e.trim().is_empty() {
245            self.evidence = Some(crate::util::truncate(e.trim(), 600));
246        }
247        self
248    }
249
250    pub fn took(mut self, ms: u64) -> Self {
251        self.duration_ms = ms;
252        self
253    }
254
255    pub fn metric_f64(&self, k: &str) -> Option<f64> {
256        self.metrics.get(k).and_then(|v| v.as_f64())
257    }
258
259    pub fn metric_bool(&self, k: &str) -> Option<bool> {
260        self.metrics.get(k).and_then(|v| v.as_bool())
261    }
262}
263
264// ── verdict ────────────────────────────────────────────────────────────────
265
266/// Axis 1 — is the response genuine?
267#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
268#[serde(rename_all = "snake_case")]
269pub enum Authenticity {
270    Authentic,
271    AuthenticDegraded,
272    ThirdParty,
273    Suspicious,
274    Counterfeit,
275    /// The default: a verdict nobody has reached yet is "we do not know", never
276    /// "clean".
277    #[default]
278    Inconclusive,
279}
280
281impl Authenticity {
282    pub fn label(&self, lang: Lang) -> &'static str {
283        match self {
284            Self::Authentic => ts!(lang, "Genuine", "正品"),
285            Self::AuthenticDegraded => ts!(lang, "Genuine, with defects", "正品（有瑕疵）"),
286            Self::ThirdParty => ts!(lang, "Relayed", "第三方转发"),
287            Self::Suspicious => ts!(lang, "Suspicious", "存疑"),
288            Self::Counterfeit => ts!(lang, "Counterfeit", "假冒"),
289            Self::Inconclusive => ts!(lang, "Inconclusive", "无法判定"),
290        }
291    }
292
293    pub fn desc(&self, lang: Lang) -> &'static str {
294        match self {
295            Self::Authentic => ts!(
296                lang,
297                "Contract, metering and identity signals all line up. Safe to rely on.",
298                "协议契约、计量与身份信号全部对齐，可以放心使用。"
299            ),
300            Self::AuthenticDegraded => ts!(
301                lang,
302                "The model itself looks real, but the path injects content, misreports \
303                 usage, or has capabilities missing.",
304                "模型本身看起来是真的，但链路上存在注入、计量偏差或能力缺失。"
305            ),
306            Self::ThirdParty => ts!(
307                lang,
308                "Behaviour is broadly correct, but vendor markers are absent or the \
309                 billing ratio runs high. A real model behind a relay.",
310                "行为基本正常，但缺少官方特征、或计量倍率偏高，是经过转发的真模型。"
311            ),
312            Self::Suspicious => ts!(
313                lang,
314                "Several anomalies at once. Possibly tampered with, downgraded, or \
315                 served through a reconstructed channel.",
316                "多项异常同时出现，可能被篡改、降级或经过逆向渠道。"
317            ),
318            Self::Counterfeit => ts!(
319                lang,
320                "The echoed model and the model's own self-identification both \
321                 disagree with the claim. Most likely not the model advertised.",
322                "模型回显与自我认同同时对不上，背后大概率不是它声称的模型。"
323            ),
324            Self::Inconclusive => ts!(
325                lang,
326                "Connectivity or probe coverage was too thin to support a verdict.",
327                "连通性或数据覆盖不足，不足以下判断。"
328            ),
329        }
330    }
331
332    pub fn css(&self) -> &'static str {
333        match self {
334            Self::Authentic => "v-good",
335            Self::AuthenticDegraded | Self::ThirdParty => "v-mid",
336            Self::Suspicious => "v-warn",
337            Self::Counterfeit => "v-bad",
338            Self::Inconclusive => "v-none",
339        }
340    }
341}
342
343/// Axis 2 — where did it come from? Independent of axis 1: a real model
344/// behind a relay is still a real model.
345#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
346#[serde(rename_all = "kebab-case")]
347pub enum Channel {
348    Official,
349    Cloud,
350    Subscription,
351    Proxy,
352    ReverseProxy,
353    #[default]
354    Unknown,
355}
356
357impl Channel {
358    pub fn label(&self, lang: Lang) -> &'static str {
359        match self {
360            Self::Official => ts!(lang, "Direct from vendor", "官方直连"),
361            Self::Cloud => ts!(lang, "Cloud platform", "云平台"),
362            Self::Subscription => ts!(lang, "Subscription-derived", "订阅号"),
363            Self::Proxy => ts!(lang, "Relay", "普通中转"),
364            Self::ReverseProxy => ts!(lang, "Reconstructed channel", "逆向渠道"),
365            Self::Unknown => ts!(lang, "Undetermined", "无法确定"),
366        }
367    }
368
369    pub fn desc(&self, lang: Lang) -> &'static str {
370        match self {
371            Self::Official => ts!(
372                lang,
373                "Response headers carry vendor markers; this looks like a direct \
374                 connection to the provider's own API.",
375                "响应头带官方特征，看起来是直连厂商 API。"
376            ),
377            Self::Cloud => ts!(
378                lang,
379                "Resold through a cloud platform such as AWS Bedrock or Google Vertex.",
380                "经由 AWS Bedrock / Google Vertex 等云平台转售。"
381            ),
382            Self::Subscription => ts!(
383                lang,
384                "Feature-complete but without vendor headers — the shape of an \
385                 interface derived from a subscription account.",
386                "功能完整但缺少官方响应头，像是订阅账号导出的接口。"
387            ),
388            Self::Proxy => ts!(
389                lang,
390                "Works correctly but carries no vendor markers. One relay hop.",
391                "功能正常但没有官方特征，是一层普通中转。"
392            ),
393            Self::ReverseProxy => ts!(
394                lang,
395                "Several first-party capabilities are missing, matching an \
396                 interface reconstructed from a web session.",
397                "多项官方能力缺失，特征符合从网页端逆向出来的接口。"
398            ),
399            Self::Unknown => ts!(
400                lang,
401                "Too few signals to place this endpoint on the path.",
402                "信号不足，无法确定链路来源。"
403            ),
404        }
405    }
406}
407
408#[derive(Debug, Clone, Serialize, Deserialize, Default)]
409pub struct Verdict {
410    pub authenticity: Authenticity,
411    pub channel: Channel,
412    /// 0–100 weighted score across the scored groups.
413    pub score: f64,
414    /// 0–1. Reflects both signal strength and coverage.
415    pub confidence: f64,
416    pub hard_gate_hits: Vec<GateHit>,
417    pub signals: Vec<String>,
418    /// Every step the decision took, so a user can audit the conclusion.
419    pub trace: Vec<String>,
420    pub group_scores: BTreeMap<String, f64>,
421    /// Fraction of probes that errored out — high values force a downgrade.
422    pub coverage_gap: f64,
423}
424
425#[derive(Debug, Clone, Serialize, Deserialize, Default)]
426pub struct GateHit {
427    pub name: String,
428    pub probe: String,
429    pub reason: String,
430}
431
432// ── identity ───────────────────────────────────────────────────────────────
433
434#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
435#[serde(rename_all = "snake_case")]
436pub enum IdentityStatus {
437    /// Family and tier both line up with the claim.
438    Match,
439    /// Family lines up; tier could not be confirmed.
440    FamilyOnly,
441    /// Family lines up but measured capability points at a different tier.
442    TierMismatch,
443    /// Fingerprints point at a different family than claimed.
444    FamilyMismatch,
445    /// Signals contradict each other.
446    Ambiguous,
447    #[default]
448    Insufficient,
449}
450
451impl IdentityStatus {
452    pub fn label(&self, lang: Lang) -> &'static str {
453        match self {
454            Self::Match => ts!(lang, "Matches the claim", "相符"),
455            Self::FamilyOnly => ts!(
456                lang,
457                "Family matches, tier unverified",
458                "家族相符，档位未验证"
459            ),
460            Self::TierMismatch => ts!(
461                lang,
462                "Family matches, but the tier was downgraded",
463                "家族相符，但档位被降级"
464            ),
465            Self::FamilyMismatch => ts!(lang, "Different model family", "家族不符"),
466            Self::Ambiguous => ts!(lang, "Signals contradict each other", "信号矛盾"),
467            Self::Insufficient => ts!(lang, "Not enough data", "数据不足"),
468        }
469    }
470
471    pub fn css(&self) -> &'static str {
472        match self {
473            Self::Match => "v-good",
474            Self::FamilyOnly => "v-mid",
475            Self::TierMismatch | Self::FamilyMismatch => "v-bad",
476            Self::Ambiguous => "v-warn",
477            Self::Insufficient => "v-none",
478        }
479    }
480}
481
482#[derive(Debug, Clone, Serialize, Deserialize, Default)]
483pub struct Identity {
484    pub claimed_model: String,
485    pub claimed_family: Option<String>,
486    pub claimed_tier: Option<String>,
487    pub observed_family: Option<String>,
488    pub family_confidence: f64,
489    pub estimated_tier: Option<String>,
490    pub tier_confidence: f64,
491    /// 0 = aligned, 1 = one step apart, 2 = two or more (e.g. Opus vs Haiku).
492    pub tier_severity: u8,
493    pub status: IdentityStatus,
494    pub evidence: Vec<String>,
495    pub tier_scores: BTreeMap<String, f64>,
496    pub accuracy_by_difficulty: BTreeMap<String, f64>,
497    /// How many capability questions the tier estimate rests on, and how far
498    /// the winning hypothesis beat the runner-up. Both belong in the report:
499    /// a tier call from a handful of questions with a narrow margin is a much
500    /// weaker claim than the same call from a wide one.
501    pub tier_questions: u32,
502    pub tier_margin: f64,
503}
504
505// ── billing ────────────────────────────────────────────────────────────────
506
507#[derive(Debug, Clone, Serialize, Deserialize, Default)]
508pub struct BillingAudit {
509    pub rounds: Vec<BillingRound>,
510    /// `authoritative` when the endpoint's own count_tokens answered,
511    /// `estimated` when we fell back to the local heuristic.
512    pub method: String,
513    pub billed_input: u32,
514    pub billed_output: u32,
515    pub honest_input: u32,
516    pub honest_output: u32,
517    pub input_ratio: f64,
518    pub billed_cost_usd: f64,
519    pub honest_cost_usd: f64,
520    pub cost_ratio: f64,
521    pub pricing_source: String,
522    pub anomalies: Vec<String>,
523}
524
525#[derive(Debug, Clone, Serialize, Deserialize)]
526pub struct BillingRound {
527    pub probe: String,
528    pub billed_input: u32,
529    pub honest_input: u32,
530    pub billed_output: u32,
531    pub ratio: f64,
532}
533
534// ── channel ────────────────────────────────────────────────────────────────
535
536#[derive(Debug, Clone, Serialize, Deserialize, Default)]
537pub struct ChannelSignature {
538    /// Stable classifier key. The verdict layer routes on this.
539    pub key: String,
540    /// Localised name for display.
541    pub display: String,
542    pub confidence: f64,
543    pub tier: u8,
544    pub evidence: Vec<String>,
545    /// Every relay vendor we saw a signature for; more than one means the
546    /// request passed through more than one hop.
547    pub all_hops: Vec<String>,
548}
549
550// ── perf ───────────────────────────────────────────────────────────────────
551
552#[derive(Debug, Clone, Serialize, Deserialize, Default)]
553pub struct PerfSummary {
554    pub samples: usize,
555    pub ttft_ms: Vec<f64>,
556    pub latency_ms: Vec<f64>,
557    pub tps: Vec<f64>,
558    pub ttft_p50: f64,
559    pub ttft_p95: f64,
560    pub latency_p50: f64,
561    pub latency_p95: f64,
562    pub tps_mean: f64,
563    pub latency_cv: f64,
564}
565
566// ── whole run ──────────────────────────────────────────────────────────────
567
568#[derive(Debug, Clone, Serialize, Deserialize)]
569pub struct Report {
570    /// Shape of this JSON, independent of [`Report::tool_version`].
571    ///
572    /// A caller that stores reports needs to know whether an old row can still
573    /// be deserialised, and the tool version cannot answer that: most releases
574    /// change no field at all. Bumped only when a field is removed or its
575    /// meaning changes; additive fields do not move it.
576    #[serde(default = "schema_version")]
577    pub schema_version: u32,
578    pub tool_version: String,
579    /// Language every human-readable string in this report is written in.
580    pub lang: Lang,
581    pub started_at: String,
582    pub finished_at: String,
583    pub duration_ms: u64,
584    pub host: String,
585    pub base_url: String,
586    pub protocol: Protocol,
587    pub model: String,
588    pub claimed_model: String,
589    pub depth: String,
590    /// The random seed this run's payloads were generated from. Replaying the
591    /// same seed, depth and step list reproduces the same probes — which is
592    /// what makes a verdict auditable after the fact.
593    ///
594    /// Serialised as a **string**, and that is not cosmetic. A `u64` seed
595    /// routinely exceeds 2^53, and JSON numbers above that lose precision in
596    /// every JavaScript consumer — a report rendered in a browser showed
597    /// `2543944364647182727` as `25439443646471830000`, next to a sentence
598    /// promising the run could be replayed from it. A seed that cannot be
599    /// copied accurately is worse than no seed, because it looks like evidence.
600    #[serde(default, with = "seed_repr")]
601    pub seed: u64,
602    /// Registry ids of the steps that ran, in order. A report that covers part
603    /// of the suite must say which part: absent evidence is not evidence.
604    #[serde(default)]
605    pub steps: Vec<String>,
606    pub request_count: u32,
607    pub results: Vec<ProbeResult>,
608    pub verdict: Verdict,
609    pub identity: Identity,
610    pub billing: BillingAudit,
611    pub channel: ChannelSignature,
612    pub perf: PerfSummary,
613    /// Probes that never ran, and why — so "not tested" is never read as "passed".
614    pub skipped: Vec<String>,
615}
616
617/// A `u64` that survives a JSON round trip through a JavaScript consumer.
618///
619/// Writes a string; reads either, so reports written before the change still
620/// load.
621mod seed_repr {
622    use serde::{Deserialize, Deserializer, Serializer};
623
624    pub fn serialize<S: Serializer>(v: &u64, s: S) -> Result<S::Ok, S::Error> {
625        s.serialize_str(&v.to_string())
626    }
627
628    pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<u64, D::Error> {
629        #[derive(Deserialize)]
630        #[serde(untagged)]
631        enum Either {
632            S(String),
633            N(u64),
634        }
635        Ok(match Either::deserialize(d)? {
636            Either::S(s) => s.parse().unwrap_or(0),
637            Either::N(n) => n,
638        })
639    }
640}
641
642/// Current [`Report::schema_version`].
643pub const fn schema_version() -> u32 {
644    1
645}
646
647impl Report {
648    pub fn by_group(&self, g: Group) -> Vec<&ProbeResult> {
649        self.results.iter().filter(|r| r.group == g).collect()
650    }
651
652    pub fn count(&self, s: Status) -> usize {
653        self.results.iter().filter(|r| r.status == s).count()
654    }
655
656    /// Exit code contract: 0 clean, 1 failing score, 2 hard gate tripped.
657    pub fn exit_code(&self) -> i32 {
658        if !self.verdict.hard_gate_hits.is_empty() {
659            return 2;
660        }
661        if matches!(
662            self.verdict.authenticity,
663            Authenticity::Counterfeit | Authenticity::Suspicious
664        ) || self.verdict.score < 60.0
665        {
666            return 1;
667        }
668        0
669    }
670}
671
672#[cfg(test)]
673mod tests {
674    use super::*;
675
676    /// A caller that stores a verdict has to be able to read it back — for an
677    /// evidence view, for an appeal, for a rescore after the policy changes.
678    /// A report that only serialises is half a feature, and the half that is
679    /// missing is the one an accused seller needs.
680    #[test]
681    fn a_report_survives_a_round_trip_through_json() {
682        let before = report_with(Authenticity::Suspicious, 61.5, vec![]);
683        let json = serde_json::to_string(&before).unwrap();
684        let after: Report = serde_json::from_str(&json).unwrap();
685        assert_eq!(after.schema_version, before.schema_version);
686        assert_eq!(after.verdict.authenticity, before.verdict.authenticity);
687        assert_eq!(after.verdict.score, before.verdict.score);
688        assert_eq!(after.seed, before.seed);
689        assert_eq!(after.steps, before.steps);
690    }
691
692    /// An unknown verdict must never deserialise as a clean one.
693    /// A seed that a browser cannot read back accurately is not evidence.
694    #[test]
695    fn a_large_seed_survives_a_json_round_trip() {
696        let mut before = report_with(Authenticity::ThirdParty, 90.0, vec![]);
697        // Bigger than 2^53, which is where a JavaScript `number` starts lying.
698        before.seed = 2_543_944_364_647_182_727;
699        let json = serde_json::to_string(&before).unwrap();
700        assert!(
701            json.contains("\"seed\":\"2543944364647182727\""),
702            "the seed must go out as a string: {json}"
703        );
704        let after: Report = serde_json::from_str(&json).unwrap();
705        assert_eq!(after.seed, before.seed);
706
707        // And a report written before the change still loads.
708        let legacy = json.replace("\"seed\":\"2543944364647182727\"", "\"seed\":123");
709        assert_eq!(serde_json::from_str::<Report>(&legacy).unwrap().seed, 123);
710    }
711
712    #[test]
713    fn the_default_verdict_is_inconclusive_not_authentic() {
714        assert_eq!(Authenticity::default(), Authenticity::Inconclusive);
715        assert_eq!(Channel::default(), Channel::Unknown);
716    }
717
718    #[test]
719    fn status_credit_matches_scoring_rules() {
720        assert_eq!(Status::Pass.credit(), 1.0);
721        assert_eq!(Status::Warn.credit(), 0.5);
722        assert_eq!(Status::Fail.credit(), 0.0);
723        // Skip and Error must not be worth partial credit, and must not
724        // count in the denominator either.
725        assert_eq!(Status::Skip.credit(), 0.0);
726        assert!(!Status::Skip.scored());
727        assert!(!Status::Error.scored());
728        assert!(Status::Fail.scored());
729    }
730
731    #[test]
732    fn probe_builder_chains() {
733        let p = ProbeResult::new("x", "测试", Group::Contract)
734            .weight(3)
735            .fail("boom")
736            .finding("detail")
737            .metric("n", 4)
738            .took(120);
739        assert_eq!(p.status, Status::Fail);
740        assert_eq!(p.weight, 3);
741        assert_eq!(p.metric_f64("n"), Some(4.0));
742        assert_eq!(p.duration_ms, 120);
743        assert_eq!(p.findings.len(), 1);
744    }
745
746    #[test]
747    fn evidence_is_trimmed_and_empty_is_dropped() {
748        let p = ProbeResult::new("x", "l", Group::Contract).evidence("   ");
749        assert!(p.evidence.is_none());
750        let p = ProbeResult::new("x", "l", Group::Contract).evidence("  hi  ");
751        assert_eq!(p.evidence.as_deref(), Some("hi"));
752    }
753
754    fn report_with(auth: Authenticity, score: f64, gates: Vec<GateHit>) -> Report {
755        Report {
756            schema_version: schema_version(),
757            tool_version: "t".into(),
758            lang: Lang::En,
759            started_at: String::new(),
760            finished_at: String::new(),
761            duration_ms: 0,
762            host: String::new(),
763            base_url: String::new(),
764            protocol: Protocol::Anthropic,
765            model: String::new(),
766            claimed_model: String::new(),
767            depth: "balanced".into(),
768            seed: 0,
769            steps: vec![],
770            request_count: 0,
771            results: vec![],
772            verdict: Verdict {
773                authenticity: auth,
774                channel: Channel::Unknown,
775                score,
776                confidence: 0.5,
777                hard_gate_hits: gates,
778                signals: vec![],
779                trace: vec![],
780                group_scores: BTreeMap::new(),
781                coverage_gap: 0.0,
782            },
783            identity: Identity::default(),
784            billing: BillingAudit::default(),
785            channel: ChannelSignature::default(),
786            perf: PerfSummary::default(),
787            skipped: vec![],
788        }
789    }
790
791    #[test]
792    fn exit_code_prioritises_hard_gates() {
793        let gate = GateHit {
794            name: "g".into(),
795            probe: "p".into(),
796            reason: "r".into(),
797        };
798        assert_eq!(
799            report_with(Authenticity::Authentic, 99.0, vec![gate]).exit_code(),
800            2
801        );
802        assert_eq!(
803            report_with(Authenticity::Counterfeit, 99.0, vec![]).exit_code(),
804            1
805        );
806        assert_eq!(
807            report_with(Authenticity::Authentic, 42.0, vec![]).exit_code(),
808            1
809        );
810        assert_eq!(
811            report_with(Authenticity::Authentic, 88.0, vec![]).exit_code(),
812            0
813        );
814        assert_eq!(
815            report_with(Authenticity::ThirdParty, 88.0, vec![]).exit_code(),
816            0,
817            "a relayed real model is not a failure"
818        );
819    }
820}
llm_verify/report.rs

llm_verify/
report.rs
