harn_vm/llm_config/
reviewer.rs

1//! Complementary-reviewer selection: pick a different-family, price-capped
2//! reviewer model for review/critique/plan-review workloads, with
3//! deterministic fallback to the author model.
4use std::collections::BTreeSet;
5
6use serde::Serialize;
7
8use super::*;
9
10#[derive(Debug, Clone, PartialEq)]
11pub struct ComplementaryReviewerOptions {
12    pub author_model: String,
13    pub author_provider: Option<String>,
14    pub intent: ComplementaryReviewerIntent,
15    pub max_price_multiplier: Option<f64>,
16}
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum ComplementaryReviewerIntent {
20    Review,
21    Critique,
22    PlanReview,
23}
24
25impl ComplementaryReviewerIntent {
26    pub fn parse(value: &str) -> Option<Self> {
27        match value {
28            "review" => Some(Self::Review),
29            "critique" => Some(Self::Critique),
30            "plan_review" => Some(Self::PlanReview),
31            _ => None,
32        }
33    }
34
35    pub fn as_str(self) -> &'static str {
36        match self {
37            Self::Review => "review",
38            Self::Critique => "critique",
39            Self::PlanReview => "plan_review",
40        }
41    }
42}
43
44#[derive(Debug, Clone, Serialize, PartialEq)]
45pub struct ComplementaryReviewerSelection {
46    pub intent: String,
47    pub author: ComplementaryModelIdentity,
48    pub reviewer: ComplementaryModelIdentity,
49    pub fallback: bool,
50    pub fallback_reason: Option<String>,
51    /// Machine-readable reason a caller can branch on when `fallback` is
52    /// `true`, distinct from the human-readable `fallback_reason`/`reason`
53    /// prose. `None` on the success path. Lets a caller hard-fail an
54    /// independent-review step rather than silently degrade to self-review.
55    /// See [`ReviewerFallbackCode`] for the stable set of values.
56    #[serde(skip_serializing_if = "Option::is_none")]
57    pub fallback_code: Option<String>,
58    pub reason: String,
59    pub estimated_incremental_cost: Option<ComplementaryCostEstimate>,
60}
61
62/// Stable, machine-readable reasons `pick_complementary_reviewer` falls back
63/// to the author model. Serialized as the `fallback_code` string so harn
64/// pipelines and Rust callers can branch deterministically instead of parsing
65/// prose. New variants are additive; existing codes are append-only contract.
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67pub enum ReviewerFallbackCode {
68    /// The author model's family could not be resolved, so no independent
69    /// family comparison is possible.
70    UnknownAuthorFamily,
71    /// Different-family candidates exist but none satisfy `max_price_multiplier`.
72    NoDiffFamilyWithinPrice,
73    /// No active, serverless, different-family reviewer is cataloged at all.
74    NoDiffFamilyServerless,
75    /// Different-family candidates exist but were all excluded (e.g. every
76    /// one declares `avoid_as_reviewer_for` the author).
77    AllDiffFamilyExcluded,
78}
79
80impl ReviewerFallbackCode {
81    pub fn as_code(self) -> &'static str {
82        match self {
83            Self::UnknownAuthorFamily => "unknown_author_family",
84            Self::NoDiffFamilyWithinPrice => "no_diff_family_within_price",
85            Self::NoDiffFamilyServerless => "no_diff_family_serverless",
86            Self::AllDiffFamilyExcluded => "all_diff_family_excluded",
87        }
88    }
89}
90
91#[derive(Debug, Clone, Serialize, PartialEq)]
92pub struct ComplementaryModelIdentity {
93    pub id: String,
94    pub provider: String,
95    pub family: String,
96    pub lineage: String,
97    pub tier: String,
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub pricing: Option<ModelPricing>,
100}
101
102#[derive(Debug, Clone, Serialize, PartialEq)]
103pub struct ComplementaryCostEstimate {
104    pub input_per_mtok: f64,
105    pub output_per_mtok: f64,
106    pub total_per_mtok: f64,
107    #[serde(skip_serializing_if = "Option::is_none")]
108    pub multiplier_vs_author: Option<f64>,
109}
110
111pub fn pick_complementary_reviewer(
112    options: ComplementaryReviewerOptions,
113) -> ComplementaryReviewerSelection {
114    let config = effective_config();
115    let mut author = resolve_model_info(&options.author_model);
116    if let Some(provider) = options
117        .author_provider
118        .as_deref()
119        .map(str::trim)
120        .filter(|provider| !provider.is_empty())
121    {
122        author.provider = provider.to_string();
123        author.family = model_family_with_config(&config, &author.provider, &author.id);
124        author.lineage = model_lineage_with_config(&config, &author.provider, &author.id);
125        author.tool_format = default_tool_format_with_config(&config, &author.id, &author.provider);
126    }
127    let author_entry = config.models.get(&author.id);
128    let author_identity = complementary_identity(
129        author.id.clone(),
130        author.provider.clone(),
131        author.family.clone(),
132        author.lineage.clone(),
133        author.tier.clone(),
134        author_entry.and_then(|model| model.pricing.clone()),
135    );
136
137    let fallback =
138        |code: ReviewerFallbackCode, fallback_reason: String| ComplementaryReviewerSelection {
139            intent: options.intent.as_str().to_string(),
140            reviewer: author_identity.clone(),
141            estimated_incremental_cost: cost_estimate(
142                author_identity.pricing.as_ref(),
143                author_identity.pricing.as_ref(),
144            ),
145            author: author_identity.clone(),
146            fallback: true,
147            reason: format!(
148                "using author model {} because {fallback_reason}",
149                author_identity.id
150            ),
151            fallback_reason: Some(fallback_reason),
152            fallback_code: Some(code.as_code().to_string()),
153        };
154
155    if author_identity.family == "unknown" {
156        return fallback(
157            ReviewerFallbackCode::UnknownAuthorFamily,
158            "author model family is unknown".to_string(),
159        );
160    }
161
162    let preferred_families = author_entry
163        .map(|model| model.complementary_with.clone())
164        .unwrap_or_default();
165    let author_refs = reviewer_match_refs(&author_identity);
166    let mut rejected_by_price = 0usize;
167    let mut diff_family_seen = 0usize;
168    let mut candidates = Vec::new();
169
170    for (id, model) in config.models.iter() {
171        if id == &author_identity.id && model.provider == author_identity.provider {
172            continue;
173        }
174        if model.deprecated || model.availability != ModelAvailability::Serverless {
175            continue;
176        }
177        let family = model_family_with_config(&config, &model.provider, id);
178        if family == "unknown" || family == author_identity.family {
179            continue;
180        }
181        diff_family_seen += 1;
182        let lineage = model_lineage_with_config(&config, &model.provider, id);
183        let candidate_identity = complementary_identity(
184            id.clone(),
185            model.provider.clone(),
186            family,
187            lineage,
188            model_tier_with_config(&config, id),
189            model.pricing.clone(),
190        );
191        if model
192            .avoid_as_reviewer_for
193            .iter()
194            .any(|selector| refs_contain_selector(&author_refs, selector))
195        {
196            continue;
197        }
198        if exceeds_price_cap(
199            author_identity.pricing.as_ref(),
200            candidate_identity.pricing.as_ref(),
201            options.max_price_multiplier,
202        ) {
203            rejected_by_price += 1;
204            continue;
205        }
206        let score = reviewer_score(
207            &options,
208            &author_identity,
209            &candidate_identity,
210            model,
211            &preferred_families,
212        );
213        candidates.push(ReviewerCandidate {
214            identity: candidate_identity,
215            score,
216        });
217    }
218
219    candidates.sort_by(|left, right| {
220        right
221            .score
222            .partial_cmp(&left.score)
223            .unwrap_or(std::cmp::Ordering::Equal)
224            .then_with(|| left.identity.provider.cmp(&right.identity.provider))
225            .then_with(|| left.identity.id.cmp(&right.identity.id))
226    });
227
228    let Some(best) = candidates.into_iter().next() else {
229        if rejected_by_price > 0 {
230            let cap = options.max_price_multiplier.unwrap_or_default();
231            return fallback(
232                ReviewerFallbackCode::NoDiffFamilyWithinPrice,
233                format!("no different-family reviewer satisfied max_price_multiplier {cap}"),
234            );
235        }
236        if diff_family_seen == 0 {
237            return fallback(
238                ReviewerFallbackCode::NoDiffFamilyServerless,
239                "no active serverless different-family reviewer is cataloged".to_string(),
240            );
241        }
242        return fallback(
243            ReviewerFallbackCode::AllDiffFamilyExcluded,
244            "all different-family reviewer candidates were excluded".to_string(),
245        );
246    };
247
248    let estimate = cost_estimate(
249        best.identity.pricing.as_ref(),
250        author_identity.pricing.as_ref(),
251    );
252    ComplementaryReviewerSelection {
253        intent: options.intent.as_str().to_string(),
254        reason: reviewer_reason(&author_identity, &best.identity, estimate.as_ref()),
255        estimated_incremental_cost: estimate,
256        author: author_identity,
257        reviewer: best.identity,
258        fallback: false,
259        fallback_reason: None,
260        fallback_code: None,
261    }
262}
263
264#[derive(Debug, Clone)]
265struct ReviewerCandidate {
266    identity: ComplementaryModelIdentity,
267    score: f64,
268}
269
270fn complementary_identity(
271    id: String,
272    provider: String,
273    family: String,
274    lineage: String,
275    tier: String,
276    pricing: Option<ModelPricing>,
277) -> ComplementaryModelIdentity {
278    ComplementaryModelIdentity {
279        id,
280        provider,
281        family,
282        lineage,
283        tier,
284        pricing,
285    }
286}
287
288fn reviewer_score(
289    options: &ComplementaryReviewerOptions,
290    author: &ComplementaryModelIdentity,
291    candidate: &ComplementaryModelIdentity,
292    model: &ModelDef,
293    preferred_families: &[String],
294) -> f64 {
295    let candidate_refs = reviewer_match_refs(candidate);
296    let mut score = 0.0;
297    if let Some(rank) = preferred_families
298        .iter()
299        .position(|selector| refs_contain_selector(&candidate_refs, selector))
300    {
301        score += 1_000.0 - rank as f64;
302    }
303    if candidate.provider != author.provider {
304        score += 100.0;
305    }
306    score += match tier_distance(&author.tier, &candidate.tier) {
307        0 => 80.0,
308        1 => 45.0,
309        2 => 15.0,
310        _ => 0.0,
311    };
312    for strength in intent_strengths(options.intent) {
313        if model.strengths.iter().any(|tag| tag == strength) {
314            score += 8.0;
315        }
316    }
317    if model.capabilities.iter().any(|tag| tag == "tools") {
318        score += 4.0;
319    }
320    if let (Some(author_total), Some(candidate_total)) = (
321        pricing_total(author.pricing.as_ref()),
322        pricing_total(candidate.pricing.as_ref()),
323    ) {
324        if author_total > 0.0 {
325            let ratio = candidate_total / author_total;
326            if ratio <= 1.0 {
327                score += 20.0;
328            }
329            score -= (ratio - 1.0).abs().min(10.0) * 8.0;
330        }
331    }
332    score
333}
334
335fn intent_strengths(intent: ComplementaryReviewerIntent) -> &'static [&'static str] {
336    match intent {
337        ComplementaryReviewerIntent::Review => &["reasoning", "coding", "tool_use"],
338        ComplementaryReviewerIntent::Critique => &["reasoning", "long_context", "tool_use"],
339        ComplementaryReviewerIntent::PlanReview => {
340            &["reasoning", "coding", "agentic", "long_context", "tool_use"]
341        }
342    }
343}
344
345fn tier_distance(left: &str, right: &str) -> u8 {
346    let left = tier_rank(left);
347    let right = tier_rank(right);
348    left.abs_diff(right)
349}
350
351fn tier_rank(tier: &str) -> u8 {
352    match tier {
353        "small" => 0,
354        "mid" => 1,
355        "frontier" | "reasoning" => 2,
356        _ => 1,
357    }
358}
359
360fn exceeds_price_cap(
361    author_pricing: Option<&ModelPricing>,
362    candidate_pricing: Option<&ModelPricing>,
363    max_price_multiplier: Option<f64>,
364) -> bool {
365    let Some(max_price_multiplier) = max_price_multiplier else {
366        return false;
367    };
368    let Some(author_total) = pricing_total(author_pricing) else {
369        return false;
370    };
371    let Some(candidate_total) = pricing_total(candidate_pricing) else {
372        return true;
373    };
374    author_total > 0.0 && candidate_total > author_total * max_price_multiplier
375}
376
377fn cost_estimate(
378    reviewer_pricing: Option<&ModelPricing>,
379    author_pricing: Option<&ModelPricing>,
380) -> Option<ComplementaryCostEstimate> {
381    let reviewer_pricing = reviewer_pricing?;
382    let total_per_mtok = reviewer_pricing.input_per_mtok + reviewer_pricing.output_per_mtok;
383    let multiplier_vs_author = pricing_total(author_pricing)
384        .filter(|author_total| *author_total > 0.0)
385        .map(|author_total| total_per_mtok / author_total);
386    Some(ComplementaryCostEstimate {
387        input_per_mtok: reviewer_pricing.input_per_mtok,
388        output_per_mtok: reviewer_pricing.output_per_mtok,
389        total_per_mtok,
390        multiplier_vs_author,
391    })
392}
393
394fn pricing_total(pricing: Option<&ModelPricing>) -> Option<f64> {
395    pricing.map(|pricing| pricing.input_per_mtok + pricing.output_per_mtok)
396}
397
398fn reviewer_reason(
399    author: &ComplementaryModelIdentity,
400    reviewer: &ComplementaryModelIdentity,
401    estimate: Option<&ComplementaryCostEstimate>,
402) -> String {
403    let cost = estimate
404        .and_then(|estimate| estimate.multiplier_vs_author)
405        .map(|multiplier| format!("{multiplier:.2}x the author model price"))
406        .unwrap_or_else(|| "price ratio unavailable".to_string());
407    format!(
408        "selected {} via {} because family {} differs from author family {}, tier {} matches author tier {}, and {}",
409        reviewer.id,
410        reviewer.provider,
411        reviewer.family,
412        author.family,
413        reviewer.tier,
414        author.tier,
415        cost
416    )
417}
418
419fn reviewer_match_refs(identity: &ComplementaryModelIdentity) -> BTreeSet<String> {
420    BTreeSet::from([
421        identity.id.to_ascii_lowercase(),
422        identity.provider.to_ascii_lowercase(),
423        format!("{}/{}", identity.provider, identity.id).to_ascii_lowercase(),
424        format!("{}:{}", identity.provider, identity.id).to_ascii_lowercase(),
425        identity.family.to_ascii_lowercase(),
426        identity.lineage.to_ascii_lowercase(),
427    ])
428}
429
430fn refs_contain_selector(refs: &BTreeSet<String>, selector: &str) -> bool {
431    normalized_catalog_token(Some(selector))
432        .or_else(|| Some(selector.trim().to_ascii_lowercase()))
433        .is_some_and(|selector| refs.contains(&selector))
434}
harn_vm/llm_config/reviewer.rs

harn_vm/llm_config/
reviewer.rs
