tga 2.1.0

Developer productivity analytics — git commit collection, classification, and reporting
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
//! Tier 4: optional LLM fallback.
//!
//! Sends an OpenAI-compatible chat completion request asking the model to
//! emit a JSON object with `category`, `subcategory`, and `confidence`. The
//! LLM is consulted only when tiers 1–3 all failed and the engine has been
//! configured with `use_llm = true`.
//!
//! All failures are **non-fatal**: a network error, parse error, or missing
//! API key results in `None` so the pipeline can fall back to
//! "uncategorized" rather than crashing.

use reqwest::header::{HeaderMap, HeaderValue};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use tracing::{debug, info, warn};

use crate::classify::tiers::bedrock::BedrockClassifier;
use crate::classify::tiers::ClassificationResult;
use crate::core::models::ClassificationMethod;

/// OpenAI-compatible chat completion endpoint.
const DEFAULT_ENDPOINT: &str = "https://api.openai.com/v1/chat/completions";

/// OpenRouter chat completion endpoint (OpenAI-compatible schema).
const OPENROUTER_ENDPOINT: &str = "https://openrouter.ai/api/v1/chat/completions";

/// Project identity sent to OpenRouter as `HTTP-Referer` (for usage analytics).
const OPENROUTER_REFERER: &str = "https://github.com/bobmatnyc/trusty-git-analytics";

/// Project identity sent to OpenRouter as `X-Title`.
const OPENROUTER_TITLE: &str = "trusty-git-analytics";

/// System prompt instructing the model to return strict JSON.
const SYSTEM_PROMPT: &str = "You are a git commit classifier. Respond with ONLY a JSON \
object: {\"category\": \"feature|bugfix|chore|documentation|refactor|test|ci|performance|style|build|revert|merge|breaking|uncategorized\", \
\"subcategory\": \"optional string or null\", \"confidence\": 0.0-1.0, \
\"complexity\": <integer 1-5>}. \
Complexity 1-5: \
1=trivial (config/version bump/typo), 2=simple (single-file bugfix), \
3=moderate (multi-file feature), 4=complex (cross-module/arch change), \
5=highly complex (system design/major refactor). \
No prose, no markdown. \
Example: {\"category\": \"bugfix\", \"subcategory\": \"null-check\", \
\"confidence\": 0.9, \"complexity\": 2}";

/// Tier-4 LLM-fallback classifier.
pub struct LlmClassifier {
    client: Client,
    model: String,
    api_key: Option<String>,
    endpoint: String,
    /// Provider-specific extra headers (e.g. OpenRouter attribution).
    extra_headers: HeaderMap,
    /// Bedrock backend, populated only when provider == `"bedrock"`. When
    /// `Some`, [`Self::classify`] routes through Bedrock instead of HTTP.
    bedrock: Option<BedrockClassifier>,
}

impl LlmClassifier {
    /// Construct a new LLM classifier targeting the OpenAI chat-completions
    /// endpoint.
    ///
    /// `model` is provider-specific (e.g. `"gpt-4o-mini"`). If `api_key` is
    /// `None`, classification calls will return `None` immediately.
    pub fn new(model: &str, api_key: Option<String>) -> Self {
        Self {
            client: Client::new(),
            model: model.to_string(),
            api_key,
            endpoint: DEFAULT_ENDPOINT.to_string(),
            extra_headers: HeaderMap::new(),
            bedrock: None,
        }
    }

    /// Construct an LLM classifier configured for a specific provider.
    ///
    /// `provider` accepts:
    /// - `"openrouter"` — uses the OpenRouter endpoint. API key comes from
    ///   `openrouter_api_key` if `Some`, else the `OPENROUTER_API_KEY`
    ///   environment variable. Adds the `HTTP-Referer` / `X-Title` headers
    ///   that OpenRouter uses for attribution.
    /// - `"openai"` — uses the OpenAI endpoint. API key comes from
    ///   `OPENAI_API_KEY`.
    /// - `"auto"` (default) — tries OpenRouter first, falls back to OpenAI.
    ///
    /// If no API key can be resolved, the classifier is still constructed
    /// but every `classify` call will short-circuit to `None`.
    pub fn from_provider(
        provider: &str,
        model: &str,
        openrouter_api_key: Option<String>,
    ) -> Result<Self, String> {
        let normalized = provider.trim().to_ascii_lowercase();
        match normalized.as_str() {
            "openrouter" => Ok(Self::build_openrouter(model, openrouter_api_key)),
            "openai" => Ok(Self::new(model, std::env::var("OPENAI_API_KEY").ok())),
            "bedrock" => {
                // Bedrock requires async SDK initialization. Direct sync
                // callers get a clear error so they know to use
                // [`Self::from_provider_async`] (or rebuild without the
                // feature, depending on the binary configuration).
                info!(model, "LLM provider: bedrock (requested via sync path)");
                #[cfg(feature = "bedrock")]
                {
                    Err("bedrock provider requires the async constructor; use \
                         LlmClassifier::from_provider_async"
                        .to_string())
                }
                #[cfg(not(feature = "bedrock"))]
                {
                    let _ = model;
                    Err(
                        "bedrock feature not compiled in — rebuild with --features bedrock"
                            .to_string(),
                    )
                }
            }
            "auto" | "" => {
                let or_key =
                    openrouter_api_key.or_else(|| std::env::var("OPENROUTER_API_KEY").ok());
                if or_key.is_some() {
                    info!("LLM provider auto-selected: openrouter");
                    Ok(Self::build_openrouter(model, or_key))
                } else {
                    info!("LLM provider auto-selected: openai");
                    Ok(Self::new(model, std::env::var("OPENAI_API_KEY").ok()))
                }
            }
            other => {
                warn!(
                    provider = %other,
                    "unknown LLM provider; falling back to OpenAI endpoint"
                );
                Ok(Self::new(model, std::env::var("OPENAI_API_KEY").ok()))
            }
        }
    }

    /// Async variant of [`Self::from_provider`] that supports the `"bedrock"`
    /// provider (whose SDK requires async credential resolution).
    ///
    /// All other provider strings delegate to the sync constructor.
    ///
    /// Why: `from_provider` is called from sync engine setup; only the
    /// Bedrock arm truly needs `.await`, so the sync entry point handles
    /// the common case and this exists for async callers needing Bedrock.
    /// What: matches on `provider`, calls `BedrockClassifier::new(...)` for
    /// `"bedrock"`, otherwise falls back to sync.
    /// Test: indirectly via the CLI integration when invoked with
    /// `--provider bedrock`; the missing-feature path is asserted in the
    /// `bedrock` module's stub test.
    pub async fn from_provider_async(
        provider: &str,
        model: &str,
        openrouter_api_key: Option<String>,
    ) -> Result<Self, String> {
        if provider.trim().eq_ignore_ascii_case("bedrock") {
            info!(model, "LLM provider: bedrock (async init)");
            let bedrock = BedrockClassifier::new(model).await?;
            return Ok(Self {
                client: Client::new(),
                model: model.to_string(),
                api_key: None,
                endpoint: String::new(),
                extra_headers: HeaderMap::new(),
                bedrock: Some(bedrock),
            });
        }
        Self::from_provider(provider, model, openrouter_api_key)
    }

    /// Internal helper: build an OpenRouter-configured classifier with
    /// attribution headers set.
    fn build_openrouter(model: &str, api_key: Option<String>) -> Self {
        let key = api_key.or_else(|| std::env::var("OPENROUTER_API_KEY").ok());
        let mut headers = HeaderMap::new();
        // These are static, valid ASCII strings — `from_static` cannot panic
        // on them at runtime.
        headers.insert("HTTP-Referer", HeaderValue::from_static(OPENROUTER_REFERER));
        headers.insert("X-Title", HeaderValue::from_static(OPENROUTER_TITLE));
        Self {
            client: Client::new(),
            model: model.to_string(),
            api_key: key,
            endpoint: OPENROUTER_ENDPOINT.to_string(),
            extra_headers: headers,
            bedrock: None,
        }
    }

    /// Override the chat-completions endpoint URL (e.g. for Azure / local proxies).
    pub fn with_endpoint(mut self, endpoint: impl Into<String>) -> Self {
        self.endpoint = endpoint.into();
        self
    }

    /// Whether this classifier has a usable credential.
    ///
    /// `true` when an API key was resolved (for HTTP providers) or when a
    /// Bedrock backend is wired up. `false` indicates `classify` would
    /// short-circuit to `None` for HTTP providers — the pipeline uses this
    /// at startup to emit a single warning instead of silently producing
    /// "LLM fallback did not improve confidence" for every commit.
    pub fn has_api_key(&self) -> bool {
        self.bedrock.is_some() || self.api_key.is_some()
    }

    /// Classify `message` by calling the LLM.
    ///
    /// Returns `None` if the LLM is disabled (no API key), the request
    /// fails, or the response cannot be parsed.
    pub async fn classify(&self, message: &str) -> Option<ClassificationResult> {
        if let Some(bedrock) = &self.bedrock {
            return bedrock
                .classify_batch_bedrock(&[message])
                .await
                .into_iter()
                .next()
                .flatten();
        }
        let api_key = self.api_key.as_deref()?;

        let body = ChatRequest {
            model: &self.model,
            messages: vec![
                ChatMessage {
                    role: "system",
                    content: SYSTEM_PROMPT.to_string(),
                },
                ChatMessage {
                    role: "user",
                    content: format!("Classify this commit message:\n\n{message}"),
                },
            ],
            temperature: 0.0,
            response_format: Some(ResponseFormat {
                kind: "json_object".to_string(),
            }),
        };

        let response = match self
            .client
            .post(&self.endpoint)
            .bearer_auth(api_key)
            .headers(self.extra_headers.clone())
            .json(&body)
            .send()
            .await
        {
            Ok(r) => r,
            Err(e) => {
                warn!(error = %e, "LLM request failed");
                return None;
            }
        };

        if !response.status().is_success() {
            warn!(status = %response.status(), "LLM returned non-success status");
            return None;
        }

        let parsed: ChatResponse = match response.json().await {
            Ok(j) => j,
            Err(e) => {
                warn!(error = %e, "LLM response JSON decode failed");
                return None;
            }
        };

        let content = parsed.choices.first()?.message.content.clone();
        debug!(content = %content, "LLM raw response");

        let verdict: LlmVerdict = serde_json::from_str(&content)
            .map_err(|e| warn!(error = %e, "LLM JSON parse failed"))
            .ok()?;

        Some(ClassificationResult {
            category: verdict.category,
            subcategory: verdict.subcategory,
            top_level: None, // resolved by ClassificationEngine via the taxonomy registry
            confidence: verdict.confidence.clamp(0.0, 1.0),
            method: ClassificationMethod::LlmFallback,
            ticket_id: None,
            // Clamp out-of-range LLM scores into the documented 1–5 band.
            complexity: verdict.complexity.map(|v| v.clamp(1, 5)),
        })
    }
}

// ---- request / response DTOs (private) ----

#[derive(Serialize)]
struct ChatRequest<'a> {
    model: &'a str,
    messages: Vec<ChatMessage>,
    temperature: f64,
    #[serde(skip_serializing_if = "Option::is_none")]
    response_format: Option<ResponseFormat>,
}

#[derive(Serialize)]
struct ChatMessage {
    role: &'static str,
    content: String,
}

#[derive(Serialize)]
struct ResponseFormat {
    #[serde(rename = "type")]
    kind: String,
}

#[derive(Deserialize)]
struct ChatResponse {
    choices: Vec<ChatChoice>,
}

#[derive(Deserialize)]
struct ChatChoice {
    message: ChatChoiceMessage,
}

#[derive(Deserialize)]
struct ChatChoiceMessage {
    content: String,
}

#[derive(Debug, Deserialize)]
struct LlmVerdict {
    category: String,
    #[serde(default)]
    subcategory: Option<String>,
    #[serde(default = "default_confidence")]
    confidence: f64,
    /// Optional 1–5 complexity score. Missing or out-of-range values are
    /// handled by the caller (clamped to 1–5, or left `None`).
    #[serde(default)]
    complexity: Option<u8>,
}

fn default_confidence() -> f64 {
    0.5
}

#[cfg(test)]
mod tests {
    use super::*;
    use wiremock::matchers::{method, path};
    use wiremock::{Mock, MockServer, ResponseTemplate};

    #[test]
    fn has_api_key_reflects_key_state() {
        let with_key = LlmClassifier::new("gpt-4o-mini", Some("sk-test".to_string()));
        assert!(with_key.has_api_key());
        let without_key = LlmClassifier::new("gpt-4o-mini", None);
        assert!(!without_key.has_api_key());
    }

    #[tokio::test]
    async fn classify_returns_none_without_api_key() {
        let llm = LlmClassifier::new("gpt-4o-mini", None);
        assert!(llm.classify("feat: anything").await.is_none());
    }

    /// Regression for the hive-review finding in PR #(issue 99): the raw
    /// `LlmClassifier` always sets `ticket_id: None`. The engine wrapper
    /// (`Engine::llm_classify_only`) backfills it via regex extraction.
    /// This test pins the raw classifier's contract so any future change
    /// that starts surfacing `ticket_id` from the LLM verdict prompts the
    /// engine wrapper to be revisited (so we don't double-backfill).
    #[tokio::test]
    async fn classify_does_not_set_ticket_id() {
        let server = MockServer::start().await;
        let body = serde_json::json!({
            "choices": [{
                "message": {
                    "content": "{\"category\": \"bugfix\", \
                                 \"subcategory\": null, \
                                 \"confidence\": 0.8}"
                }
            }]
        });
        Mock::given(method("POST"))
            .and(path("/v1/chat/completions"))
            .respond_with(ResponseTemplate::new(200).set_body_json(body))
            .mount(&server)
            .await;

        let llm = LlmClassifier::new("gpt-4o-mini", Some("sk-test".to_string()))
            .with_endpoint(format!("{}/v1/chat/completions", server.uri()));
        let r = llm
            .classify("fix: handle null in PROJ-1234 endpoint")
            .await
            .expect("LLM verdict");
        assert_eq!(r.ticket_id, None);
    }

    /// Why: the LLM tier is the only producer of complexity scores; if the
    /// `complexity` key stops deserializing, all scoring silently breaks.
    /// What: a JSON verdict with `"complexity": 3` must deserialize to
    /// `Some(3)`, and a verdict omitting the key must default to `None`.
    /// Test: `serde_json::from_str` two payloads and assert the field.
    #[test]
    fn llm_verdict_deserializes_complexity() {
        let with: LlmVerdict =
            serde_json::from_str(r#"{"category":"feature","confidence":0.9,"complexity":3}"#)
                .expect("deserialize verdict with complexity");
        assert_eq!(with.complexity, Some(3));

        let without: LlmVerdict =
            serde_json::from_str(r#"{"category":"feature","confidence":0.9}"#)
                .expect("deserialize verdict without complexity");
        assert_eq!(without.complexity, None);
    }

    /// Why: the model only emits complexity if the prompt asks for it; this
    /// pins the prompt so a future edit can't drop the request silently.
    /// What: asserts the system prompt mentions the `complexity` key.
    /// Test: substring check on `SYSTEM_PROMPT`.
    #[test]
    fn system_prompt_requests_complexity() {
        assert!(
            SYSTEM_PROMPT.contains("complexity"),
            "system prompt must instruct the model to return a complexity score"
        );
    }

    #[tokio::test]
    async fn classify_dispatches_to_endpoint_when_keyed() {
        // Regression: issue #99 — when the pipeline asks the LLM tier
        // directly, an HTTP call must happen even for messages a regex
        // tier would have caught. This test verifies the raw classifier
        // hits its configured endpoint and returns the LLM verdict.
        let server = MockServer::start().await;
        let body = serde_json::json!({
            "choices": [{
                "message": {
                    "content": "{\"category\": \"feature\", \
                                 \"subcategory\": \"new-auth\", \
                                 \"confidence\": 0.91}"
                }
            }]
        });
        Mock::given(method("POST"))
            .and(path("/v1/chat/completions"))
            .respond_with(ResponseTemplate::new(200).set_body_json(body))
            .mount(&server)
            .await;

        let llm = LlmClassifier::new("gpt-4o-mini", Some("sk-test".to_string()))
            .with_endpoint(format!("{}/v1/chat/completions", server.uri()));
        let r = llm.classify("chore: bump deps").await.expect("LLM verdict");
        assert_eq!(r.category, "feature");
        assert_eq!(r.subcategory.as_deref(), Some("new-auth"));
        assert!((r.confidence - 0.91).abs() < 1e-6);
        assert_eq!(r.method, ClassificationMethod::LlmFallback);
    }
}