khive_pack_code/
ingest.rs

1//! Pure, fail-closed `findings.json` to deterministic KG-record mapping.
2
3use std::collections::BTreeMap;
4
5use chrono::{DateTime, Utc};
6use khive_storage::{Edge, Entity, LinkId, Note};
7use khive_types::EdgeRelation;
8use serde_json::{json, Map, Value};
9use uuid::Uuid;
10
11use crate::error::CodeIngestError;
12use crate::vocab::{is_valid_confidence, is_valid_severity};
13
14/// UUIDv5 namespace for all code-pack ingest identity, derived from
15/// `Uuid::new_v5(Uuid::NAMESPACE_URL, b"https://github.com/ohdearquant/khive/adr/085/code-pack/v1")`.
16pub const CODE_INGEST_NAMESPACE: Uuid = Uuid::from_u128(0x288fe3dc_ac69_5aef_ab1e_9b170fb07376);
17
18/// Namespace, observation time, and optional stable run identity for one ingest.
19///
20/// See `crates/khive-pack-code/docs/api/findings-ingest.md`.
21#[derive(Clone, Debug)]
22pub struct CodeIngestOptions<'a> {
23    /// KG namespace the produced records belong to.
24    pub namespace: &'a str,
25    /// Record timestamp, excluded from identity so later retries reproduce IDs.
26    pub observed_at: DateTime<Utc>,
27    /// Stable sweep identity. When absent, derived as `audit.date:audit.commit`.
28    pub source_run: Option<&'a str>,
29}
30
31/// Deterministic entity, finding-note, and annotation-edge records ready for persistence.
32///
33/// See `crates/khive-pack-code/docs/api/findings-ingest.md`.
34#[derive(Clone, Debug)]
35pub struct CodeIngestBatch {
36    pub entities: Vec<Entity>,
37    pub notes: Vec<Note>,
38    pub edges: Vec<Edge>,
39}
40
41fn uuid5_tuple<T: serde::Serialize>(parts: &T) -> Result<Uuid, CodeIngestError> {
42    let bytes = serde_json::to_vec(parts)?;
43    Ok(Uuid::new_v5(&CODE_INGEST_NAMESPACE, &bytes))
44}
45
46/// Recursively sort object keys while preserving content-significant array order.
47fn sort_json_keys(value: &Value) -> Value {
48    match value {
49        Value::Object(map) => {
50            let sorted: BTreeMap<&str, Value> = map
51                .iter()
52                .map(|(k, v)| (k.as_str(), sort_json_keys(v)))
53                .collect();
54            let mut out = Map::with_capacity(sorted.len());
55            for (k, v) in sorted {
56                out.insert(k.to_string(), v);
57            }
58            Value::Object(out)
59        }
60        Value::Array(arr) => Value::Array(arr.iter().map(sort_json_keys).collect()),
61        other => other.clone(),
62    }
63}
64
65/// Render strings verbatim, other JSON canonically, and absent values as empty text.
66fn value_to_display(value: Option<&Value>) -> String {
67    match value {
68        None | Some(Value::Null) => String::new(),
69        Some(Value::String(s)) => s.clone(),
70        Some(other) => other.to_string(),
71    }
72}
73
74/// Clone a tolerated field, treating explicit JSON null as absence.
75fn tolerated_field(obj: &Map<String, Value>, key: &str) -> Option<Value> {
76    match obj.get(key) {
77        None | Some(Value::Null) => None,
78        Some(v) => Some(v.clone()),
79    }
80}
81
82/// Map fixed/false-positive producer states to lifecycle states; default to open.
83fn map_producer_status_to_kind_status(audit_status: Option<&Value>) -> &'static str {
84    match audit_status.and_then(Value::as_str) {
85        Some("fixed") => "resolved",
86        Some("false_positive") => "invalid",
87        _ => "open",
88    }
89}
90
91fn require_str<'a>(
92    obj: &'a Map<String, Value>,
93    key: &'static str,
94    path: &'static str,
95) -> Result<&'a str, CodeIngestError> {
96    match obj.get(key) {
97        None => Err(CodeIngestError::MissingField { path }),
98        Some(Value::String(s)) => Ok(s.as_str()),
99        Some(_) => Err(CodeIngestError::InvalidType {
100            path: path.to_string(),
101            expected: "string",
102        }),
103    }
104}
105
106fn optional_str(
107    obj: &Map<String, Value>,
108    key: &'static str,
109    path: &'static str,
110) -> Result<Option<String>, CodeIngestError> {
111    match obj.get(key) {
112        None | Some(Value::Null) => Ok(None),
113        Some(Value::String(s)) => Ok(Some(s.clone())),
114        Some(_) => Err(CodeIngestError::InvalidType {
115            path: path.to_string(),
116            expected: "string",
117        }),
118    }
119}
120
121fn normalize_title(title: &str) -> Result<String, CodeIngestError> {
122    let normalized = title
123        .split_whitespace()
124        .collect::<Vec<_>>()
125        .join(" ")
126        .to_ascii_lowercase();
127    if normalized.is_empty() {
128        return Err(CodeIngestError::InvalidType {
129            path: "findings[].title".to_string(),
130            expected: "non-empty string",
131        });
132    }
133    Ok(normalized)
134}
135
136/// Canonicalize supported evidence shapes or report the finding/evidence index.
137fn canonicalize_evidence(
138    value: Option<&Value>,
139    finding_index: usize,
140) -> Result<Vec<Value>, CodeIngestError> {
141    let items: Vec<Value> = match value {
142        None | Some(Value::Null) => Vec::new(),
143        Some(Value::String(s)) => vec![Value::String(s.clone())],
144        Some(Value::Object(m)) => vec![Value::Object(m.clone())],
145        Some(Value::Array(arr)) => arr.clone(),
146        Some(_) => {
147            return Err(CodeIngestError::InvalidEvidence {
148                finding_index,
149                evidence_index: 0,
150            });
151        }
152    };
153
154    let mut out = Vec::with_capacity(items.len());
155    for (evidence_index, item) in items.into_iter().enumerate() {
156        let canonical = match item {
157            Value::String(s) => {
158                if s.trim().is_empty() {
159                    return Err(CodeIngestError::InvalidEvidence {
160                        finding_index,
161                        evidence_index,
162                    });
163                }
164                json!({ "description": s })
165            }
166            Value::Object(m) => Value::Object(m),
167            _ => {
168                return Err(CodeIngestError::InvalidEvidence {
169                    finding_index,
170                    evidence_index,
171                });
172            }
173        };
174        out.push(canonical);
175    }
176    Ok(out)
177}
178
179struct ValidatedAudit {
180    date: String,
181    scope: String,
182    repo: String,
183    branch: String,
184    commit: String,
185    standards_file: String,
186    extra: Map<String, Value>,
187}
188
189const KNOWN_AUDIT_KEYS: &[&str] = &[
190    "date",
191    "scope",
192    "repo",
193    "branch",
194    "commit",
195    "standards_file",
196];
197
198impl ValidatedAudit {
199    fn parse(obj: &Map<String, Value>) -> Result<Self, CodeIngestError> {
200        let mut extra = Map::new();
201        for (k, v) in obj {
202            if !KNOWN_AUDIT_KEYS.contains(&k.as_str()) {
203                extra.insert(k.clone(), v.clone());
204            }
205        }
206        Ok(Self {
207            date: require_str(obj, "date", "audit.date")?.to_string(),
208            scope: require_str(obj, "scope", "audit.scope")?.to_string(),
209            repo: require_str(obj, "repo", "audit.repo")?.to_string(),
210            branch: require_str(obj, "branch", "audit.branch")?.to_string(),
211            commit: require_str(obj, "commit", "audit.commit")?.to_string(),
212            standards_file: require_str(obj, "standards_file", "audit.standards_file")?.to_string(),
213            extra,
214        })
215    }
216}
217
218struct ValidatedFinding {
219    id: String,
220    title: String,
221    normalized_title: String,
222    severity: String,
223    confidence: String,
224    categories: Option<Value>,
225    standard: Option<Value>,
226    evidence: Vec<Value>,
227    impact: Option<Value>,
228    recommendation: Option<Value>,
229    verification: Option<Value>,
230    failure_scenario: Option<String>,
231    priority: Option<Value>,
232    audit_status: Option<Value>,
233    refs: Option<Value>,
234    raw: Map<String, Value>,
235}
236
237const KNOWN_FINDING_KEYS: &[&str] = &[
238    "id",
239    "title",
240    "severity",
241    "confidence",
242    "categories",
243    "status",
244    "standard",
245    "evidence",
246    "impact",
247    "recommendation",
248    "verification",
249    "failure_scenario",
250    "priority",
251    "refs",
252];
253
254impl ValidatedFinding {
255    fn parse(obj: &Map<String, Value>, finding_index: usize) -> Result<Self, CodeIngestError> {
256        let id = require_str(obj, "id", "findings[].id")?.to_string();
257        let title = require_str(obj, "title", "findings[].title")?.to_string();
258        if title.trim().is_empty() {
259            return Err(CodeIngestError::InvalidType {
260                path: "findings[].title".to_string(),
261                expected: "non-empty string",
262            });
263        }
264        let normalized_title = normalize_title(&title)?;
265
266        let severity = require_str(obj, "severity", "findings[].severity")?.to_string();
267        if !is_valid_severity(&severity) {
268            return Err(CodeIngestError::InvalidValue {
269                field: "severity",
270                value: severity,
271                valid: "critical | high | medium | low | info",
272            });
273        }
274
275        let confidence = require_str(obj, "confidence", "findings[].confidence")?.to_string();
276        if !is_valid_confidence(&confidence) {
277            return Err(CodeIngestError::InvalidValue {
278                field: "confidence",
279                value: confidence,
280                valid: "high | medium | low",
281            });
282        }
283
284        // Preserve ungoverned JSON verbatim; only governed fields fail closed.
285        let categories = tolerated_field(obj, "categories");
286        let standard = tolerated_field(obj, "standard");
287        let evidence = canonicalize_evidence(obj.get("evidence"), finding_index)?;
288        let impact = tolerated_field(obj, "impact");
289        let recommendation = tolerated_field(obj, "recommendation");
290        let verification = tolerated_field(obj, "verification");
291        let failure_scenario =
292            optional_str(obj, "failure_scenario", "findings[].failure_scenario")?;
293        let audit_status = tolerated_field(obj, "status");
294        let priority = tolerated_field(obj, "priority");
295        let refs = tolerated_field(obj, "refs");
296
297        if matches!(severity.as_str(), "medium" | "high" | "critical") {
298            let has_scenario = failure_scenario
299                .as_deref()
300                .is_some_and(|s| !s.trim().is_empty());
301            if !has_scenario {
302                return Err(CodeIngestError::MissingFailureScenario {
303                    id: id.clone(),
304                    severity: severity.clone(),
305                });
306            }
307        }
308
309        let mut raw = Map::new();
310        for (k, v) in obj {
311            if !KNOWN_FINDING_KEYS.contains(&k.as_str()) {
312                raw.insert(k.clone(), v.clone());
313            }
314        }
315
316        Ok(Self {
317            id,
318            title,
319            normalized_title,
320            severity,
321            confidence,
322            categories,
323            standard,
324            evidence,
325            impact,
326            recommendation,
327            verification,
328            failure_scenario,
329            priority,
330            audit_status,
331            refs,
332            raw,
333        })
334    }
335}
336
337/// Map a `findings.json` document into deterministic KG records.
338///
339/// Validation completes before construction, so errors yield no partial batch. Equal content and
340/// identity options reproduce IDs. See `crates/khive-pack-code/docs/api/findings-ingest.md`.
341pub fn ingest_findings_json(
342    input: &[u8],
343    options: CodeIngestOptions<'_>,
344) -> Result<CodeIngestBatch, CodeIngestError> {
345    let root: Value = serde_json::from_slice(input)?;
346    let root_obj = root.as_object().ok_or(CodeIngestError::InvalidRoot)?;
347
348    let audit_obj = root_obj
349        .get("audit")
350        .and_then(Value::as_object)
351        .ok_or(CodeIngestError::InvalidRoot)?;
352    let findings_arr = root_obj
353        .get("findings")
354        .and_then(Value::as_array)
355        .ok_or(CodeIngestError::InvalidRoot)?;
356
357    let audit = ValidatedAudit::parse(audit_obj)?;
358
359    let source_run = match options.source_run.filter(|s| !s.trim().is_empty()) {
360        Some(explicit) => explicit.to_string(),
361        None => {
362            if audit.date.trim().is_empty() || audit.commit.trim().is_empty() {
363                return Err(CodeIngestError::MissingSourceRun);
364            }
365            format!("{}:{}", audit.date, audit.commit)
366        }
367    };
368
369    let mut validated_findings = Vec::with_capacity(findings_arr.len());
370    for (finding_index, raw) in findings_arr.iter().enumerate() {
371        let obj = raw
372            .as_object()
373            .ok_or_else(|| CodeIngestError::InvalidType {
374                path: format!("findings[{finding_index}]"),
375                expected: "object",
376            })?;
377        validated_findings.push(ValidatedFinding::parse(obj, finding_index)?);
378    }
379
380    // All validation above must succeed before any record is constructed.
381
382    let project_id = uuid5_tuple(&(
383        "code-project",
384        1u8,
385        options.namespace,
386        audit.repo.as_str(),
387        audit.scope.as_str(),
388    ))?;
389    let project_external_id = serde_json::to_string(&(
390        "code-project",
391        1u8,
392        options.namespace,
393        audit.repo.as_str(),
394        audit.scope.as_str(),
395    ))?;
396
397    let mut project_properties = Map::new();
398    project_properties.insert("repo".into(), json!(audit.repo));
399    project_properties.insert("branch".into(), json!(audit.branch));
400    project_properties.insert("commit".into(), json!(audit.commit));
401    project_properties.insert("date".into(), json!(audit.date));
402    project_properties.insert("standards_file".into(), json!(audit.standards_file));
403    project_properties.insert("source_run".into(), json!(source_run));
404    project_properties.insert("external_id".into(), json!(project_external_id));
405    if !audit.extra.is_empty() {
406        project_properties.insert("audit_extra".into(), Value::Object(audit.extra));
407    }
408
409    let observed_micros = options.observed_at.timestamp_micros();
410
411    let mut project_entity = Entity::new(options.namespace, "project", audit.scope.as_str());
412    project_entity.id = project_id;
413    project_entity.properties = Some(Value::Object(project_properties));
414    project_entity.created_at = observed_micros;
415    project_entity.updated_at = observed_micros;
416
417    let mut notes = Vec::with_capacity(validated_findings.len());
418    let mut edges = Vec::with_capacity(validated_findings.len());
419
420    for finding in &validated_findings {
421        // Excluding observation time makes retries stable while content changes create new IDs.
422        let identity_value = sort_json_keys(&json!({
423            "kind": "code-finding",
424            "schema_version": 1,
425            "namespace": options.namespace,
426            "source_run": source_run,
427            "scope": audit.scope,
428            "id": finding.id,
429            "normalized_title": finding.normalized_title,
430            "severity": finding.severity,
431            "confidence": finding.confidence,
432            "categories": finding.categories,
433            "standard": finding.standard,
434            "evidence": finding.evidence,
435            "impact": finding.impact,
436            "recommendation": finding.recommendation,
437            "verification": finding.verification,
438            "failure_scenario": finding.failure_scenario,
439            "priority": finding.priority,
440            "audit_status": finding.audit_status,
441            "refs": finding.refs,
442            "raw": finding.raw,
443        }));
444        let finding_id = uuid5_tuple(&identity_value)?;
445        let finding_external_id = serde_json::to_string(&identity_value)?;
446
447        let mut props = Map::new();
448        props.insert("external_id".into(), json!(finding_external_id));
449        props.insert("finding_id".into(), json!(finding.id));
450        props.insert("severity".into(), json!(finding.severity));
451        props.insert("confidence".into(), json!(finding.confidence));
452        props.insert("source_run".into(), json!(source_run));
453        props.insert("evidence".into(), Value::Array(finding.evidence.clone()));
454        if let Some(categories) = &finding.categories {
455            props.insert("categories".into(), categories.clone());
456        }
457        if let Some(standard) = &finding.standard {
458            props.insert("standard".into(), standard.clone());
459        }
460        if let Some(refs) = &finding.refs {
461            props.insert("refs".into(), refs.clone());
462        }
463        if let Some(priority) = &finding.priority {
464            props.insert("priority".into(), priority.clone());
465        }
466        if let Some(status) = &finding.audit_status {
467            props.insert("audit_status".into(), status.clone());
468        }
469        if let Some(scenario) = &finding.failure_scenario {
470            props.insert("failure_scenario".into(), json!(scenario));
471        }
472        if let Some(impact) = &finding.impact {
473            props.insert("impact".into(), impact.clone());
474        }
475        if let Some(recommendation) = &finding.recommendation {
476            props.insert("recommendation".into(), recommendation.clone());
477        }
478        if let Some(verification) = &finding.verification {
479            props.insert("verification".into(), verification.clone());
480        }
481        props.insert(
482            "kind_status".into(),
483            json!(map_producer_status_to_kind_status(
484                finding.audit_status.as_ref()
485            )),
486        );
487        if !finding.raw.is_empty() {
488            props.insert("raw".into(), Value::Object(finding.raw.clone()));
489        }
490
491        let content = format!(
492            "{}: {}",
493            finding.title,
494            value_to_display(finding.impact.as_ref())
495        );
496        let mut note = Note::new(options.namespace, "finding", content);
497        note.id = finding_id;
498        note.name = Some(finding.title.clone());
499        note.properties = Some(Value::Object(props));
500        note.created_at = observed_micros;
501        note.updated_at = observed_micros;
502        notes.push(note);
503
504        let edge_id = uuid5_tuple(&(
505            "code-edge",
506            1u8,
507            options.namespace,
508            finding_id,
509            project_id,
510            EdgeRelation::Annotates.as_str(),
511        ))?;
512        edges.push(Edge {
513            id: LinkId::from(edge_id),
514            namespace: options.namespace.to_string(),
515            source_id: finding_id,
516            target_id: project_id,
517            relation: EdgeRelation::Annotates,
518            weight: 1.0,
519            created_at: options.observed_at,
520            updated_at: options.observed_at,
521            deleted_at: None,
522            metadata: None,
523            target_backend: None,
524        });
525    }
526
527    Ok(CodeIngestBatch {
528        entities: vec![project_entity],
529        notes,
530        edges,
531    })
532}
khive_pack_code/ingest.rs

khive_pack_code/
ingest.rs
