khive_pack_code/
ingest.rs

1//! Pure `findings.json -> Vec<Entity>, Vec<Note>, Vec<Edge>` mapper.
2//!
3//! `ingest_findings_json` validates the entire input before constructing any
4//! output record, so a malformed file never yields a partial batch. Record
5//! identity is a content-derived UUIDv5 hash of the validated finding record
6//! (`observed_at` excluded), so re-ingesting the same `findings.json` under
7//! the same [`CodeIngestOptions`] reproduces the same entity, note, and edge
8//! IDs, while a content change (severity, evidence, impact, ...) produces a
9//! new finding id rather than colliding with the prior record.
10//!
11//! Only `severity` and `confidence` values, the evidence shape, and
12//! `failure_scenario` presence are governed at ingest time — they reject
13//! unknown/malformed values by design (ADR-085 D4 "fail closed; no silent
14//! coercion"). Every other field (`categories`, `standard`, `refs`,
15//! `priority`, raw audit `status`, `impact`, `recommendation`,
16//! `verification`) is tolerated per ADR-085 Amendment 1 A1: ingest neither
17//! rejects nor coerces them, it preserves whatever JSON value was provided
18//! and omits the key when the field is absent. Raw `status` is preserved
19//! verbatim under `properties.audit_status` — it has no agreed mapping to
20//! the finding lifecycle (`kind_status`) yet.
21
22use std::collections::BTreeMap;
23
24use chrono::{DateTime, Utc};
25use khive_storage::{Edge, Entity, LinkId, Note};
26use khive_types::EdgeRelation;
27use serde_json::{json, Map, Value};
28use uuid::Uuid;
29
30use crate::error::CodeIngestError;
31use crate::vocab::{is_valid_confidence, is_valid_severity};
32
33/// UUIDv5 namespace for all code-pack ingest identity, derived from
34/// `Uuid::new_v5(Uuid::NAMESPACE_URL, b"https://github.com/ohdearquant/khive/adr/085/code-pack/v1")`.
35pub const CODE_INGEST_NAMESPACE: Uuid = Uuid::from_u128(0x288fe3dc_ac69_5aef_ab1e_9b170fb07376);
36
37/// Options controlling one `ingest_findings_json` call.
38#[derive(Clone, Debug)]
39pub struct CodeIngestOptions<'a> {
40    /// KG namespace the produced records belong to.
41    pub namespace: &'a str,
42    /// Wall-clock timestamp stamped on produced records. Excluded from every
43    /// identity tuple so re-ingesting the same sweep at a later time still
44    /// reproduces the same IDs.
45    pub observed_at: DateTime<Utc>,
46    /// Stable sweep identity. When absent, derived as `audit.date:audit.commit`.
47    pub source_run: Option<&'a str>,
48}
49
50/// Output of a successful ingest: deterministic entity/note/edge records
51/// ready for the caller to persist through existing storage/runtime paths.
52#[derive(Clone, Debug)]
53pub struct CodeIngestBatch {
54    pub entities: Vec<Entity>,
55    pub notes: Vec<Note>,
56    pub edges: Vec<Edge>,
57}
58
59fn uuid5_tuple<T: serde::Serialize>(parts: &T) -> Result<Uuid, CodeIngestError> {
60    let bytes = serde_json::to_vec(parts)?;
61    Ok(Uuid::new_v5(&CODE_INGEST_NAMESPACE, &bytes))
62}
63
64/// Recursively sort object keys so two JSON values that differ only in key
65/// order serialize identically. Array element order is content and is left
66/// untouched.
67fn sort_json_keys(value: &Value) -> Value {
68    match value {
69        Value::Object(map) => {
70            let sorted: BTreeMap<&str, Value> = map
71                .iter()
72                .map(|(k, v)| (k.as_str(), sort_json_keys(v)))
73                .collect();
74            let mut out = Map::with_capacity(sorted.len());
75            for (k, v) in sorted {
76                out.insert(k.to_string(), v);
77            }
78            Value::Object(out)
79        }
80        Value::Array(arr) => Value::Array(arr.iter().map(sort_json_keys).collect()),
81        other => other.clone(),
82    }
83}
84
85/// Render a tolerated (ungoverned) field for inclusion in note content text.
86/// Strings pass through verbatim; any other JSON shape renders as its
87/// canonical JSON text; absence renders as empty.
88fn value_to_display(value: Option<&Value>) -> String {
89    match value {
90        None | Some(Value::Null) => String::new(),
91        Some(Value::String(s)) => s.clone(),
92        Some(other) => other.to_string(),
93    }
94}
95
96/// A tolerated field is present when the key exists and its value is not
97/// JSON `null` — an explicit `null` is treated the same as absence.
98fn tolerated_field(obj: &Map<String, Value>, key: &str) -> Option<Value> {
99    match obj.get(key) {
100        None | Some(Value::Null) => None,
101        Some(v) => Some(v.clone()),
102    }
103}
104
105/// Amendment 1 A2 governed mapping: `fixed -> resolved`, `false_positive ->
106/// invalid`, everything else (including absent/non-string status) stays
107/// `open`. The raw producer value is preserved verbatim under
108/// `properties.audit_status` regardless of how it maps here.
109fn map_producer_status_to_kind_status(audit_status: Option<&Value>) -> &'static str {
110    match audit_status.and_then(Value::as_str) {
111        Some("fixed") => "resolved",
112        Some("false_positive") => "invalid",
113        _ => "open",
114    }
115}
116
117fn require_str<'a>(
118    obj: &'a Map<String, Value>,
119    key: &'static str,
120    path: &'static str,
121) -> Result<&'a str, CodeIngestError> {
122    match obj.get(key) {
123        None => Err(CodeIngestError::MissingField { path }),
124        Some(Value::String(s)) => Ok(s.as_str()),
125        Some(_) => Err(CodeIngestError::InvalidType {
126            path: path.to_string(),
127            expected: "string",
128        }),
129    }
130}
131
132fn optional_str(
133    obj: &Map<String, Value>,
134    key: &'static str,
135    path: &'static str,
136) -> Result<Option<String>, CodeIngestError> {
137    match obj.get(key) {
138        None | Some(Value::Null) => Ok(None),
139        Some(Value::String(s)) => Ok(Some(s.clone())),
140        Some(_) => Err(CodeIngestError::InvalidType {
141            path: path.to_string(),
142            expected: "string",
143        }),
144    }
145}
146
147fn normalize_title(title: &str) -> Result<String, CodeIngestError> {
148    let normalized = title
149        .split_whitespace()
150        .collect::<Vec<_>>()
151        .join(" ")
152        .to_ascii_lowercase();
153    if normalized.is_empty() {
154        return Err(CodeIngestError::InvalidType {
155            path: "findings[].title".to_string(),
156            expected: "non-empty string",
157        });
158    }
159    Ok(normalized)
160}
161
162/// Canonicalize the tolerated `evidence` shapes (null, string, object, array
163/// of strings/objects) into an array of `{description}`-or-richer objects.
164/// Any other shape is rejected with the finding/evidence index named.
165fn canonicalize_evidence(
166    value: Option<&Value>,
167    finding_index: usize,
168) -> Result<Vec<Value>, CodeIngestError> {
169    let items: Vec<Value> = match value {
170        None | Some(Value::Null) => Vec::new(),
171        Some(Value::String(s)) => vec![Value::String(s.clone())],
172        Some(Value::Object(m)) => vec![Value::Object(m.clone())],
173        Some(Value::Array(arr)) => arr.clone(),
174        Some(_) => {
175            return Err(CodeIngestError::InvalidEvidence {
176                finding_index,
177                evidence_index: 0,
178            });
179        }
180    };
181
182    let mut out = Vec::with_capacity(items.len());
183    for (evidence_index, item) in items.into_iter().enumerate() {
184        let canonical = match item {
185            Value::String(s) => {
186                if s.trim().is_empty() {
187                    return Err(CodeIngestError::InvalidEvidence {
188                        finding_index,
189                        evidence_index,
190                    });
191                }
192                json!({ "description": s })
193            }
194            Value::Object(m) => Value::Object(m),
195            _ => {
196                return Err(CodeIngestError::InvalidEvidence {
197                    finding_index,
198                    evidence_index,
199                });
200            }
201        };
202        out.push(canonical);
203    }
204    Ok(out)
205}
206
207struct ValidatedAudit {
208    date: String,
209    scope: String,
210    repo: String,
211    branch: String,
212    commit: String,
213    standards_file: String,
214    extra: Map<String, Value>,
215}
216
217const KNOWN_AUDIT_KEYS: &[&str] = &[
218    "date",
219    "scope",
220    "repo",
221    "branch",
222    "commit",
223    "standards_file",
224];
225
226impl ValidatedAudit {
227    fn parse(obj: &Map<String, Value>) -> Result<Self, CodeIngestError> {
228        let mut extra = Map::new();
229        for (k, v) in obj {
230            if !KNOWN_AUDIT_KEYS.contains(&k.as_str()) {
231                extra.insert(k.clone(), v.clone());
232            }
233        }
234        Ok(Self {
235            date: require_str(obj, "date", "audit.date")?.to_string(),
236            scope: require_str(obj, "scope", "audit.scope")?.to_string(),
237            repo: require_str(obj, "repo", "audit.repo")?.to_string(),
238            branch: require_str(obj, "branch", "audit.branch")?.to_string(),
239            commit: require_str(obj, "commit", "audit.commit")?.to_string(),
240            standards_file: require_str(obj, "standards_file", "audit.standards_file")?.to_string(),
241            extra,
242        })
243    }
244}
245
246struct ValidatedFinding {
247    id: String,
248    title: String,
249    normalized_title: String,
250    severity: String,
251    confidence: String,
252    categories: Option<Value>,
253    standard: Option<Value>,
254    evidence: Vec<Value>,
255    impact: Option<Value>,
256    recommendation: Option<Value>,
257    verification: Option<Value>,
258    failure_scenario: Option<String>,
259    priority: Option<Value>,
260    audit_status: Option<Value>,
261    refs: Option<Value>,
262    raw: Map<String, Value>,
263}
264
265const KNOWN_FINDING_KEYS: &[&str] = &[
266    "id",
267    "title",
268    "severity",
269    "confidence",
270    "categories",
271    "status",
272    "standard",
273    "evidence",
274    "impact",
275    "recommendation",
276    "verification",
277    "failure_scenario",
278    "priority",
279    "refs",
280];
281
282impl ValidatedFinding {
283    fn parse(obj: &Map<String, Value>, finding_index: usize) -> Result<Self, CodeIngestError> {
284        let id = require_str(obj, "id", "findings[].id")?.to_string();
285        let title = require_str(obj, "title", "findings[].title")?.to_string();
286        if title.trim().is_empty() {
287            return Err(CodeIngestError::InvalidType {
288                path: "findings[].title".to_string(),
289                expected: "non-empty string",
290            });
291        }
292        let normalized_title = normalize_title(&title)?;
293
294        let severity = require_str(obj, "severity", "findings[].severity")?.to_string();
295        if !is_valid_severity(&severity) {
296            return Err(CodeIngestError::InvalidValue {
297                field: "severity",
298                value: severity,
299                valid: "critical | high | medium | low | info",
300            });
301        }
302
303        let confidence = require_str(obj, "confidence", "findings[].confidence")?.to_string();
304        if !is_valid_confidence(&confidence) {
305            return Err(CodeIngestError::InvalidValue {
306                field: "confidence",
307                value: confidence,
308                valid: "high | medium | low",
309            });
310        }
311
312        // `categories`, `standard`, `refs`, `priority`, `status`, `impact`,
313        // `recommendation`, and `verification` are tolerated (ADR-085
314        // Amendment 1 A1): ingest neither rejects nor coerces their shape,
315        // it preserves whatever JSON value was provided or omits the key
316        // when absent. Only `evidence` shape and `severity`/`confidence`/
317        // `failure_scenario` presence remain fail-closed.
318        let categories = tolerated_field(obj, "categories");
319        let standard = tolerated_field(obj, "standard");
320        let evidence = canonicalize_evidence(obj.get("evidence"), finding_index)?;
321        let impact = tolerated_field(obj, "impact");
322        let recommendation = tolerated_field(obj, "recommendation");
323        let verification = tolerated_field(obj, "verification");
324        let failure_scenario =
325            optional_str(obj, "failure_scenario", "findings[].failure_scenario")?;
326        let audit_status = tolerated_field(obj, "status");
327        let priority = tolerated_field(obj, "priority");
328        let refs = tolerated_field(obj, "refs");
329
330        if matches!(severity.as_str(), "medium" | "high" | "critical") {
331            let has_scenario = failure_scenario
332                .as_deref()
333                .is_some_and(|s| !s.trim().is_empty());
334            if !has_scenario {
335                return Err(CodeIngestError::MissingFailureScenario {
336                    id: id.clone(),
337                    severity: severity.clone(),
338                });
339            }
340        }
341
342        let mut raw = Map::new();
343        for (k, v) in obj {
344            if !KNOWN_FINDING_KEYS.contains(&k.as_str()) {
345                raw.insert(k.clone(), v.clone());
346            }
347        }
348
349        Ok(Self {
350            id,
351            title,
352            normalized_title,
353            severity,
354            confidence,
355            categories,
356            standard,
357            evidence,
358            impact,
359            recommendation,
360            verification,
361            failure_scenario,
362            priority,
363            audit_status,
364            refs,
365            raw,
366        })
367    }
368}
369
370/// Map a `findings.json` document into deterministic KG records.
371///
372/// Validates the entire document before constructing any output record —
373/// malformed input never produces a partial batch. Output identity is
374/// content-derived (see module docs), so calling this twice with the same
375/// bytes and options reproduces identical entity/note/edge IDs.
376pub fn ingest_findings_json(
377    input: &[u8],
378    options: CodeIngestOptions<'_>,
379) -> Result<CodeIngestBatch, CodeIngestError> {
380    let root: Value = serde_json::from_slice(input)?;
381    let root_obj = root.as_object().ok_or(CodeIngestError::InvalidRoot)?;
382
383    let audit_obj = root_obj
384        .get("audit")
385        .and_then(Value::as_object)
386        .ok_or(CodeIngestError::InvalidRoot)?;
387    let findings_arr = root_obj
388        .get("findings")
389        .and_then(Value::as_array)
390        .ok_or(CodeIngestError::InvalidRoot)?;
391
392    let audit = ValidatedAudit::parse(audit_obj)?;
393
394    let source_run = match options.source_run.filter(|s| !s.trim().is_empty()) {
395        Some(explicit) => explicit.to_string(),
396        None => {
397            if audit.date.trim().is_empty() || audit.commit.trim().is_empty() {
398                return Err(CodeIngestError::MissingSourceRun);
399            }
400            format!("{}:{}", audit.date, audit.commit)
401        }
402    };
403
404    let mut validated_findings = Vec::with_capacity(findings_arr.len());
405    for (finding_index, raw) in findings_arr.iter().enumerate() {
406        let obj = raw
407            .as_object()
408            .ok_or_else(|| CodeIngestError::InvalidType {
409                path: format!("findings[{finding_index}]"),
410                expected: "object",
411            })?;
412        validated_findings.push(ValidatedFinding::parse(obj, finding_index)?);
413    }
414
415    // All validation above must succeed before any record is constructed.
416
417    let project_id = uuid5_tuple(&(
418        "code-project",
419        1u8,
420        options.namespace,
421        audit.repo.as_str(),
422        audit.scope.as_str(),
423    ))?;
424    let project_external_id = serde_json::to_string(&(
425        "code-project",
426        1u8,
427        options.namespace,
428        audit.repo.as_str(),
429        audit.scope.as_str(),
430    ))?;
431
432    let mut project_properties = Map::new();
433    project_properties.insert("repo".into(), json!(audit.repo));
434    project_properties.insert("branch".into(), json!(audit.branch));
435    project_properties.insert("commit".into(), json!(audit.commit));
436    project_properties.insert("date".into(), json!(audit.date));
437    project_properties.insert("standards_file".into(), json!(audit.standards_file));
438    project_properties.insert("source_run".into(), json!(source_run));
439    project_properties.insert("external_id".into(), json!(project_external_id));
440    if !audit.extra.is_empty() {
441        project_properties.insert("audit_extra".into(), Value::Object(audit.extra));
442    }
443
444    let observed_micros = options.observed_at.timestamp_micros();
445
446    let mut project_entity = Entity::new(options.namespace, "project", audit.scope.as_str());
447    project_entity.id = project_id;
448    project_entity.properties = Some(Value::Object(project_properties));
449    project_entity.created_at = observed_micros;
450    project_entity.updated_at = observed_micros;
451
452    let mut notes = Vec::with_capacity(validated_findings.len());
453    let mut edges = Vec::with_capacity(validated_findings.len());
454
455    for finding in &validated_findings {
456        // Identity is a canonical content hash of the validated finding,
457        // `observed_at` excluded: same content re-ingested at a different
458        // time reproduces the same id, and any content change (severity,
459        // evidence, impact, ...) produces a different id rather than
460        // silently overwriting the prior record under the old one.
461        let identity_value = sort_json_keys(&json!({
462            "kind": "code-finding",
463            "schema_version": 1,
464            "namespace": options.namespace,
465            "source_run": source_run,
466            "scope": audit.scope,
467            "id": finding.id,
468            "normalized_title": finding.normalized_title,
469            "severity": finding.severity,
470            "confidence": finding.confidence,
471            "categories": finding.categories,
472            "standard": finding.standard,
473            "evidence": finding.evidence,
474            "impact": finding.impact,
475            "recommendation": finding.recommendation,
476            "verification": finding.verification,
477            "failure_scenario": finding.failure_scenario,
478            "priority": finding.priority,
479            "audit_status": finding.audit_status,
480            "refs": finding.refs,
481            "raw": finding.raw,
482        }));
483        let finding_id = uuid5_tuple(&identity_value)?;
484        let finding_external_id = serde_json::to_string(&identity_value)?;
485
486        let mut props = Map::new();
487        props.insert("external_id".into(), json!(finding_external_id));
488        props.insert("finding_id".into(), json!(finding.id));
489        props.insert("severity".into(), json!(finding.severity));
490        props.insert("confidence".into(), json!(finding.confidence));
491        props.insert("source_run".into(), json!(source_run));
492        props.insert("evidence".into(), Value::Array(finding.evidence.clone()));
493        if let Some(categories) = &finding.categories {
494            props.insert("categories".into(), categories.clone());
495        }
496        if let Some(standard) = &finding.standard {
497            props.insert("standard".into(), standard.clone());
498        }
499        if let Some(refs) = &finding.refs {
500            props.insert("refs".into(), refs.clone());
501        }
502        if let Some(priority) = &finding.priority {
503            props.insert("priority".into(), priority.clone());
504        }
505        if let Some(status) = &finding.audit_status {
506            props.insert("audit_status".into(), status.clone());
507        }
508        if let Some(scenario) = &finding.failure_scenario {
509            props.insert("failure_scenario".into(), json!(scenario));
510        }
511        if let Some(impact) = &finding.impact {
512            props.insert("impact".into(), impact.clone());
513        }
514        if let Some(recommendation) = &finding.recommendation {
515            props.insert("recommendation".into(), recommendation.clone());
516        }
517        if let Some(verification) = &finding.verification {
518            props.insert("verification".into(), verification.clone());
519        }
520        props.insert(
521            "kind_status".into(),
522            json!(map_producer_status_to_kind_status(
523                finding.audit_status.as_ref()
524            )),
525        );
526        if !finding.raw.is_empty() {
527            props.insert("raw".into(), Value::Object(finding.raw.clone()));
528        }
529
530        let content = format!(
531            "{}: {}",
532            finding.title,
533            value_to_display(finding.impact.as_ref())
534        );
535        let mut note = Note::new(options.namespace, "finding", content);
536        note.id = finding_id;
537        note.name = Some(finding.title.clone());
538        note.properties = Some(Value::Object(props));
539        note.created_at = observed_micros;
540        note.updated_at = observed_micros;
541        notes.push(note);
542
543        let edge_id = uuid5_tuple(&(
544            "code-edge",
545            1u8,
546            options.namespace,
547            finding_id,
548            project_id,
549            EdgeRelation::Annotates.as_str(),
550        ))?;
551        edges.push(Edge {
552            id: LinkId::from(edge_id),
553            namespace: options.namespace.to_string(),
554            source_id: finding_id,
555            target_id: project_id,
556            relation: EdgeRelation::Annotates,
557            weight: 1.0,
558            created_at: options.observed_at,
559            updated_at: options.observed_at,
560            deleted_at: None,
561            metadata: None,
562            target_backend: None,
563        });
564    }
565
566    Ok(CodeIngestBatch {
567        entities: vec![project_entity],
568        notes,
569        edges,
570    })
571}
khive_pack_code/ingest.rs

khive_pack_code/
ingest.rs
