1use super::ALGORITHM_RUN_CONTRACT_VERSION;
4use super::ALGORITHM_RUN_EVENT_CONTRACT_VERSION;
5use super::ALGORITHM_RUN_EVENT_SCHEMA;
6use super::ALGORITHM_RUN_SCHEMA;
7use super::KnowledgeError;
8use super::binary_column;
9use super::fixed_32_at;
10use super::fixed_column;
11use super::invalid;
12use super::optional_fixed_32;
13use super::optional_text;
14use super::require_schema;
15use super::require_uuid;
16use super::require_v7;
17use super::required_binary;
18use super::required_i64;
19use super::required_text;
20use super::required_u32;
21use super::string_column;
22use super::timestamp_column;
23use super::u32_column;
24use super::uuid_at;
25use arrow::array::BinaryArray;
26use arrow::array::FixedSizeBinaryBuilder;
27use arrow::array::StringArray;
28use arrow::array::TimestampMicrosecondArray;
29use arrow::array::UInt32Array;
30use arrow::record_batch::RecordBatch;
31use std::collections::HashMap;
32use std::collections::HashSet;
33use std::sync::Arc;
34use uuid::Uuid;
35
36#[cfg(test)]
37std::thread_local! {
38 static RUN_ROW_VALIDATIONS: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
39 static EVENT_ROW_VALIDATIONS: std::cell::Cell<Option<usize>> = const { std::cell::Cell::new(None) };
40}
41
42#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
44#[serde(rename_all = "snake_case")]
45pub enum AlgorithmRunState {
46 Started,
48 Completed,
50 Failed,
52 Cancelled,
54 Interrupted,
56}
57
58impl AlgorithmRunState {
59 #[must_use]
61 pub const fn as_str(self) -> &'static str {
62 match self {
63 Self::Started => "started",
64 Self::Completed => "completed",
65 Self::Failed => "failed",
66 Self::Cancelled => "cancelled",
67 Self::Interrupted => "interrupted",
68 }
69 }
70
71 pub(super) fn parse(value: &str) -> Result<Self, KnowledgeError> {
72 match value {
73 "started" => Ok(Self::Started),
74 "completed" => Ok(Self::Completed),
75 "failed" => Ok(Self::Failed),
76 "cancelled" => Ok(Self::Cancelled),
77 "interrupted" => Ok(Self::Interrupted),
78 _ => Err(invalid("state", "unknown closed value")),
79 }
80 }
81
82 #[must_use]
84 pub const fn is_terminal(self) -> bool {
85 !matches!(self, Self::Started)
86 }
87}
88
89#[derive(Clone, Debug, PartialEq, Eq)]
91pub struct AlgorithmRun {
92 pub run_uuid: Uuid,
94 pub algorithm: String,
96 pub algorithm_version: u32,
98 pub descriptor_version: u32,
100 pub descriptor: Vec<u8>,
102 pub projection_fingerprint: [u8; 32],
104 pub provenance_uuid: Uuid,
106 pub started_at_micros: i64,
108 pub contract_version: u32,
110}
111
112impl AlgorithmRun {
113 #[allow(clippy::too_many_arguments)]
115 pub fn new(
116 run_uuid: Uuid,
117 algorithm: String,
118 algorithm_version: u32,
119 descriptor_version: u32,
120 descriptor: Vec<u8>,
121 projection_fingerprint: [u8; 32],
122 provenance_uuid: Uuid,
123 started_at_micros: i64,
124 ) -> Result<Self, KnowledgeError> {
125 let row = Self {
126 run_uuid,
127 algorithm,
128 algorithm_version,
129 descriptor_version,
130 descriptor,
131 projection_fingerprint,
132 provenance_uuid,
133 started_at_micros,
134 contract_version: ALGORITHM_RUN_CONTRACT_VERSION,
135 };
136 validate_algorithm_run(&row)?;
137 Ok(row)
138 }
139}
140
141#[derive(Clone, Debug, PartialEq, Eq)]
143pub struct AlgorithmRunEvent {
144 pub event_uuid: Uuid,
146 pub run_uuid: Uuid,
148 pub state: AlgorithmRunState,
150 pub result_fingerprint: Option<[u8; 32]>,
152 pub error_code: Option<String>,
154 pub recorded_at_micros: i64,
156 pub provenance_uuid: Uuid,
158 pub contract_version: u32,
160}
161
162impl AlgorithmRunEvent {
163 #[allow(clippy::too_many_arguments)]
165 pub fn new(
166 event_uuid: Uuid,
167 run_uuid: Uuid,
168 state: AlgorithmRunState,
169 result_fingerprint: Option<[u8; 32]>,
170 error_code: Option<String>,
171 recorded_at_micros: i64,
172 provenance_uuid: Uuid,
173 ) -> Result<Self, KnowledgeError> {
174 let row = Self {
175 event_uuid,
176 run_uuid,
177 state,
178 result_fingerprint,
179 error_code,
180 recorded_at_micros,
181 provenance_uuid,
182 contract_version: ALGORITHM_RUN_EVENT_CONTRACT_VERSION,
183 };
184 validate_algorithm_run_event(&row)?;
185 Ok(row)
186 }
187}
188
189#[derive(Clone, Debug, Default, PartialEq, Eq)]
199pub struct AlgorithmRunLedger {
200 runs: Vec<AlgorithmRun>,
202 events: Vec<AlgorithmRunEvent>,
204}
205
206impl AlgorithmRunLedger {
207 pub fn new(
209 runs: Vec<AlgorithmRun>,
210 events: Vec<AlgorithmRunEvent>,
211 ) -> Result<Self, KnowledgeError> {
212 Self::new_with_limits(
213 runs,
214 events,
215 crate::MAX_KNOWLEDGE_ROWS,
216 crate::MAX_KNOWLEDGE_ROWS,
217 )
218 }
219
220 fn new_with_limits(
221 mut runs: Vec<AlgorithmRun>,
222 mut events: Vec<AlgorithmRunEvent>,
223 run_limit: usize,
224 event_limit: usize,
225 ) -> Result<Self, KnowledgeError> {
226 runs.sort_by_key(|row| (row.started_at_micros, row.run_uuid));
227 events.sort_by_key(|row| (row.recorded_at_micros, row.event_uuid));
228 validate_algorithm_run_rows_with_limits(&runs, &events, run_limit, event_limit)?;
229 Ok(Self { runs, events })
230 }
231
232 pub fn merge(&self, staged: &Self) -> Result<Self, KnowledgeError> {
234 self.merge_with_limits(staged, crate::MAX_KNOWLEDGE_ROWS, crate::MAX_KNOWLEDGE_ROWS)
235 }
236
237 pub fn append_events(&self, appended: Vec<AlgorithmRunEvent>) -> Result<Self, KnowledgeError> {
244 self.append_events_with_limits(
245 appended,
246 crate::MAX_KNOWLEDGE_ROWS,
247 crate::MAX_KNOWLEDGE_ROWS,
248 )
249 }
250
251 fn append_events_with_limits(
252 &self,
253 appended: Vec<AlgorithmRunEvent>,
254 run_limit: usize,
255 event_limit: usize,
256 ) -> Result<Self, KnowledgeError> {
257 check_limit_with_limit("algorithm_runs", self.runs.len(), run_limit)?;
258 check_limit_with_limit(
259 "algorithm_run_events",
260 self.events.len().saturating_add(appended.len()),
261 event_limit,
262 )?;
263 for event in &appended {
264 validate_algorithm_run_event(event)?;
265 }
266
267 let mut events = self.events.clone();
268 for event in appended {
269 events.push(event);
270 }
271 events.sort_by_key(|row| (row.recorded_at_micros, row.event_uuid));
272 validate_algorithm_run_relationships(&self.runs, &events, run_limit, event_limit)?;
273 Ok(Self {
274 runs: self.runs.clone(),
275 events,
276 })
277 }
278
279 fn merge_with_limits(
280 &self,
281 staged: &Self,
282 run_limit: usize,
283 event_limit: usize,
284 ) -> Result<Self, KnowledgeError> {
285 let mut runs = self.runs.clone();
286 for row in &staged.runs {
287 match runs.iter().find(|current| current.run_uuid == row.run_uuid) {
288 Some(current) if current == row => {}
289 Some(_) => return Err(KnowledgeError::Conflict("run_uuid")),
290 None => runs.push(row.clone()),
291 }
292 }
293 let mut events = self.events.clone();
294 for row in &staged.events {
295 match events
296 .iter()
297 .find(|current| current.event_uuid == row.event_uuid)
298 {
299 Some(current) if current == row => {}
300 Some(_) => return Err(KnowledgeError::Conflict("event_uuid")),
301 None => events.push(row.clone()),
302 }
303 }
304 runs.sort_by_key(|row| (row.started_at_micros, row.run_uuid));
305 events.sort_by_key(|row| (row.recorded_at_micros, row.event_uuid));
306 validate_algorithm_run_relationships(&runs, &events, run_limit, event_limit)?;
307 Ok(Self { runs, events })
308 }
309
310 #[must_use]
312 pub fn runs(&self) -> &[AlgorithmRun] {
313 &self.runs
314 }
315
316 #[must_use]
318 pub fn events(&self) -> &[AlgorithmRunEvent] {
319 &self.events
320 }
321
322 #[must_use]
324 pub fn into_runs(self) -> Vec<AlgorithmRun> {
325 self.runs
326 }
327
328 #[must_use]
330 pub fn into_events(self) -> Vec<AlgorithmRunEvent> {
331 self.events
332 }
333
334 #[must_use]
336 pub fn into_parts(self) -> (Vec<AlgorithmRun>, Vec<AlgorithmRunEvent>) {
337 (self.runs, self.events)
338 }
339
340 #[must_use]
342 pub fn run(&self, run_uuid: Uuid) -> Option<&AlgorithmRun> {
343 self.runs.iter().find(|row| row.run_uuid == run_uuid)
344 }
345
346 #[must_use]
348 pub fn events_for(&self, run_uuid: Uuid) -> Vec<AlgorithmRunEvent> {
349 self.events
350 .iter()
351 .filter(|row| row.run_uuid == run_uuid)
352 .cloned()
353 .collect()
354 }
355
356 #[must_use]
358 pub fn terminal_event(&self, run_uuid: Uuid) -> Option<&AlgorithmRunEvent> {
359 self.events
360 .iter()
361 .find(|row| row.run_uuid == run_uuid && row.state.is_terminal())
362 }
363
364 pub fn run_batch(&self) -> Result<RecordBatch, KnowledgeError> {
366 algorithm_run_batch(&self.runs)
367 }
368
369 pub fn event_batch(&self) -> Result<RecordBatch, KnowledgeError> {
371 algorithm_run_event_batch(&self.events)
372 }
373
374 pub fn from_batches(
376 run_batches: &[RecordBatch],
377 event_batches: &[RecordBatch],
378 ) -> Result<Self, KnowledgeError> {
379 let mut runs = Vec::new();
380 for batch in run_batches {
381 require_schema(batch, &ALGORITHM_RUN_SCHEMA, "algorithm_runs")?;
382 let ids = fixed_column(batch, "run_uuid")?;
383 let algorithms = string_column(batch, "algorithm")?;
384 let algorithm_versions = u32_column(batch, "algorithm_version")?;
385 let descriptor_versions = u32_column(batch, "descriptor_version")?;
386 let descriptors = binary_column(batch, "descriptor")?;
387 let projections = fixed_column(batch, "projection_fingerprint")?;
388 let provenance = fixed_column(batch, "provenance_uuid")?;
389 let started = timestamp_column(batch, "started_at")?;
390 let contracts = u32_column(batch, "contract_version")?;
391 for row in 0..batch.num_rows() {
392 runs.push(AlgorithmRun {
393 run_uuid: uuid_at(ids, row, "run_uuid")?,
394 algorithm: required_text(algorithms, row, "algorithm")?.to_owned(),
395 algorithm_version: required_u32(algorithm_versions, row, "algorithm_version")?,
396 descriptor_version: required_u32(
397 descriptor_versions,
398 row,
399 "descriptor_version",
400 )?,
401 descriptor: required_binary(descriptors, row, "descriptor")?.to_vec(),
402 projection_fingerprint: fixed_32_at(
403 projections,
404 row,
405 "projection_fingerprint",
406 )?,
407 provenance_uuid: uuid_at(provenance, row, "provenance_uuid")?,
408 started_at_micros: required_i64(started, row, "started_at")?,
409 contract_version: required_u32(contracts, row, "contract_version")?,
410 });
411 }
412 }
413 let mut events = Vec::new();
414 for batch in event_batches {
415 require_schema(batch, &ALGORITHM_RUN_EVENT_SCHEMA, "algorithm_run_events")?;
416 let ids = fixed_column(batch, "event_uuid")?;
417 let runs_column = fixed_column(batch, "run_uuid")?;
418 let states = string_column(batch, "state")?;
419 let results = fixed_column(batch, "result_fingerprint")?;
420 let errors = string_column(batch, "error_code")?;
421 let recorded = timestamp_column(batch, "recorded_at")?;
422 let provenance = fixed_column(batch, "provenance_uuid")?;
423 let contracts = u32_column(batch, "contract_version")?;
424 for row in 0..batch.num_rows() {
425 events.push(AlgorithmRunEvent {
426 event_uuid: uuid_at(ids, row, "event_uuid")?,
427 run_uuid: uuid_at(runs_column, row, "run_uuid")?,
428 state: AlgorithmRunState::parse(required_text(states, row, "state")?)?,
429 result_fingerprint: optional_fixed_32(results, row, "result_fingerprint")?,
430 error_code: optional_text(errors, row),
431 recorded_at_micros: required_i64(recorded, row, "recorded_at")?,
432 provenance_uuid: uuid_at(provenance, row, "provenance_uuid")?,
433 contract_version: required_u32(contracts, row, "contract_version")?,
434 });
435 }
436 }
437 Self::new(runs, events)
438 }
439}
440
441fn validate_algorithm_run(row: &AlgorithmRun) -> Result<(), KnowledgeError> {
442 #[cfg(test)]
443 RUN_ROW_VALIDATIONS.with(|count| {
444 if let Some(value) = count.get() {
445 count.set(Some(value + 1));
446 }
447 });
448 require_v7(row.run_uuid, "run_uuid")?;
449 require_uuid(row.provenance_uuid, "provenance_uuid")?;
450 if row.algorithm.is_empty()
451 || row.algorithm.len() as u64 > graphforge_core::canonical::MAX_CANONICAL_TEXT_BYTES
452 {
453 return Err(invalid("algorithm", "must be bounded non-empty UTF-8"));
454 }
455 if row.algorithm_version != 1 {
456 return Err(invalid("algorithm_version", "unsupported version"));
457 }
458 if row.descriptor_version != 1 {
459 return Err(invalid("descriptor_version", "unsupported version"));
460 }
461 if row.descriptor.is_empty()
462 || row.descriptor.len() as u64 > graphforge_core::canonical::MAX_CANONICAL_BINARY_BYTES
463 {
464 return Err(invalid("descriptor", "must be bounded and non-empty"));
465 }
466 if row.contract_version != ALGORITHM_RUN_CONTRACT_VERSION {
467 return Err(invalid(
468 "contract_version",
469 "unsupported algorithm-run version",
470 ));
471 }
472 Ok(())
473}
474
475fn validate_algorithm_run_event(row: &AlgorithmRunEvent) -> Result<(), KnowledgeError> {
476 #[cfg(test)]
477 EVENT_ROW_VALIDATIONS.with(|count| {
478 if let Some(value) = count.get() {
479 count.set(Some(value + 1));
480 }
481 });
482 require_uuid(row.event_uuid, "event_uuid")?;
483 require_v7(row.run_uuid, "run_uuid")?;
484 require_uuid(row.provenance_uuid, "provenance_uuid")?;
485 if row.contract_version != ALGORITHM_RUN_EVENT_CONTRACT_VERSION {
486 return Err(invalid(
487 "contract_version",
488 "unsupported algorithm-run-event version",
489 ));
490 }
491 if row.error_code.as_ref().is_some_and(|code| {
492 code.is_empty()
493 || code.len() as u64 > graphforge_core::canonical::MAX_CANONICAL_TEXT_BYTES
494 || !code.starts_with("GF_")
495 }) {
496 return Err(invalid("error_code", "must be a bounded stable GF_ code"));
497 }
498 match row.state {
499 AlgorithmRunState::Started => {
500 if row.result_fingerprint.is_some() || row.error_code.is_some() {
501 return Err(invalid("state", "started has no terminal payload"));
502 }
503 }
504 AlgorithmRunState::Completed => {
505 if row.result_fingerprint.is_none() || row.error_code.is_some() {
506 return Err(invalid(
507 "state",
508 "completed requires only a result fingerprint",
509 ));
510 }
511 }
512 AlgorithmRunState::Failed
513 | AlgorithmRunState::Cancelled
514 | AlgorithmRunState::Interrupted => {
515 if row.result_fingerprint.is_some() || row.error_code.is_none() {
516 return Err(invalid(
517 "state",
518 "non-success terminal requires only an error code",
519 ));
520 }
521 }
522 }
523 Ok(())
524}
525
526fn validate_algorithm_run_rows_with_limits(
527 runs: &[AlgorithmRun],
528 events: &[AlgorithmRunEvent],
529 run_limit: usize,
530 event_limit: usize,
531) -> Result<(), KnowledgeError> {
532 check_limit_with_limit("algorithm_runs", runs.len(), run_limit)?;
533 check_limit_with_limit("algorithm_run_events", events.len(), event_limit)?;
534 for row in runs {
535 validate_algorithm_run(row)?;
536 }
537 for row in events {
538 validate_algorithm_run_event(row)?;
539 }
540 validate_algorithm_run_relationships(runs, events, run_limit, event_limit)
541}
542
543fn validate_algorithm_run_relationships(
544 runs: &[AlgorithmRun],
545 events: &[AlgorithmRunEvent],
546 run_limit: usize,
547 event_limit: usize,
548) -> Result<(), KnowledgeError> {
549 check_limit_with_limit("algorithm_runs", runs.len(), run_limit)?;
550 check_limit_with_limit("algorithm_run_events", events.len(), event_limit)?;
551 let mut run_ids = HashSet::with_capacity(runs.len());
552 let mut run_index = HashMap::with_capacity(runs.len());
553 for row in runs {
554 if !run_ids.insert(row.run_uuid) {
555 return Err(KnowledgeError::Duplicate("run_uuid"));
556 }
557 run_index.insert(row.run_uuid, row);
558 }
559 let mut event_ids = HashSet::with_capacity(events.len());
560 let mut per_run: HashMap<Uuid, (usize, usize)> = HashMap::new();
561 for row in events {
562 if !event_ids.insert(row.event_uuid) {
563 return Err(KnowledgeError::Duplicate("event_uuid"));
564 }
565 let run = run_index
566 .get(&row.run_uuid)
567 .ok_or(KnowledgeError::Dangling("run_uuid"))?;
568 if row.recorded_at_micros < run.started_at_micros {
569 return Err(invalid("recorded_at", "event precedes run start"));
570 }
571 let counts = per_run.entry(row.run_uuid).or_default();
572 if row.state == AlgorithmRunState::Started {
573 counts.0 += 1;
574 if row.recorded_at_micros != run.started_at_micros
575 || row.provenance_uuid != run.provenance_uuid
576 {
577 return Err(invalid(
578 "started",
579 "start event must match immutable run identity",
580 ));
581 }
582 } else {
583 counts.1 += 1;
584 }
585 }
586 for run in runs {
587 let (started, terminal) = per_run.get(&run.run_uuid).copied().unwrap_or_default();
588 if started != 1 {
589 return Err(invalid("started", "run requires exactly one start event"));
590 }
591 if terminal > 1 {
592 return Err(invalid(
593 "terminal",
594 "run permits at most one terminal event",
595 ));
596 }
597 }
598 Ok(())
599}
600
601fn check_limit_with_limit(
602 family: &'static str,
603 observed: usize,
604 limit: usize,
605) -> Result<(), KnowledgeError> {
606 if observed > limit {
607 return Err(KnowledgeError::Limit {
608 participant: family,
609 observed,
610 limit,
611 });
612 }
613 Ok(())
614}
615
616fn algorithm_run_batch(rows: &[AlgorithmRun]) -> Result<RecordBatch, KnowledgeError> {
617 let mut ids = FixedSizeBinaryBuilder::with_capacity(rows.len(), 16);
618 let mut projections = FixedSizeBinaryBuilder::with_capacity(rows.len(), 32);
619 let mut provenance = FixedSizeBinaryBuilder::with_capacity(rows.len(), 16);
620 for row in rows {
621 ids.append_value(row.run_uuid.as_bytes())?;
622 projections.append_value(row.projection_fingerprint)?;
623 provenance.append_value(row.provenance_uuid.as_bytes())?;
624 }
625 RecordBatch::try_new(
626 Arc::clone(&ALGORITHM_RUN_SCHEMA),
627 vec![
628 Arc::new(ids.finish()),
629 Arc::new(StringArray::from_iter_values(
630 rows.iter().map(|row| row.algorithm.as_str()),
631 )),
632 Arc::new(UInt32Array::from_iter_values(
633 rows.iter().map(|row| row.algorithm_version),
634 )),
635 Arc::new(UInt32Array::from_iter_values(
636 rows.iter().map(|row| row.descriptor_version),
637 )),
638 Arc::new(BinaryArray::from_iter_values(
639 rows.iter().map(|row| row.descriptor.as_slice()),
640 )),
641 Arc::new(projections.finish()),
642 Arc::new(provenance.finish()),
643 Arc::new(
644 TimestampMicrosecondArray::from_iter_values(
645 rows.iter().map(|row| row.started_at_micros),
646 )
647 .with_timezone("UTC"),
648 ),
649 Arc::new(UInt32Array::from_iter_values(
650 rows.iter().map(|row| row.contract_version),
651 )),
652 ],
653 )
654 .map_err(Into::into)
655}
656
657fn algorithm_run_event_batch(rows: &[AlgorithmRunEvent]) -> Result<RecordBatch, KnowledgeError> {
658 let mut ids = FixedSizeBinaryBuilder::with_capacity(rows.len(), 16);
659 let mut runs = FixedSizeBinaryBuilder::with_capacity(rows.len(), 16);
660 let mut results = FixedSizeBinaryBuilder::with_capacity(rows.len(), 32);
661 let mut provenance = FixedSizeBinaryBuilder::with_capacity(rows.len(), 16);
662 for row in rows {
663 ids.append_value(row.event_uuid.as_bytes())?;
664 runs.append_value(row.run_uuid.as_bytes())?;
665 match row.result_fingerprint {
666 Some(value) => results.append_value(value)?,
667 None => results.append_null(),
668 }
669 provenance.append_value(row.provenance_uuid.as_bytes())?;
670 }
671 RecordBatch::try_new(
672 Arc::clone(&ALGORITHM_RUN_EVENT_SCHEMA),
673 vec![
674 Arc::new(ids.finish()),
675 Arc::new(runs.finish()),
676 Arc::new(StringArray::from_iter_values(
677 rows.iter().map(|row| row.state.as_str()),
678 )),
679 Arc::new(results.finish()),
680 Arc::new(StringArray::from(
681 rows.iter()
682 .map(|row| row.error_code.as_deref())
683 .collect::<Vec<_>>(),
684 )),
685 Arc::new(
686 TimestampMicrosecondArray::from_iter_values(
687 rows.iter().map(|row| row.recorded_at_micros),
688 )
689 .with_timezone("UTC"),
690 ),
691 Arc::new(provenance.finish()),
692 Arc::new(UInt32Array::from_iter_values(
693 rows.iter().map(|row| row.contract_version),
694 )),
695 ],
696 )
697 .map_err(Into::into)
698}
699
700#[cfg(test)]
701mod tests;