Struct CanonicalIngestRecord 

pub struct CanonicalIngestRecord {
    pub id: String,
    pub tenant_id: String,
    pub doc_id: String,
    pub received_at: DateTime<Utc>,
    pub original_source: Option<String>,
    pub source: IngestSource,
    pub normalized_payload: Option<CanonicalPayload>,
    pub attributes: Option<Value>,
}

Normalized record produced by ingest.

CanonicalIngestRecord is the output of the ingest pipeline. It represents a cleaned, validated, and deterministic version of the input that downstream stages can rely on.

Guarantees

• All required fields are present (tenant_id, doc_id, received_at)
• Metadata is sanitized (control characters stripped)
• Payload is normalized (text whitespace collapsed, binary preserved)
• Document ID is stable (derived deterministically if not provided)

Examples

use ingest::{ingest, IngestConfig, RawIngestRecord, CanonicalPayload};
use ingest::{IngestMetadata, IngestSource, IngestPayload};

let config = IngestConfig::default();
let record = RawIngestRecord {
    id: "test-001".to_string(),
    source: IngestSource::RawText,
    metadata: IngestMetadata {
        tenant_id: Some("tenant".to_string()),
        doc_id: None, // Will be derived
        received_at: None, // Will default to now
        original_source: None,
        attributes: None,
    },
    payload: Some(IngestPayload::Text("  Hello   world  ".to_string())),
};

let canonical = ingest(record, &config).unwrap();

// All fields are guaranteed present
assert!(!canonical.tenant_id.is_empty());
assert!(!canonical.doc_id.is_empty());

// Text is normalized
match &canonical.normalized_payload {
    Some(CanonicalPayload::Text(text)) => {
        assert_eq!(text, "Hello world");
    }
    _ => panic!("Expected text payload"),
}

Fields

id: String

Unique identifier for this ingest operation (mirrors RawIngestRecord::id).

This is the sanitized version of the original ID (control characters stripped).

tenant_id: String

Tenant identifier for multi-tenant isolation.

This is the effective tenant ID after applying defaults:

• If provided and non-empty: the sanitized provided value
• Otherwise: IngestConfig::default_tenant_id

doc_id: String

Document identifier.

This is the effective document ID after derivation:

• If provided and non-empty: the sanitized provided value
• Otherwise: UUIDv5 derived from tenant + record ID

received_at: DateTime<Utc>

Timestamp when the record was received.

This is the effective timestamp after applying defaults:

• If provided: the sanitized provided value
• Otherwise: current UTC time at ingest

original_source: Option<String>

Original source information if provided.

Sanitized version of IngestMetadata::original_source with control characters stripped. None if not provided.

source: IngestSource

Source of the content (mirrors RawIngestRecord::source).

normalized_payload: Option<CanonicalPayload>

Normalized payload ready for downstream stages.

• For text: whitespace collapsed, size limits enforced
• For binary: preserved unchanged, non-empty check performed
• None if no payload was provided

attributes: Option<Value>

Attributes JSON preserved for downstream use.

This is the sanitized and size-checked version of IngestMetadata::attributes. None if not provided.

Implementations

impl CanonicalIngestRecord

pub fn has_text_payload(&self) -> bool

Returns true if this record has a text payload.

Example

use ingest::{CanonicalIngestRecord, CanonicalPayload};

let record = CanonicalIngestRecord {
    id: "test".to_string(),
    tenant_id: "tenant".to_string(),
    doc_id: "doc".to_string(),
    received_at: chrono::Utc::now(),
    original_source: None,
    source: ingest::IngestSource::RawText,
    normalized_payload: Some(CanonicalPayload::Text("hello".to_string())),
    attributes: None,
};

assert!(record.has_text_payload());

pub fn has_binary_payload(&self) -> bool

Returns true if this record has a binary payload.

Example

use ingest::{CanonicalIngestRecord, CanonicalPayload};

let record = CanonicalIngestRecord {
    id: "test".to_string(),
    tenant_id: "tenant".to_string(),
    doc_id: "doc".to_string(),
    received_at: chrono::Utc::now(),
    original_source: None,
    source: ingest::IngestSource::File {
        filename: "test.bin".to_string(),
        content_type: None,
    },
    normalized_payload: Some(CanonicalPayload::Binary(vec![1, 2, 3])),
    attributes: None,
};

assert!(record.has_binary_payload());

pub fn text_payload(&self) -> Option<&str>

Returns the text payload if present, otherwise None.

Example

use ingest::{CanonicalIngestRecord, CanonicalPayload};

let record = CanonicalIngestRecord {
    id: "test".to_string(),
    tenant_id: "tenant".to_string(),
    doc_id: "doc".to_string(),
    received_at: chrono::Utc::now(),
    original_source: None,
    source: ingest::IngestSource::RawText,
    normalized_payload: Some(CanonicalPayload::Text("hello world".to_string())),
    attributes: None,
};

assert_eq!(record.text_payload(), Some("hello world"));

pub fn binary_payload(&self) -> Option<&[u8]>

Returns the binary payload if present, otherwise None.

Trait Implementations

impl Clone for CanonicalIngestRecord

fn clone(&self) -> CanonicalIngestRecord

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CanonicalIngestRecord

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for CanonicalIngestRecord

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl PartialEq for CanonicalIngestRecord

fn eq(&self, other: &CanonicalIngestRecord) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for CanonicalIngestRecord

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Eq for CanonicalIngestRecord

impl StructuralPartialEq for CanonicalIngestRecord