flyby 0.1.1

A high-performance Rust framework for composable data-ingestion pipelines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

//! The [`Message`] model: typed records flowing downstream.
//!
//! A message is the unit of work inside a FlyBy pipeline. It carries a
//! schema identifier, a timestamp, optional metadata, and the typed
//! payload itself.
//!
//! Encoding/decoding live in separate traits ([`crate::Encode`],
//! [`crate::Decoder`]); the message type may implement `Encode` when
//! sinks need bytes.

use std::fmt;

/// An opaque identifier for a message schema.
///
/// Backends and decoders use this to dispatch to the correct parser /
/// encoder pair without re-inspecting the payload.
pub trait SchemaId: Copy + Eq + fmt::Debug + Send + Sync + 'static {
    /// A stable, human-readable name for the schema.
    fn name(&self) -> &'static str;

    /// A compact numeric identifier used for on-wire and on-disk encoding.
    ///
    /// Must be unique within a deployed system. The value `0` is reserved
    /// for "unknown / unset" by convention; concrete schemas should use
    /// non-zero values.
    fn id(&self) -> u16;
}

/// Default schema identifier backed by a u16 numeric id.
///
/// Sufficient for early-stage work; production deployments are expected
/// to plug in their own schema-registry-backed identifier.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct DefaultSchemaId(pub u16);

impl SchemaId for DefaultSchemaId {
    fn name(&self) -> &'static str {
        // Type-level name only; the numeric id distinguishes instances.
        "default"
    }

    fn id(&self) -> u16 {
        self.0
    }
}

/// A monotonically meaningful timestamp.
///
/// Stored as nanoseconds since the UNIX epoch to match the resolution of
/// modern hardware timestamps. Sources that only have coarser clocks
/// should zero-fill the low bits rather than lie about precision.
/// Clock domain (hardware vs wall vs logical) is not encoded here.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Timestamp(pub u64);

impl Timestamp {
    /// Construct a timestamp from nanoseconds since the epoch.
    pub const fn from_nanos(nanos: u64) -> Self {
        Self(nanos)
    }

    /// The raw nanosecond value.
    pub const fn as_nanos(self) -> u64 {
        self.0
    }
}

/// Per-message metadata: provenance, sequence numbers, flags.
///
/// Deliberately small and `Copy` so it can travel alongside a message
/// without heap traffic. Richer context lives in user-defined extensions
/// on the message payload itself.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Metadata {
    /// Monotonic sequence number assigned by the source.
    pub sequence: u64,
    /// True when the source marks this message as suspect.
    pub suspect: bool,
}

/// A typed record flowing through the pipeline.
///
/// Each message has:
///
/// - a schema identifier ([`SchemaId`]),
/// - metadata ([`Metadata`]),
/// - a timestamp ([`Timestamp`]),
/// - optional user extensions (on the concrete type).
///
/// Encoding is optional via [`crate::Encode`]. Decoding is performed by a
/// separate [`crate::Decoder`] type.
///
/// Implementors should be cheap to move and `Send` so that messages can
/// cross thread boundaries during placement. `Sync` is required when
/// messages are shared across threads without exclusive ownership.
pub trait Message: Send + Sync + 'static {
    /// The schema identifier type used by this message family.
    type Schema: SchemaId;

    /// Returns this message's schema identifier.
    fn schema_id(&self) -> Self::Schema;

    /// Returns the timestamp assigned to this message.
    fn timestamp(&self) -> Timestamp;

    /// Returns the metadata associated with this message.
    fn metadata(&self) -> Metadata;
}