cu29-logstream 1.2.0

Bounded semantic log streaming protocol and FEC for Copper
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
//! Local sender plans and self-describing receiver requirements.

use crate::{
    ContinuousSenderConfig, DensityThreshold, EncodingSymbolId, Error, Field,
    FiniteObjectSenderConfig, Lane, LogStreamSenderConfig, PACKET_HEADER_LEN, RecordKind,
    RecoverySenderConfig, Result, RlcConfig, StreamIdentity,
    decode_record as decode_semantic_record, encode_record,
};
use alloc::{
    string::{String, ToString},
    vec::Vec,
};
use bincode::{Decode, Encode};
use cu29_runtime::config::{LogStreamDestinationConfig, LogStreamRepairDensity, LogStreamRlcField};
use cu29_traits::TaskOutputSpec;

/// Link and codec policy after RON validation and MTU resolution.
#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
pub struct LogStreamPlan {
    pub feedback: Option<crate::feedback::FeedbackPolicy>,
    pub destination_id: String,
    pub mtu_bytes: u16,
    pub symbol_size: u16,
    pub bitrate_bps: u64,
    pub memory_budget_kib: u32,
    pub max_latency_ms: u32,
    pub burst_packets: u32,
    pub continuous: ResolvedContinuousFec,
    pub objects: ResolvedObjectFec,
    /// Recovery interval in CopperLists.
    pub recovery_interval: u32,
    pub max_record_bytes: u64,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Encode, Decode)]
pub struct ResolvedContinuousFec {
    pub field: ResolvedRlcField,
    pub window_symbols: u16,
    pub repair_every_source_symbols: u16,
    pub repair_density: u8,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Encode, Decode)]
pub enum ResolvedRlcField {
    Gf2,
    Gf256,
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, Encode, Decode)]
pub struct ResolvedObjectFec {
    pub max_object_bytes: u64,
    pub repair_symbols_per_block: u32,
}

/// Minimal application schema required to interpret recovered CopperList slots.
#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
pub struct ApplicationSchema {
    pub outputs: Vec<ApplicationOutputSchema>,
    /// Per-output reconstruction flags (`true` means reconstruct); empty means full capture.
    pub reconstruction: Vec<bool>,
}

#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
pub struct ApplicationOutputSchema {
    pub task_id: String,
    pub message_type: String,
    pub payload_type: String,
}

impl ApplicationSchema {
    pub fn from_output_specs(specs: &[TaskOutputSpec]) -> Self {
        Self {
            reconstruction: Vec::new(),
            outputs: specs
                .iter()
                .map(|spec| ApplicationOutputSchema {
                    task_id: spec.task_id.into(),
                    message_type: spec.msg_type.into(),
                    payload_type: spec.payload_type_path().into(),
                })
                .collect(),
        }
    }
}

/// Continuous decoder geometry and record bounds carried by a session manifest.
/// Receiver-local limits still cap admission; sender pacing and repair policy stay local.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Encode, Decode)]
pub struct ReceiverRequirements {
    /// Present when the sender accepts advisory return reports.
    pub feedback: Option<crate::feedback::FeedbackRequirements>,
    /// Bytes in each continuous FEC symbol, including its protected fragment header.
    pub symbol_size: u16,
    /// Finite field used by the continuous RLC lane.
    pub field: ResolvedRlcField,
    /// Maximum symbols retained in the continuous coding window.
    pub window_symbols: u16,
    /// Maximum complete framed CopperList record size.
    pub max_record_bytes: u64,
}

impl ReceiverRequirements {
    /// Validates untrusted geometry before constructing a continuous decoder.
    pub fn validate(&self) -> Result<()> {
        if self
            .feedback
            .is_some_and(|feedback| feedback.report_interval_ms == 0)
        {
            return Err(Error::InvalidConfig(
                "feedback report interval must be nonzero",
            ));
        }
        if usize::from(self.symbol_size) <= crate::rlc::FRAGMENT_HEADER_LEN {
            return Err(Error::InvalidConfig(
                "symbol leaves no room for continuous record bytes",
            ));
        }
        if usize::from(self.symbol_size) > crate::DEFAULT_MAX_SYMBOL_SIZE
            || usize::from(self.window_symbols) > crate::DEFAULT_MAX_WINDOW_SYMBOLS
        {
            return Err(Error::InvalidConfig(
                "receiver requirements exceed supported FEC capacities",
            ));
        }
        if self.window_symbols == 0 || self.max_record_bytes == 0 {
            return Err(Error::InvalidConfig(
                "receiver requirements contain a zero bound",
            ));
        }
        if self.max_record_bytes > u64::from(u32::MAX) {
            return Err(Error::InvalidConfig("maximum record size exceeds u32"));
        }
        usize::try_from(self.max_record_bytes)
            .map_err(|_| Error::InvalidConfig("record bound exceeds usize"))?;
        self.rlc_config()?;
        Ok(())
    }

    /// Constructs the FEC configuration used by the continuous decoder.
    pub fn rlc_config(&self) -> Result<RlcConfig> {
        RlcConfig::new(
            usize::from(self.symbol_size),
            usize::from(self.window_symbols),
            self.field.into(),
        )
        .map_err(Into::into)
    }
}

/// Control object carrying receiver requirements and application schema checks.
///
/// No content or protocol version is encoded. Receivers must use the matching
/// application's decoder; these schema checks do not establish binary compatibility.
#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
pub struct SessionManifest {
    pub identity: StreamIdentity,
    pub requirements: ReceiverRequirements,
    pub application_schema: ApplicationSchema,
}

impl SessionManifest {
    pub fn new(
        identity: StreamIdentity,
        requirements: ReceiverRequirements,
        application_schema: ApplicationSchema,
    ) -> Self {
        Self {
            identity,
            requirements,
            application_schema,
        }
    }

    pub fn encode_record(&self) -> Result<Vec<u8>> {
        let payload = bincode::encode_to_vec(self, bincode::config::standard())
            .map_err(|error| Error::Codec(error.to_string()))?;
        encode_record(RecordKind::Manifest, 0, &payload)
    }

    pub fn decode_record(record: &[u8]) -> Result<Self> {
        let record = decode_semantic_record(record)?;
        Self::from_decoded(record)
    }

    pub(crate) fn from_decoded(record: crate::DecodedRecord<'_>) -> Result<Self> {
        if record.kind != RecordKind::Manifest {
            return Err(Error::InvalidConfig("record is not a session manifest"));
        }
        let (manifest, consumed): (Self, usize) =
            bincode::decode_from_slice(record.payload, bincode::config::standard())
                .map_err(|error| Error::Codec(error.to_string()))?;
        if consumed != record.payload.len() {
            return Err(Error::Codec("session manifest has trailing bytes".into()));
        }
        manifest.requirements.validate()?;
        Ok(manifest)
    }
}

impl LogStreamPlan {
    /// Resolves one RON destination into local sender policy and wire geometry.
    /// Symbol size is bounded by both the link MTU and preallocated sender storage;
    /// a larger MTU does not increase the runtime's memory footprint.
    pub fn resolve(config: &LogStreamDestinationConfig) -> Result<Self> {
        let symbol_size = config
            .link
            .mtu_bytes
            .checked_sub(PACKET_HEADER_LEN as u16)
            .ok_or(Error::InvalidConfig("MTU does not fit the packet header"))?
            .min(crate::DEFAULT_MAX_SYMBOL_SIZE as u16);
        if usize::from(config.fec.continuous.window_symbols) > crate::DEFAULT_MAX_WINDOW_SYMBOLS {
            return Err(Error::InvalidConfig(
                "RLC window exceeds the generated sender capacity",
            ));
        }
        let field = match config.fec.continuous.field {
            LogStreamRlcField::Gf2 => ResolvedRlcField::Gf2,
            LogStreamRlcField::Gf256 => ResolvedRlcField::Gf256,
        };
        let repair_density = match config.fec.continuous.repair_density {
            LogStreamRepairDensity::Full => DensityThreshold::FULL.get(),
            LogStreamRepairDensity::Threshold(value) => value,
        };
        DensityThreshold::new(repair_density)?;
        let plan = Self {
            feedback: config
                .feedback
                .as_ref()
                .map(crate::feedback::FeedbackPolicy::from),
            destination_id: config.id.clone(),
            mtu_bytes: config.link.mtu_bytes,
            symbol_size,
            bitrate_bps: config.link.bitrate_bps,
            memory_budget_kib: config.link.memory_budget_kib,
            max_latency_ms: config.link.max_latency_ms,
            burst_packets: config.link.burst_packets,
            continuous: ResolvedContinuousFec {
                field,
                window_symbols: config.fec.continuous.window_symbols,
                repair_every_source_symbols: config.fec.continuous.repair_every_source_symbols,
                repair_density,
            },
            objects: ResolvedObjectFec {
                max_object_bytes: config.fec.objects.max_object_bytes,
                repair_symbols_per_block: config.fec.objects.repair_symbols_per_block,
            },
            recovery_interval: config.recovery_interval,
            max_record_bytes: config.max_record_bytes,
        };
        plan.validate()?;
        Ok(plan)
    }

    /// Validates local sender policy, capacities, and memory budget.
    pub fn validate(&self) -> Result<()> {
        if let Some(feedback) = self.feedback {
            feedback.validate(self.continuous.repair_every_source_symbols)?;
        }
        let packet_bytes = PACKET_HEADER_LEN
            .checked_add(usize::from(self.symbol_size))
            .ok_or(Error::InvalidConfig("resolved MTU overflow"))?;
        if usize::from(self.mtu_bytes) < packet_bytes {
            return Err(Error::InvalidConfig("resolved symbol size exceeds the MTU"));
        }
        if usize::from(self.symbol_size) <= crate::rlc::FRAGMENT_HEADER_LEN {
            return Err(Error::InvalidConfig(
                "MTU leaves no room for continuous record bytes",
            ));
        }
        if usize::from(self.symbol_size) > crate::DEFAULT_MAX_SYMBOL_SIZE
            || usize::from(self.continuous.window_symbols) > crate::DEFAULT_MAX_WINDOW_SYMBOLS
        {
            return Err(Error::InvalidConfig(
                "resolved FEC values exceed generated sender capacities",
            ));
        }
        if self.bitrate_bps == 0
            || self.memory_budget_kib == 0
            || self.max_latency_ms == 0
            || self.burst_packets == 0
            || self.continuous.window_symbols == 0
            || self.continuous.repair_every_source_symbols == 0
            || self.objects.max_object_bytes == 0
            || self.objects.repair_symbols_per_block == 0
            || self.recovery_interval == 0
            || self.max_record_bytes == 0
        {
            return Err(Error::InvalidConfig(
                "resolved stream plan contains a zero bound",
            ));
        }
        DensityThreshold::new(self.continuous.repair_density)?;
        let max_record_bytes = usize::try_from(self.max_record_bytes)
            .map_err(|_| Error::InvalidConfig("record bound exceeds usize"))?;
        let rlc = self.rlc_config()?;
        let continuous_bytes = rlc
            .symbol_size()
            .checked_mul(rlc.window_symbols())
            .and_then(|bytes| bytes.checked_add(max_record_bytes))
            .and_then(|bytes| bytes.checked_add(usize::from(self.mtu_bytes)))
            .ok_or(Error::InvalidConfig("stream memory estimate overflow"))?;
        let memory_budget_bytes = usize::try_from(self.memory_budget_kib)
            .ok()
            .and_then(|kib| kib.checked_mul(1024))
            .ok_or(Error::InvalidConfig("stream memory budget overflow"))?;
        if continuous_bytes > memory_budget_bytes {
            return Err(Error::InvalidConfig(
                "continuous sender working set exceeds memory budget",
            ));
        }
        Ok(())
    }

    pub fn rlc_config(&self) -> Result<RlcConfig> {
        RlcConfig::new(
            usize::from(self.symbol_size),
            usize::from(self.continuous.window_symbols),
            self.continuous.field.into(),
        )
        .map_err(Into::into)
    }

    /// Selects only the geometry and bounds needed by the receiver.
    pub fn receiver_requirements(&self) -> ReceiverRequirements {
        ReceiverRequirements {
            feedback: self
                .feedback
                .map(|policy| crate::feedback::FeedbackRequirements {
                    report_interval_ms: policy.report_interval_ms,
                    destination: crate::feedback::destination_key(&self.destination_id),
                }),
            symbol_size: self.symbol_size,
            field: self.continuous.field,
            window_symbols: self.continuous.window_symbols,
            max_record_bytes: self.max_record_bytes,
        }
    }

    /// Builds both sender lanes and their canonical repeated manifest.
    pub fn sender_config(
        &self,
        identity: StreamIdentity,
        schema: ApplicationSchema,
    ) -> Result<LogStreamSenderConfig> {
        self.validate()?;
        let max_record_bytes = usize::try_from(self.max_record_bytes)
            .map_err(|_| Error::InvalidConfig("record bound exceeds usize"))?;
        let manifest =
            SessionManifest::new(identity, self.receiver_requirements(), schema).encode_record()?;
        if manifest.len() as u64 > self.objects.max_object_bytes {
            return Err(Error::ObjectTooLarge {
                actual: manifest.len() as u64,
                maximum: self.objects.max_object_bytes,
            });
        }
        Ok(LogStreamSenderConfig {
            feedback: self.feedback,
            pacing: crate::PacingConfig {
                bitrate_bps: self.bitrate_bps,
                burst_packets: self.burst_packets,
                max_latency: cu29_clock::CuDuration::from_millis(u64::from(self.max_latency_ms)),
                memory_budget_bytes: self.memory_budget_kib as usize * 1024,
            },
            continuous: ContinuousSenderConfig {
                identity,
                lane: Lane::ReplayCritical,
                fec: self.rlc_config()?,
                max_record_bytes,
                initial_esi: EncodingSymbolId::new(0),
                repair_every_source_symbols: usize::from(
                    self.continuous.repair_every_source_symbols,
                ),
                first_repair_key: 0,
                repair_density: DensityThreshold::new(self.continuous.repair_density)?,
            },
            recovery: RecoverySenderConfig {
                finite: FiniteObjectSenderConfig {
                    identity,
                    lane: Lane::Control,
                    symbol_size: self.symbol_size,
                    max_object_bytes: self.objects.max_object_bytes,
                    repair_symbols_per_block: self.objects.repair_symbols_per_block,
                },
                manifest_record: manifest,
                recovery_interval: self.recovery_interval,
            },
        })
    }
}

impl From<ResolvedRlcField> for Field {
    fn from(field: ResolvedRlcField) -> Self {
        match field {
            ResolvedRlcField::Gf2 => Self::Gf2,
            ResolvedRlcField::Gf256 => Self::Gf256,
        }
    }
}

/// Creates a process-local session identity without adding configuration or environment inputs.
#[cfg(feature = "std")]
pub fn new_session_id() -> [u8; 16] {
    use core::sync::atomic::{AtomicU64, Ordering};
    use std::time::{SystemTime, UNIX_EPOCH};

    static NEXT_SESSION: AtomicU64 = AtomicU64::new(0);
    let counter = NEXT_SESSION.fetch_add(1, Ordering::Relaxed);
    let now = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap_or_default()
        .as_nanos();
    let mut hasher = blake3::Hasher::new();
    hasher.update(&now.to_le_bytes());
    hasher.update(&counter.to_le_bytes());
    hasher.update(&std::process::id().to_le_bytes());
    hasher.finalize().as_bytes()[..16].try_into().unwrap()
}