Skip to main content

khive_runtime/
streams.rs

1//! Ordered note streams. Embedding preparation precedes one SQL-only writer
2//! transaction containing the sequence predicates, note/index writes and ledger
3//! rows. A batch is that transaction over several members (atomic mode) or one
4//! such transaction per member, in list order (per-member mode).
5use std::any::Any;
6use std::collections::{HashMap, HashSet};
7use std::sync::{
8    atomic::{AtomicBool, Ordering},
9    Arc,
10};
11
12use khive_storage::{
13    AtomicUnitOp, Note, SqlAccess, SqlStatement, SqlValue, SqlWriter, StorageCapability,
14    StorageError, WriterTaskRequestState,
15};
16use khive_types::{Details, KhiveError};
17use serde::Deserialize;
18use serde_json::{json, Value};
19use uuid::Uuid;
20
21use crate::atomic_message::{
22    prepare_atomic_note_requests, AtomicNoteOptions, AtomicNoteRequest, AtomicNoteSpec,
23};
24use crate::atomic_plan::{PlanStatement, PostCommitEffect};
25use crate::atomic_runner::{
26    apply_plan, run_prepared_atomic_unit, AtomicOpFailure, AtomicOpPlan,
27    CommittedPostCommitEffects, PreparedAtomicError, PreparedAtomicOp, PreparedAtomicOutcome,
28};
29use crate::note_write::{
30    NoteFence, NoteFences, NoteWriteConflict, NoteWriteGuard, NoteWriteOptions,
31};
32use crate::{
33    micros_to_iso, DomainDisposition, KhiveRuntime, NamespaceToken, RuntimeError, RuntimeResult,
34    VerbRegistry,
35};
36
37mod truncate;
38
39#[cfg(test)]
40mod append_failure_tests;
41
42fn validate_stream(stream: &str) -> RuntimeResult<()> {
43    crate::note_validation::validate_key(
44        stream,
45        "stream must be at most 512 UTF-8 bytes and contain no U+0000",
46    )
47}
48
49fn write_failure(message: &str) -> StorageError {
50    StorageError::Conflict {
51        capability: StorageCapability::Notes,
52        operation: "stream.append".into(),
53        message: message.into(),
54    }
55}
56
57/// What the append path can prove about its own failed append, independently
58/// of an enclosing dispatch or a nested error's wire disposition.
59#[derive(Clone, Copy, Debug, PartialEq, Eq)]
60pub enum StreamAppendDisposition {
61    /// This append either never reached a write or its transaction rolled back.
62    NotCommitted,
63    /// The append may have committed; no absence of a record is asserted.
64    Unknown,
65}
66
67/// An unchanged source error together with evidence local to one append.
68#[derive(Debug, thiserror::Error)]
69#[error("{source}")]
70pub struct StreamAppendFailure {
71    #[source]
72    source: RuntimeError,
73    disposition: StreamAppendDisposition,
74}
75
76impl StreamAppendFailure {
77    fn not_committed(source: RuntimeError) -> Self {
78        Self {
79            source,
80            disposition: StreamAppendDisposition::NotCommitted,
81        }
82    }
83
84    /// Retain an error when no append-local committedness proof is available.
85    /// This constructor cannot assert that an append was dropped.
86    pub fn unknown(source: RuntimeError) -> Self {
87        Self {
88            source,
89            disposition: StreamAppendDisposition::Unknown,
90        }
91    }
92
93    fn after_submission(source: RuntimeError, refused_before_write: bool) -> Self {
94        let disposition = if let Some(context) = source.writer_task_failure_context() {
95            match context.request_state {
96                WriterTaskRequestState::NotStarted
97                | WriterTaskRequestState::TransactionRolledBack => {
98                    StreamAppendDisposition::NotCommitted
99                }
100                WriterTaskRequestState::SideEffectsUnknown => StreamAppendDisposition::Unknown,
101            }
102        } else if refused_before_write
103            || source.admission_failure_context().is_some()
104            || matches!(
105                &source,
106                RuntimeError::Storage(error) if matches!(error.without_sqlite_write_stage(), StorageError::WriterTaskBusy { .. })
107            )
108        {
109            StreamAppendDisposition::NotCommitted
110        } else {
111            StreamAppendDisposition::Unknown
112        };
113        Self {
114            source,
115            disposition,
116        }
117    }
118
119    /// Borrow the original error without rewriting its fields or provenance.
120    pub fn source(&self) -> &RuntimeError {
121        &self.source
122    }
123
124    /// Evidence about this append, not the enclosing handler's disposition.
125    pub fn disposition(&self) -> StreamAppendDisposition {
126        self.disposition
127    }
128
129    /// Separate the unchanged source from append-local committedness evidence.
130    pub fn into_parts(self) -> (RuntimeError, StreamAppendDisposition) {
131        (self.source, self.disposition)
132    }
133
134    /// Preserve the compatibility error path without an additional wrapper.
135    pub fn into_source(self) -> RuntimeError {
136        self.source
137    }
138}
139
140/// One append, shape-validated by the verb layer; the runtime validates the
141/// stream name and the record's serialization before any write.
142pub struct StreamAppendSpec {
143    pub stream: String,
144    pub record: Value,
145    pub expected_seq: Option<i64>,
146    pub embed: Option<bool>,
147    pub embedding_model: Option<String>,
148    pub note_kind: String,
149    pub tags: Option<Vec<String>>,
150    pub fence: Option<NoteFences>,
151}
152
153impl StreamAppendSpec {
154    fn validate_embedding(&self, member: Option<usize>) -> RuntimeResult<()> {
155        if self.embedding_model.is_some() && self.embed != Some(true) {
156            let mut error = KhiveError::invalid_input("embedding_model requires embed=true");
157            if let Some(member) = member {
158                error = error.with_details(Details::new_owned([("member", member.to_string())]));
159            }
160            return Err(error.into());
161        }
162        Ok(())
163    }
164
165    fn note_options(&self) -> AtomicNoteOptions<'_> {
166        AtomicNoteOptions {
167            embed: Some(self.embed.unwrap_or(false)),
168            embedding_model: self.embedding_model.as_deref(),
169            ..Default::default()
170        }
171    }
172}
173
174/// A keyed document write. Kinds are canonical note-kind names. A missing
175/// expected version creates only; a positive version updates only.
176#[derive(Clone, Debug)]
177pub struct StreamWriteSpec {
178    pub key: String,
179    pub kind: String,
180    pub doc: Value,
181    pub tags: Option<Vec<String>>,
182    pub embed: Option<bool>,
183    pub expected_version: Option<i64>,
184}
185
186/// An exact live-key observation; a null version asserts that the key is unheld.
187/// An optional identity pins which live note must hold the key at that version.
188#[derive(Clone, Debug)]
189pub struct StreamObservation {
190    pub key: String,
191    pub kind: String,
192    pub version: Option<i64>,
193    pub id: Option<Uuid>,
194    /// Dotted document path whose RFC 3339 value must exceed the writer clock.
195    pub live_until: Option<String>,
196}
197
198/// A batch member or its already established refusal. The mode places it.
199pub enum StreamBatchMember {
200    Append(StreamAppendSpec),
201    Write(StreamWriteSpec),
202    Refused(KhiveError),
203}
204
205// Like the per-list fence and observation bounds, this limits precondition
206// reads while holding the writer; member count alone does not bound their sum.
207const MAX_BATCH_FENCE_ENTRIES: usize = 100;
208
209fn stream_batch_fence_count(members: &[StreamBatchMember], fence: Option<&NoteFence>) -> usize {
210    usize::from(fence.is_some())
211        + members
212            .iter()
213            .filter_map(|member| match member {
214                StreamBatchMember::Append(spec) => spec.fence.as_ref(),
215                StreamBatchMember::Write(_) | StreamBatchMember::Refused(_) => None,
216            })
217            .map(|fences| fences.entries().len())
218            .sum::<usize>()
219}
220
221fn validate_stream_batch_fence_count(count: usize) -> RuntimeResult<()> {
222    crate::note_validation::validate_fence_count(count, MAX_BATCH_FENCE_ENTRIES, || {
223        format!(
224            "stream.batch admits at most {MAX_BATCH_FENCE_ENTRIES} total fence entries; this call sent {count}: each fence is a read taken while holding the writer"
225        )
226    })
227}
228
229/// The member refusal that stopped an atomic batch; nothing was written.
230#[derive(Debug)]
231pub struct StreamBatchRefusal {
232    pub member: usize,
233    pub error: KhiveError,
234}
235
236/// A member refusal as a value: the error object a refused op carries, plus
237/// the disposition the consumer rule reads without a special case.
238pub fn refusal_value(error: &KhiveError) -> RuntimeResult<Value> {
239    let mut value = serde_json::to_value(error)
240        .map_err(|e| RuntimeError::Internal(format!("stream refusal serialization: {e}")))?;
241    value["domain_disposition"] = json!(DomainDisposition::NotCommitted.as_str());
242    Ok(value)
243}
244
245fn seq_conflict(stream: &str, expected: i64, next: i64, member: Option<usize>) -> KhiveError {
246    let mut pairs = vec![
247        ("reason", "seq_conflict".to_string()),
248        ("stream", stream.to_string()),
249        ("expected_seq", expected.to_string()),
250        ("next_seq", next.to_string()),
251    ];
252    if let Some(member) = member {
253        pairs.push(("member", member.to_string()));
254    }
255    KhiveError::conflict("stream sequence precondition failed")
256        .with_details(Details::new_owned(pairs))
257}
258
259fn sequence_exhausted() -> KhiveError {
260    KhiveError::invalid_input("stream sequence exhausted").with_details(Details::new_owned([(
261        "reason",
262        "sequence_exhausted".into(),
263    )]))
264}
265
266/// A prepared append: the note, its planned statements and the ledger key.
267struct PreparedAppend {
268    stream: String,
269    expected_seq: Option<i64>,
270    note: Note,
271    statements: Vec<PlanStatement>,
272    guard: NoteWriteGuard,
273}
274
275fn append_result(prepared: &PreparedAppend, seq: i64) -> Value {
276    json!({"seq": seq, "id": prepared.note.id, "created_at": micros_to_iso(prepared.note.created_at)})
277}
278
279enum BatchOutcome {
280    Appended(Vec<i64>),
281    Exhausted,
282    Conflict { next: i64 },
283    FenceConflict { conflict: NoteWriteConflict },
284}
285
286struct PreparedBatchMember {
287    index: usize,
288    action: PreparedBatchAction,
289}
290
291enum PreparedBatchAction {
292    Append {
293        stream: String,
294        expected_seq: Option<i64>,
295        note: Box<Note>,
296        plan: AtomicOpPlan,
297        fence: Option<NoteFences>,
298    },
299    Write {
300        key: String,
301        kind: String,
302        id: Uuid,
303        plan: AtomicOpPlan,
304        after_create: Option<Value>,
305    },
306    Refused(KhiveError),
307}
308
309#[derive(Deserialize)]
310struct StreamCreateFields {
311    content: String,
312    name: Option<String>,
313    properties: Option<Value>,
314    tags: Option<Vec<String>>,
315    salience: Option<f64>,
316    decay_factor: Option<f64>,
317    embed: Option<bool>,
318}
319
320fn normalize_stream_create_fields(fields: &mut StreamCreateFields) -> RuntimeResult<()> {
321    let tags = fields.tags.take();
322    if let crate::EffectiveCreateTags::TopLevel(tags) =
323        crate::effective_create_tags(tags.as_deref(), fields.properties.as_ref())
324    {
325        let tags = json!(tags);
326        let mut properties = match fields.properties.take() {
327            None => serde_json::Map::new(),
328            Some(Value::Object(properties)) => properties,
329            Some(_) => {
330                return Err(RuntimeError::InvalidInput(
331                    "note tags require object properties".into(),
332                ))
333            }
334        };
335        properties.insert("tags".into(), tags);
336        fields.properties = Some(Value::Object(properties));
337    }
338    Ok(())
339}
340
341struct PreparedStreamCreate {
342    key: String,
343    kind: String,
344    fields: StreamCreateFields,
345    args: Value,
346}
347
348enum StreamBatchPreparation {
349    Ready(Box<PreparedBatchAction>),
350    Append(Box<(StreamAppendSpec, String)>),
351    Create(Box<PreparedStreamCreate>),
352}
353
354fn batch_create_hooks(members: &[PreparedBatchMember]) -> Vec<(Uuid, String, Value)> {
355    members
356        .iter()
357        .filter_map(|member| match &member.action {
358            PreparedBatchAction::Write {
359                id,
360                kind,
361                after_create: Some(args),
362                ..
363            } => Some((*id, kind.clone(), args.clone())),
364            _ => None,
365        })
366        .collect()
367}
368
369fn place_member(error: KhiveError, member: Option<usize>) -> RuntimeResult<KhiveError> {
370    let pairs = member
371        .into_iter()
372        .map(|member| ("member".to_owned(), member.to_string()))
373        .chain(
374            error
375                .details()
376                .into_iter()
377                .flat_map(Details::iter)
378                .filter(|(key, _)| *key != "member")
379                .map(|(key, value)| (key.to_owned(), value.to_owned())),
380        );
381    let details = Details::deserialize(serde::de::value::MapDeserializer::<
382        _,
383        serde::de::value::Error,
384    >::new(pairs))
385    .map_err(|error| RuntimeError::Internal(format!("stream member details: {error}")))?;
386    Ok(error.with_details(details))
387}
388
389fn missing_write(key: &str) -> KhiveError {
390    KhiveError::not_found("note key", key).with_details(Details::new_owned([
391        ("reason", "stream_write_not_found".into()),
392        ("key", key.into()),
393    ]))
394}
395
396async fn check_observed(
397    writer: &mut dyn SqlWriter,
398    namespace: &str,
399    observed: &[StreamObservation],
400    now: Option<i64>,
401) -> Result<Option<KhiveError>, StorageError> {
402    for (index, entry) in observed.iter().enumerate() {
403        let current = crate::fence_identity::read_holder(
404            writer,
405            namespace,
406            &entry.kind,
407            &entry.key,
408            "stream-batch-observed",
409        )
410        .await?;
411        // No live holder is the existing version conflict. Identity conflict
412        // specifically names a replacement, so current_id is always present.
413        if let (Some(asserted), Some(holder)) = (entry.id, current.as_ref()) {
414            if asserted != holder.id {
415                let version = entry.version.ok_or_else(|| {
416                    StorageError::Internal("identity observation missing version".into())
417                })?;
418                let mut details = vec![
419                    ("reason", "identity_conflict".into()),
420                    ("key", entry.key.clone()),
421                    ("kind", entry.kind.clone()),
422                    ("version", version.to_string()),
423                ];
424                details.extend(crate::fence_identity::identity_evidence(
425                    asserted, holder.id,
426                ));
427                details.push(("index", index.to_string()));
428                return Ok(Some(
429                    KhiveError::conflict("stream observation identity precondition failed")
430                        .with_details(Details::new_owned(details)),
431                ));
432            }
433        }
434        let current = current.map(|holder| holder.version);
435        if current != entry.version {
436            let mut details = vec![
437                ("reason", "version_conflict".into()),
438                ("key", entry.key.clone()),
439                ("index", index.to_string()),
440            ];
441            if let Some(expected) = entry.version {
442                details.push(("expected_version", expected.to_string()));
443            }
444            if let Some(current) = current {
445                details.push(("current_version", current.to_string()));
446            }
447            return Ok(Some(
448                KhiveError::conflict("stream observation precondition failed")
449                    .with_details(Details::new_owned(details)),
450            ));
451        }
452        if let Some(field) = &entry.live_until {
453            let now = now
454                .ok_or_else(|| StorageError::Internal("missing stream observation clock".into()))?;
455            if let Some(refusal) = crate::live_until::evaluate(
456                writer,
457                namespace,
458                &entry.kind,
459                &entry.key,
460                field,
461                now,
462                "stream-batch-live-until",
463            )
464            .await?
465            {
466                let mut details = vec![
467                    ("reason", refusal.reason().into()),
468                    ("key", entry.key.clone()),
469                    ("kind", entry.kind.clone()),
470                    ("version", entry.version.unwrap().to_string()),
471                    ("field", field.clone()),
472                    ("index", index.to_string()),
473                ];
474                details.extend(refusal.details());
475                return Ok(Some(
476                    KhiveError::conflict("stream observation time precondition failed")
477                        .with_details(Details::new_owned(details)),
478                ));
479            }
480        }
481    }
482    Ok(None)
483}
484
485async fn observation_clock(writer: &mut dyn SqlWriter) -> Result<i64, StorageError> {
486    crate::live_until::writer_clock(writer, "stream-batch-clock").await
487}
488
489struct BatchFailure {
490    member: Option<usize>,
491    error: RuntimeError,
492}
493
494async fn run_prepared_stream_batch(
495    access: &dyn SqlAccess,
496    namespace: String,
497    members: Vec<PreparedBatchMember>,
498    fence: Option<NoteFence>,
499    observed: Vec<StreamObservation>,
500) -> RuntimeResult<Result<(Vec<Value>, CommittedPostCommitEffects), StreamBatchRefusal>> {
501    let op: PreparedAtomicOp<Vec<Value>, BatchFailure> = Box::new(move |writer| {
502        Box::pin(async move {
503            // One SQL clock read after writer admission, shared by the list.
504            let now = if observed.iter().any(|entry| entry.live_until.is_some()) {
505                Some(observation_clock(writer).await?)
506            } else {
507                None
508            };
509            let guard = NoteWriteGuard {
510                namespace: namespace.clone(),
511                target_id: Uuid::nil(),
512                expected_version: None,
513                fence: fence.map(Into::into),
514                create_key: None,
515            };
516            let predicate_error = if let Some(conflict) = guard.check_fence(writer).await? {
517                Some(conflict.into_error())
518            } else {
519                check_observed(writer, &namespace, &observed, now).await?
520            };
521            if let Some(error) = predicate_error {
522                return Err(PreparedAtomicError::Refused {
523                    failure: BatchFailure {
524                        member: None,
525                        error: error.into(),
526                    },
527                    message: "stream batch predicate refused".into(),
528                });
529            }
530            let mut results = Vec::with_capacity(members.len());
531            let mut effects = Vec::new();
532            let mut heads = HashMap::new();
533            // Member fences observe the transaction's initial state, even when
534            // an earlier keyed write changes a fenced note in this batch.
535            for member in &members {
536                if let PreparedBatchAction::Append { fence, .. } = &member.action {
537                    let guard = NoteWriteGuard {
538                        namespace: namespace.clone(),
539                        target_id: Uuid::nil(),
540                        expected_version: None,
541                        fence: fence.clone(),
542                        create_key: None,
543                    };
544                    if let Some(conflict) = guard.check_fence(writer).await? {
545                        return Err(PreparedAtomicError::Refused {
546                            failure: BatchFailure {
547                                member: Some(member.index),
548                                error: conflict.into_error().into(),
549                            },
550                            message: "stream append fence refused".into(),
551                        });
552                    }
553                }
554            }
555            for member in members {
556                let result =
557                    apply_stream_member(writer, &namespace, &mut heads, member.action).await;
558                match result {
559                    Ok((value, effect)) => {
560                        results.push(value);
561                        if let Some(effect) = effect {
562                            effects.push(effect);
563                        }
564                    }
565                    Err(error) => {
566                        return Err(PreparedAtomicError::Refused {
567                            failure: BatchFailure {
568                                member: Some(member.index),
569                                error,
570                            },
571                            message: "stream batch member refused".into(),
572                        });
573                    }
574                }
575            }
576            Ok((results, effects))
577        })
578    });
579    match run_prepared_atomic_unit(access, op).await? {
580        PreparedAtomicOutcome::Committed { value, post_commit } => Ok(Ok((value, post_commit))),
581        PreparedAtomicOutcome::RolledBack(BatchFailure {
582            member: Some(member),
583            error: RuntimeError::Khive(error),
584        }) => Ok(Err(StreamBatchRefusal {
585            member,
586            error: place_member(error, Some(member))?,
587        })),
588        PreparedAtomicOutcome::RolledBack(BatchFailure { error, .. }) => Err(error),
589    }
590}
591
592enum SequenceRefusal {
593    Exhausted,
594    Conflict { expected: i64, next: i64 },
595}
596
597fn allocate_sequence(head: &mut i64, expected: Option<i64>) -> Result<i64, SequenceRefusal> {
598    let next = head.checked_add(1).ok_or(SequenceRefusal::Exhausted)?;
599    if let Some(expected) = expected.filter(|expected| *expected != next) {
600        return Err(SequenceRefusal::Conflict { expected, next });
601    }
602    // Reserve the final i64 for a whole-prefix floor. Check every allocation,
603    // including a number obtained from this transaction's cached stream head.
604    if next == i64::MAX {
605        return Err(SequenceRefusal::Exhausted);
606    }
607    *head = next;
608    Ok(next)
609}
610
611// The persisted floor retains the head when truncation removes every entry.
612// Scalar users read the first column; read/truncate also consume the floor.
613const STREAM_POSITION_SQL: &str = "SELECT MAX(\
614    COALESCE((SELECT MAX(seq) FROM note_streams WHERE namespace=?1 AND stream=?2),0), \
615    COALESCE((SELECT floor_seq-1 FROM note_stream_floors WHERE namespace=?1 AND stream=?2),0)) AS head_seq, \
616    COALESCE((SELECT floor_seq FROM note_stream_floors WHERE namespace=?1 AND stream=?2),1) AS floor_seq";
617
618fn stream_position_statement(namespace: &str, stream: &str) -> SqlStatement {
619    SqlStatement::new(
620        STREAM_POSITION_SQL,
621        vec![
622            SqlValue::Text(namespace.into()),
623            SqlValue::Text(stream.into()),
624        ],
625    )
626    .labelled("stream")
627}
628
629async fn insert_stream_entry(
630    writer: &mut dyn SqlWriter,
631    namespace: &str,
632    stream: String,
633    seq: i64,
634    note_id: String,
635) -> Result<(), StorageError> {
636    writer
637        .execute(
638            SqlStatement::new(
639                "INSERT INTO note_streams(namespace,stream,seq,note_id) VALUES (?1,?2,?3,?4)",
640                vec![
641                    SqlValue::Text(namespace.into()),
642                    SqlValue::Text(stream),
643                    SqlValue::Integer(seq),
644                    SqlValue::Text(note_id),
645                ],
646            )
647            .labelled("stream"),
648        )
649        .await?;
650    Ok(())
651}
652
653async fn apply_stream_member(
654    writer: &mut dyn SqlWriter,
655    namespace: &str,
656    heads: &mut HashMap<String, i64>,
657    action: PreparedBatchAction,
658) -> RuntimeResult<(Value, Option<PostCommitEffect>)> {
659    match action {
660        PreparedBatchAction::Refused(error) => Err(error.into()),
661        PreparedBatchAction::Append {
662            stream,
663            expected_seq,
664            note,
665            plan,
666            ..
667        } => {
668            let head = match heads.entry(stream.clone()) {
669                std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
670                std::collections::hash_map::Entry::Vacant(entry) => {
671                    let head = writer
672                        .query_scalar(stream_position_statement(namespace, &stream))
673                        .await?;
674                    let Some(SqlValue::Integer(head)) = head else {
675                        return Err(RuntimeError::Internal("invalid stream head".into()));
676                    };
677                    entry.insert(head)
678                }
679            };
680            let next =
681                allocate_sequence(head, expected_seq).map_err(|refusal| -> RuntimeError {
682                    match refusal {
683                        SequenceRefusal::Exhausted => sequence_exhausted().into(),
684                        SequenceRefusal::Conflict { expected, next } => {
685                            seq_conflict(&stream, expected, next, None).into()
686                        }
687                    }
688                })?;
689            let applied = apply_plan(writer, &plan, false).await.map_err(|error| {
690                RuntimeError::Internal(format!("stream append plan failed: {error:?}"))
691            })?;
692            insert_stream_entry(writer, namespace, stream, next, note.id.to_string()).await?;
693            Ok((
694                json!({"seq": next, "id": note.id, "created_at": micros_to_iso(note.created_at)}),
695                applied.effect,
696            ))
697        }
698        PreparedBatchAction::Write {
699            key,
700            kind,
701            id,
702            plan,
703            ..
704        } => match apply_plan(writer, &plan, false).await {
705            Ok(applied) => {
706                // Read the row under this writer, before commit or any later
707                // writer. Preserve its existing timestamp/CAS semantics.
708                let stored = writer
709                    .query_row(SqlStatement {
710                        sql: "SELECT version, updated_at FROM notes WHERE namespace=?1 AND id=?2"
711                            .into(),
712                        params: vec![
713                            SqlValue::Text(namespace.into()),
714                            SqlValue::Text(id.to_string()),
715                        ],
716                        label: Some("stream-batch-write-time".into()),
717                    })
718                    .await?;
719                let Some(stored) = stored else {
720                    return Err(RuntimeError::Internal(
721                        "missing stream write timestamp".into(),
722                    ));
723                };
724                Ok((
725                    json!({"id": id, "version": stored.i64("version").map_err(|_| RuntimeError::Internal("stream query missing integer version".to_owned()))?, "updated_at": micros_to_iso(stored.i64("updated_at").map_err(|_| RuntimeError::Internal("stream query missing integer updated_at".to_owned()))?)}),
726                    applied.effect,
727                ))
728            }
729            Err(AtomicOpFailure::NoteConflict(conflict)) => Err(conflict.into_error().into()),
730            Err(AtomicOpFailure::GuardFailed { .. }) => {
731                let holder = writer
732                    .query_row(
733                        khive_db::statement_lists::live_holder_read(namespace, &kind, &key)
734                            .labelled("stream"),
735                    )
736                    .await?;
737                let Some(holder) = holder else {
738                    return Err(missing_write(&key).into());
739                };
740                // Versions are local to a note identity. A replacement can be
741                // at the expected version without being the prepared target.
742                if holder.text("id").map_err(|_| {
743                    RuntimeError::Internal("stream query missing text id".to_owned())
744                })? != id.to_string()
745                {
746                    if let AtomicOpPlan::Update(update) = &plan {
747                        if let Some(expected) = update
748                            .note_guard
749                            .as_ref()
750                            .and_then(|guard| guard.expected_version)
751                        {
752                            return Err(NoteWriteConflict::Version {
753                                expected,
754                                current: holder.i64("version").map_err(|_| {
755                                    RuntimeError::Internal(
756                                        "stream query missing integer version".to_owned(),
757                                    )
758                                })?,
759                            }
760                            .into_error()
761                            .into());
762                        }
763                    }
764                }
765                Err(crate::curation::stale_note_snapshot_error(id))
766            }
767            Err(error) => Err(RuntimeError::Internal(format!(
768                "stream write plan failed: {error:?}"
769            ))),
770        },
771    }
772}
773
774impl KhiveRuntime {
775    /// Validate every spec and prepare every note before any write. All
776    /// allocations for embedding, note normalization and SQL plans complete
777    /// here, so only bounded statement driving occurs while the writer is held.
778    async fn prepare_stream_appends(
779        &self,
780        token: &NamespaceToken,
781        specs: &[&StreamAppendSpec],
782        registry: &VerbRegistry,
783    ) -> RuntimeResult<Vec<PreparedAppend>> {
784        let mut fields = Vec::with_capacity(specs.len());
785        for spec in specs {
786            spec.validate_embedding(None)?;
787            validate_stream(&spec.stream)?;
788            if let Some(fences) = &spec.fence {
789                fences.validate()?;
790                for fence in fences.entries() {
791                    self.validate_note_kind(&fence.kind)?;
792                }
793            }
794            let content = serde_json::to_string(&spec.record)
795                .map_err(|e| RuntimeError::InvalidInput(e.to_string()))?;
796            let hook = registry.find_kind_hook(&spec.note_kind);
797            // A hooked kind may use its ordinary create fields from an object record (for
798            // example, `task` reads `title`), while an unhooked kind keeps the append record as
799            // opaque JSON content exactly as before.
800            let mut args = if hook.is_some() {
801                match &spec.record {
802                    Value::Object(record) => Value::Object(record.clone()),
803                    _ => json!({}),
804                }
805            } else {
806                json!({})
807            };
808            args["kind"] = json!("note");
809            args["note_kind"] = json!(&spec.note_kind);
810            args["content"] = json!(content);
811            args["namespace"] = json!(token.namespace().as_str());
812            if let Some(tags) = &spec.tags {
813                args["tags"] = json!(tags);
814            }
815            if let Some(embed) = spec.embed {
816                args["embed"] = json!(embed);
817            }
818            if let Some(hook) = hook {
819                hook.prepare_create(self, &mut args).await?;
820            }
821            let mut note_fields: StreamCreateFields =
822                serde_json::from_value(args).map_err(|error| {
823                    RuntimeError::InvalidInput(format!("stream append fields: {error}"))
824                })?;
825            normalize_stream_create_fields(&mut note_fields)?;
826            fields.push(note_fields);
827        }
828        let atomic_specs = specs
829            .iter()
830            .zip(&fields)
831            .map(|(spec, fields)| AtomicNoteRequest {
832                spec: AtomicNoteSpec {
833                    token,
834                    id: None,
835                    kind: &spec.note_kind,
836                    name: fields.name.as_deref(),
837                    content: &fields.content,
838                    properties: fields.properties.clone(),
839                },
840                options: AtomicNoteOptions {
841                    salience: fields.salience,
842                    decay_factor: fields.decay_factor,
843                    embed: Some(fields.embed.or(spec.embed).unwrap_or(false)),
844                    embedding_model: spec.embedding_model.as_deref(),
845                    ..Default::default()
846                },
847            })
848            .collect();
849        let prepared = prepare_atomic_note_requests(self, atomic_specs).await?;
850        let mut out = Vec::with_capacity(specs.len());
851        for ((spec, note), plan) in specs.iter().zip(prepared.notes).zip(prepared.plans) {
852            let AtomicOpPlan::AddNote(plan) = plan else {
853                return Err(RuntimeError::Internal(
854                    "stream preparation did not produce a note".into(),
855                ));
856            };
857            let plan = *plan;
858            let mut guard = plan.note_guard.ok_or_else(|| {
859                RuntimeError::Internal("stream preparation did not produce a note guard".into())
860            })?;
861            guard.fence = spec.fence.clone();
862            out.push(PreparedAppend {
863                stream: spec.stream.clone(),
864                expected_seq: spec.expected_seq,
865                note,
866                statements: plan.statements,
867                guard,
868            });
869        }
870        Ok(out)
871    }
872
873    /// Run prepared appends as one writer transaction. Every stream head is
874    /// read, every number assigned in list order and every `expected_seq`
875    /// checked before the first write; then notes and ledger rows land in
876    /// list order, so appends to one stream take consecutive numbers.
877    async fn run_stream_appends(
878        access: &dyn SqlAccess,
879        token: &NamespaceToken,
880        appends: &[PreparedAppend],
881    ) -> Result<BatchOutcome, StreamAppendFailure> {
882        let ns = token.namespace().as_str().to_string();
883        let entries: Vec<_> = appends
884            .iter()
885            .map(|a| {
886                (
887                    a.stream.clone(),
888                    a.expected_seq,
889                    a.note.id.to_string(),
890                    a.statements.clone(),
891                    a.guard.clone(),
892                )
893            })
894            .collect();
895        // A false flag is never proof: the backend may have accepted a closure
896        // that has not run yet. Only a completed pre-write error sets it true.
897        let refused_before_write = Arc::new(AtomicBool::new(false));
898        let refusal_proof = Arc::clone(&refused_before_write);
899        let op: AtomicUnitOp = Box::new(move |writer| {
900            Box::pin(async move {
901                let mut writes_started = false;
902                let result = async {
903                    let mut heads: Vec<(String, i64)> = Vec::new();
904                    for (stream, _, _, _, _) in &entries {
905                        if heads.iter().any(|(known, _)| known == stream) {
906                            continue;
907                        }
908                        let head = writer
909                            .query_scalar(stream_position_statement(&ns, stream))
910                            .await?;
911                        let Some(SqlValue::Integer(head)) = head else {
912                            return Err(write_failure("invalid stream head"));
913                        };
914                        heads.push((stream.clone(), head));
915                    }
916                    let mut assigned = Vec::with_capacity(entries.len());
917                    for (stream, expected_seq, _, _, guard) in &entries {
918                        if let Some(conflict) = guard.check_fence(writer).await? {
919                            return Ok(Box::new(BatchOutcome::FenceConflict { conflict })
920                                as Box<dyn Any + Send>);
921                        }
922                        let head = heads
923                            .iter_mut()
924                            .find(|(known, _)| known == stream)
925                            .map(|(_, head)| head)
926                            .ok_or_else(|| write_failure("stream head missing"))?;
927                        let next = match allocate_sequence(head, *expected_seq) {
928                            Ok(next) => next,
929                            Err(SequenceRefusal::Exhausted) => {
930                                return Ok(Box::new(BatchOutcome::Exhausted) as Box<dyn Any + Send>);
931                            }
932                            Err(SequenceRefusal::Conflict { next, .. }) => {
933                                return Ok(Box::new(BatchOutcome::Conflict { next })
934                                    as Box<dyn Any + Send>);
935                            }
936                        };
937                        assigned.push(next);
938                    }
939                    for ((stream, _, note_id, statements, _), seq) in
940                        entries.into_iter().zip(assigned.iter().copied())
941                    {
942                        // Set before invoking the first write, including a write
943                        // whose driver returns an error with ambiguous effects.
944                        writes_started = true;
945                        for planned in statements {
946                            let affected = writer.execute(planned.statement).await?;
947                            if planned
948                                .guard
949                                .is_some_and(|guard| !guard.holds_for(affected))
950                            {
951                                return Err(write_failure("prepared note write guard failed"));
952                            }
953                        }
954                        insert_stream_entry(writer, &ns, stream, seq, note_id).await?;
955                    }
956                    Ok(Box::new(BatchOutcome::Appended(assigned)) as Box<dyn Any + Send>)
957                }
958                .await;
959                if result.is_err() && !writes_started {
960                    refusal_proof.store(true, Ordering::Release);
961                }
962                result
963            })
964        });
965        let outcome = access
966            .atomic_unit(op)
967            .await
968            .map_err(|source| {
969                StreamAppendFailure::after_submission(
970                    source.into(),
971                    refused_before_write.load(Ordering::Acquire),
972                )
973            })?
974            .downcast::<BatchOutcome>()
975            .map_err(|_| {
976                StreamAppendFailure::unknown(RuntimeError::Internal(
977                    "invalid stream append outcome".into(),
978                ))
979            })?;
980        Ok(*outcome)
981    }
982
983    /// Append a JSON value as an immutable note. The sequence precondition and
984    /// every note/index/ledger statement share the same writer transaction.
985    #[allow(clippy::too_many_arguments)]
986    pub async fn stream_append(
987        &self,
988        token: &NamespaceToken,
989        stream: &str,
990        record: &Value,
991        expected_seq: Option<i64>,
992        note_kind: &str,
993        tags: Option<Vec<String>>,
994        fence: Option<NoteFences>,
995        embed: Option<bool>,
996        embedding_model: Option<String>,
997        registry: &VerbRegistry,
998    ) -> RuntimeResult<Value> {
999        self.stream_append_with_outcome(
1000            token,
1001            stream,
1002            record,
1003            expected_seq,
1004            note_kind,
1005            tags,
1006            fence,
1007            embed,
1008            embedding_model,
1009            registry,
1010        )
1011        .await
1012        .map_err(StreamAppendFailure::into_source)
1013    }
1014
1015    /// Append once while retaining evidence about the failed append's own
1016    /// committedness. The original error remains available for stop semantics
1017    /// or the shared structured error projection; this method never retries.
1018    #[allow(clippy::too_many_arguments)]
1019    pub async fn stream_append_with_outcome(
1020        &self,
1021        token: &NamespaceToken,
1022        stream: &str,
1023        record: &Value,
1024        expected_seq: Option<i64>,
1025        note_kind: &str,
1026        tags: Option<Vec<String>>,
1027        fence: Option<NoteFences>,
1028        embed: Option<bool>,
1029        embedding_model: Option<String>,
1030        registry: &VerbRegistry,
1031    ) -> Result<Value, StreamAppendFailure> {
1032        let spec = StreamAppendSpec {
1033            stream: stream.to_string(),
1034            record: record.clone(),
1035            expected_seq,
1036            note_kind: note_kind.to_string(),
1037            tags,
1038            fence,
1039            embed,
1040            embedding_model,
1041        };
1042        let prepared = self
1043            .prepare_stream_appends(token, &[&spec], registry)
1044            .await
1045            .map_err(StreamAppendFailure::not_committed)?;
1046        let access = self.sql();
1047        match Self::run_stream_appends(access.as_ref(), token, &prepared).await? {
1048            BatchOutcome::Appended(seqs) => Ok(append_result(&prepared[0], seqs[0])),
1049            BatchOutcome::Exhausted => Err(StreamAppendFailure::not_committed(
1050                sequence_exhausted().into(),
1051            )),
1052            BatchOutcome::FenceConflict { conflict } => Err(StreamAppendFailure::not_committed(
1053                conflict.into_error().into(),
1054            )),
1055            BatchOutcome::Conflict { next } => Err(StreamAppendFailure::not_committed(
1056                seq_conflict(
1057                    stream,
1058                    expected_seq.expect("only conditional appends conflict"),
1059                    next,
1060                    None,
1061                )
1062                .into(),
1063            )),
1064        }
1065    }
1066
1067    fn validate_stream_batch(
1068        &self,
1069        members: &[StreamBatchMember],
1070        atomic: bool,
1071    ) -> RuntimeResult<()> {
1072        if members.is_empty() {
1073            return Err(RuntimeError::InvalidInput(
1074                "stream.batch requires at least one member".into(),
1075            ));
1076        }
1077        let mut keys = HashSet::new();
1078        for (index, member) in members.iter().enumerate() {
1079            match member {
1080                StreamBatchMember::Append(spec) => {
1081                    spec.validate_embedding(atomic.then_some(index))?;
1082                    validate_stream(&spec.stream)?;
1083                    self.validate_note_kind(&spec.note_kind)?;
1084                    if let Some(fences) = &spec.fence {
1085                        fences.validate()?;
1086                        for fence in fences.entries() {
1087                            self.validate_note_kind(&fence.kind)?;
1088                        }
1089                    }
1090                    crate::secret_gate::check_at(
1091                        &serde_json::to_string(&spec.record)
1092                            .map_err(|error| RuntimeError::InvalidInput(error.to_string()))?,
1093                        &format!("member[{index}]"),
1094                        "record",
1095                    )?;
1096                }
1097                StreamBatchMember::Write(spec) => {
1098                    self.validate_note_kind(&spec.kind)?;
1099                    NoteWriteOptions {
1100                        key: Some(spec.key.clone()),
1101                        expected_version: spec.expected_version,
1102                        embed: spec.embed,
1103                        fence: None,
1104                    }
1105                    .validate()?;
1106                    if spec.kind == "scheduled_event" {
1107                        return Err(RuntimeError::InvalidInput(
1108                            "scheduled_event notes are not writable through stream.batch; use schedule verbs".into(),
1109                        ));
1110                    }
1111                    if !keys.insert((&spec.kind, &spec.key)) {
1112                        return Err(RuntimeError::InvalidInput(format!(
1113                            "stream.batch repeats write key {:?} for kind {:?}",
1114                            spec.key, spec.kind,
1115                        )));
1116                    }
1117                    crate::secret_gate::check_at(
1118                        &serde_json::to_string(&spec.doc)
1119                            .map_err(|error| RuntimeError::InvalidInput(error.to_string()))?,
1120                        &format!("member[{index}]"),
1121                        "doc",
1122                    )?;
1123                    if let Some(tags) = &spec.tags {
1124                        crate::secret_gate::check_json_at(
1125                            &json!({"tags": tags}),
1126                            &format!("member[{index}]"),
1127                            "tags",
1128                        )?;
1129                    }
1130                }
1131                StreamBatchMember::Refused(_) => {}
1132            }
1133        }
1134        Ok(())
1135    }
1136
1137    async fn prepare_stream_write(
1138        &self,
1139        token: &NamespaceToken,
1140        spec: StreamWriteSpec,
1141        registry: &VerbRegistry,
1142    ) -> RuntimeResult<StreamBatchPreparation> {
1143        let content = serde_json::to_string(&spec.doc)
1144            .map_err(|error| RuntimeError::InvalidInput(error.to_string()))?;
1145        if let Some(expected) = spec.expected_version {
1146            let snapshot = match self
1147                .get_note_by_key(token, &spec.key, Some(&spec.kind), false)
1148                .await
1149            {
1150                Ok(note) => note,
1151                Err(RuntimeError::Khive(error))
1152                    if error.kind() == khive_types::ErrorKind::NotFound =>
1153                {
1154                    return Ok(StreamBatchPreparation::Ready(Box::new(
1155                        PreparedBatchAction::Refused(missing_write(&spec.key)),
1156                    )));
1157                }
1158                Err(error) => return Err(error),
1159            };
1160            let id = snapshot.id;
1161            snapshot
1162                .version
1163                .checked_add(1)
1164                .ok_or_else(|| RuntimeError::InvalidInput("note version exhausted".into()))?;
1165            let mut args = json!({
1166                "id": id, "kind": "note", "note_kind": spec.kind,
1167                "content": content, "expected_version": expected,
1168                "namespace": token.namespace().as_str(),
1169            });
1170            if let Some(tags) = spec.tags {
1171                args["properties"] = json!({"tags": tags});
1172            }
1173            if let Some(embed) = spec.embed {
1174                args["embed"] = json!(embed);
1175            }
1176            let update_policy = registry
1177                .prepare_note_update_policy(self, token, &snapshot, &mut args)
1178                .await?;
1179            let (_, plan) = crate::atomic_prepare::prepare_update_from_note_snapshot(
1180                self,
1181                token,
1182                &args,
1183                None,
1184                snapshot,
1185                update_policy,
1186                registry,
1187            )
1188            .await?;
1189            return Ok(StreamBatchPreparation::Ready(Box::new(
1190                PreparedBatchAction::Write {
1191                    key: spec.key,
1192                    kind: spec.kind,
1193                    id,
1194                    plan,
1195                    after_create: None,
1196                },
1197            )));
1198        }
1199        let mut args = json!({
1200            "kind": "note", "note_kind": spec.kind, "key": spec.key,
1201            "content": content, "namespace": token.namespace().as_str(),
1202        });
1203        if let Some(tags) = spec.tags {
1204            args["tags"] = json!(tags);
1205        }
1206        if let Some(embed) = spec.embed {
1207            args["embed"] = json!(embed);
1208        }
1209        if let Some(hook) = registry.find_kind_hook(&spec.kind) {
1210            hook.prepare_create(self, &mut args).await?;
1211        }
1212        let mut fields: StreamCreateFields = serde_json::from_value(args.clone())
1213            .map_err(|error| RuntimeError::InvalidInput(format!("stream write fields: {error}")))?;
1214        normalize_stream_create_fields(&mut fields)?;
1215        let mut candidate = Note::new(token.namespace().as_str(), &spec.kind, &fields.content);
1216        candidate.name = fields.name.clone();
1217        candidate.properties = fields.properties.clone();
1218        crate::note_write::validate_head(&candidate)?;
1219        Ok(StreamBatchPreparation::Create(Box::new(
1220            PreparedStreamCreate {
1221                key: spec.key,
1222                kind: spec.kind,
1223                fields,
1224                args,
1225            },
1226        )))
1227    }
1228
1229    async fn prepare_stream_batch(
1230        &self,
1231        token: &NamespaceToken,
1232        members: Vec<StreamBatchMember>,
1233        registry: &VerbRegistry,
1234    ) -> RuntimeResult<Vec<PreparedBatchMember>> {
1235        let mut pending = Vec::with_capacity(members.len());
1236        for member in members {
1237            let action = match member {
1238                StreamBatchMember::Refused(error) => {
1239                    StreamBatchPreparation::Ready(Box::new(PreparedBatchAction::Refused(error)))
1240                }
1241                StreamBatchMember::Write(spec) => {
1242                    match self.prepare_stream_write(token, spec, registry).await {
1243                        Ok(action) => action,
1244                        Err(RuntimeError::Khive(error))
1245                            if error.kind() == khive_types::ErrorKind::Conflict =>
1246                        {
1247                            StreamBatchPreparation::Ready(Box::new(PreparedBatchAction::Refused(
1248                                error,
1249                            )))
1250                        }
1251                        Err(error) => return Err(error),
1252                    }
1253                }
1254                StreamBatchMember::Append(spec) => {
1255                    let content = serde_json::to_string(&spec.record)
1256                        .map_err(|error| RuntimeError::InvalidInput(error.to_string()))?;
1257                    StreamBatchPreparation::Append(Box::new((spec, content)))
1258                }
1259            };
1260            pending.push(action);
1261        }
1262        let requests = pending
1263            .iter()
1264            .filter_map(|action| match action {
1265                StreamBatchPreparation::Ready(_) => None,
1266                StreamBatchPreparation::Append(append) => {
1267                    let (spec, content) = append.as_ref();
1268                    Some(AtomicNoteRequest {
1269                        spec: AtomicNoteSpec {
1270                            token,
1271                            id: None,
1272                            kind: &spec.note_kind,
1273                            name: None,
1274                            content,
1275                            properties: spec.tags.clone().map(|tags| json!({"tags": tags})),
1276                        },
1277                        options: spec.note_options(),
1278                    })
1279                }
1280                StreamBatchPreparation::Create(create) => Some(AtomicNoteRequest {
1281                    spec: AtomicNoteSpec {
1282                        token,
1283                        id: None,
1284                        kind: &create.kind,
1285                        name: create.fields.name.as_deref(),
1286                        content: &create.fields.content,
1287                        properties: create.fields.properties.clone(),
1288                    },
1289                    options: AtomicNoteOptions {
1290                        salience: create.fields.salience,
1291                        decay_factor: create.fields.decay_factor,
1292                        key: Some(&create.key),
1293                        embed: Some(create.fields.embed.unwrap_or(create.kind != "head")),
1294                        ..Default::default()
1295                    },
1296                }),
1297            })
1298            .collect();
1299        let notes = prepare_atomic_note_requests(self, requests).await?;
1300        let mut note_plans = notes.notes.into_iter().zip(notes.plans);
1301        let mut prepared = Vec::with_capacity(pending.len());
1302        for (index, pending) in pending.into_iter().enumerate() {
1303            let action = match pending {
1304                StreamBatchPreparation::Ready(action) => *action,
1305                StreamBatchPreparation::Append(append) => {
1306                    let (spec, _) = *append;
1307                    let (note, plan) = note_plans.next().expect("one plan per new note");
1308                    PreparedBatchAction::Append {
1309                        stream: spec.stream,
1310                        expected_seq: spec.expected_seq,
1311                        note: Box::new(note),
1312                        plan,
1313                        fence: spec.fence,
1314                    }
1315                }
1316                StreamBatchPreparation::Create(create) => {
1317                    let (note, mut plan) = note_plans.next().expect("one plan per new note");
1318                    let AtomicOpPlan::AddNote(add) = &mut plan else {
1319                        return Err(RuntimeError::Internal(
1320                            "stream keyed create did not prepare a note".into(),
1321                        ));
1322                    };
1323                    add.post_commit = PostCommitEffect::NoteChanged {
1324                        note_id: note.id,
1325                        kind: note.kind.clone(),
1326                    };
1327                    PreparedBatchAction::Write {
1328                        key: create.key,
1329                        kind: create.kind,
1330                        id: note.id,
1331                        plan,
1332                        after_create: Some(create.args),
1333                    }
1334                }
1335            };
1336            prepared.push(PreparedBatchMember { index, action });
1337        }
1338        Ok(prepared)
1339    }
1340
1341    async fn stream_batch_after_create(
1342        &self,
1343        registry: &VerbRegistry,
1344        hooks: Vec<(Uuid, String, Value)>,
1345    ) {
1346        for (id, kind, args) in hooks {
1347            if let Some(hook) = registry.find_kind_hook(&kind) {
1348                if let Err(error) = hook.after_create(self, id, &args).await {
1349                    tracing::warn!(%id, %kind, %error, "stream batch after_create failed after commit");
1350                }
1351            }
1352        }
1353    }
1354
1355    /// All members share one transaction. Batch predicates precede member DML;
1356    /// effects become executable only after the complete unit commits.
1357    pub async fn stream_batch_atomic(
1358        &self,
1359        token: &NamespaceToken,
1360        members: Vec<StreamBatchMember>,
1361        fence: Option<NoteFence>,
1362        observed: Vec<StreamObservation>,
1363        registry: &VerbRegistry,
1364    ) -> RuntimeResult<Result<Vec<Value>, StreamBatchRefusal>> {
1365        self.validate_stream_batch(&members, true)?;
1366        if let Some(fence) = &fence {
1367            fence.validate()?;
1368            self.validate_note_kind(&fence.kind)?;
1369        }
1370        for entry in &observed {
1371            if entry.id.is_some() && entry.version.is_none() {
1372                return Err(RuntimeError::InvalidInput(
1373                    "observed id requires a positive version".into(),
1374                ));
1375            }
1376            if entry.live_until.is_some() && entry.version.is_none() {
1377                return Err(RuntimeError::InvalidInput(
1378                    "observed live_until requires a positive version".into(),
1379                ));
1380            }
1381            crate::keyed_memory::validate_memory_key(&entry.key)?;
1382            self.validate_note_kind(&entry.kind)?;
1383            if entry.version.is_some_and(|version| version < 1) {
1384                return Err(RuntimeError::InvalidInput(
1385                    "observed version must be positive or null".into(),
1386                ));
1387            }
1388        }
1389        validate_stream_batch_fence_count(stream_batch_fence_count(&members, fence.as_ref()))?;
1390        for (member, item) in members.iter().enumerate() {
1391            if let StreamBatchMember::Refused(error) = item {
1392                return Ok(Err(StreamBatchRefusal {
1393                    member,
1394                    error: place_member(error.clone(), Some(member))?,
1395                }));
1396            }
1397        }
1398        let prepared = self.prepare_stream_batch(token, members, registry).await?;
1399        let hooks = batch_create_hooks(&prepared);
1400        match run_prepared_stream_batch(
1401            self.sql().as_ref(),
1402            token.namespace().as_str().into(),
1403            prepared,
1404            fence,
1405            observed,
1406        )
1407        .await?
1408        {
1409            Ok((results, effects)) => {
1410                crate::atomic_prepare::apply_post_commit_effects_with_report(self, token, effects)
1411                    .await?;
1412                self.stream_batch_after_create(registry, hooks).await;
1413                Ok(Ok(results))
1414            }
1415            Err(refusal) => Ok(Err(refusal)),
1416        }
1417    }
1418
1419    /// One writer transaction per member, in list order. Every member is
1420    /// prepared before the first write; a member's refusal is returned as its
1421    /// value and its siblings stand, so numbers on one stream increase with
1422    /// list position but another writer's append may fall between them.
1423    pub async fn stream_batch_per_member(
1424        &self,
1425        token: &NamespaceToken,
1426        members: Vec<StreamBatchMember>,
1427        registry: &VerbRegistry,
1428    ) -> RuntimeResult<Vec<Value>> {
1429        self.validate_stream_batch(&members, false)?;
1430        validate_stream_batch_fence_count(stream_batch_fence_count(&members, None))?;
1431        let mut results = Vec::with_capacity(members.len());
1432        let prepared = self.prepare_stream_batch(token, members, registry).await?;
1433        for member in prepared {
1434            if let PreparedBatchAction::Refused(error) = &member.action {
1435                results.push(refusal_value(&place_member(error.clone(), None)?)?);
1436                continue;
1437            }
1438            let hooks = batch_create_hooks(std::slice::from_ref(&member));
1439            match run_prepared_stream_batch(
1440                self.sql().as_ref(),
1441                token.namespace().as_str().into(),
1442                vec![member],
1443                None,
1444                vec![],
1445            )
1446            .await?
1447            {
1448                Ok((mut values, effects)) => {
1449                    crate::atomic_prepare::apply_post_commit_effects_with_report(
1450                        self, token, effects,
1451                    )
1452                    .await?;
1453                    self.stream_batch_after_create(registry, hooks).await;
1454                    results.append(&mut values);
1455                }
1456                Err(refusal) => results.push(refusal_value(&place_member(refusal.error, None)?)?),
1457            }
1458        }
1459        Ok(results)
1460    }
1461
1462    /// Read one ordered page and its head from one SQL snapshot.
1463    pub async fn stream_read(
1464        &self,
1465        token: &NamespaceToken,
1466        stream: &str,
1467        after: i64,
1468        limit: i64,
1469    ) -> RuntimeResult<Value> {
1470        validate_stream(stream)?;
1471        if after < 0 || limit < 1 {
1472            return Err(RuntimeError::InvalidInput(
1473                "stream.read requires after >= 0 and limit >= 1".into(),
1474            ));
1475        }
1476        let mut reader = self.sql().reader().await?;
1477        let rows = reader.query_all(SqlStatement::new(
1478            format!("WITH position AS ({STREAM_POSITION_SQL}), \
1479             page AS (SELECT s.seq,n.id,n.content,n.created_at FROM note_streams s JOIN notes n ON n.id=s.note_id \
1480                      CROSS JOIN position WHERE s.namespace=?1 AND s.stream=?2 \
1481                      AND s.seq>CASE WHEN ?3 < position.floor_seq-1 THEN position.floor_seq-1 ELSE ?3 END \
1482                      ORDER BY s.seq LIMIT ?4) \
1483             SELECT position.head_seq,position.floor_seq,page.seq,page.id,page.content,page.created_at \
1484             FROM position LEFT JOIN page ON 1=1 ORDER BY page.seq"),
1485            vec![SqlValue::Text(token.namespace().as_str().into()), SqlValue::Text(stream.into()), SqlValue::Integer(after), SqlValue::Integer(limit)],
1486        ).labelled("stream")).await?;
1487        let head = rows
1488            .first()
1489            .map(|row| {
1490                row.i64("head_seq").map_err(|_| {
1491                    RuntimeError::Internal("stream query missing integer head_seq".to_owned())
1492                })
1493            })
1494            .transpose()?
1495            .unwrap_or(0);
1496        let floor = rows
1497            .first()
1498            .map(|row| row.i64("floor_seq"))
1499            .transpose()
1500            .map_err(|_| RuntimeError::Internal("stream query missing integer floor_seq".into()))?
1501            .unwrap_or(1);
1502        let mut entries = Vec::new();
1503        let mut last = None;
1504        for row in &rows {
1505            if matches!(row.get("seq"), Some(SqlValue::Null)) {
1506                continue;
1507            }
1508            let seq = row.i64("seq").map_err(|_| {
1509                RuntimeError::Internal("stream query missing integer seq".to_owned())
1510            })?;
1511            let record: Value = serde_json::from_str(row.text("content").map_err(|_| {
1512                RuntimeError::Internal("stream query missing text content".to_owned())
1513            })?)
1514            .map_err(|e| RuntimeError::Internal(format!("invalid stream record JSON: {e}")))?;
1515            entries.push(json!({"seq": seq, "id": row.text("id").map_err(|_| RuntimeError::Internal("stream query missing text id".to_owned()))?, "record": record, "created_at": micros_to_iso(row.i64("created_at").map_err(|_| RuntimeError::Internal("stream query missing integer created_at".to_owned()))?)}));
1516            last = Some(seq);
1517        }
1518        Ok(
1519            json!({"entries": entries, "head_seq": head, "floor_seq": floor, "next_after": last.filter(|last| *last < head)}),
1520        )
1521    }
1522
1523    /// Independently count entries and read the head in the same statement.
1524    pub async fn stream_stat(&self, token: &NamespaceToken, stream: &str) -> RuntimeResult<Value> {
1525        validate_stream(stream)?;
1526        let row = self.sql().reader().await?.query_row(SqlStatement::new(
1527            format!("WITH position AS ({STREAM_POSITION_SQL}) \
1528             SELECT position.head_seq, position.floor_seq, \
1529             (SELECT COUNT(*) FROM note_streams WHERE namespace=?1 AND stream=?2) AS count FROM position"),
1530            vec![SqlValue::Text(token.namespace().as_str().into()), SqlValue::Text(stream.into())],
1531        ).labelled("stream")).await?.ok_or_else(|| RuntimeError::Internal("stream.stat returned no aggregate row".into()))?;
1532        Ok(
1533            json!({"head_seq": row.i64("head_seq").map_err(|_| RuntimeError::Internal("stream query missing integer head_seq".to_owned()))?, "floor_seq": row.i64("floor_seq").map_err(|_| RuntimeError::Internal("stream query missing integer floor_seq".to_owned()))?, "count": row.i64("count").map_err(|_| RuntimeError::Internal("stream query missing integer count".to_owned()))?}),
1534        )
1535    }
1536
1537    /// Membership lookup only after the caller has resolved an accessible note.
1538    /// Match the stored namespace as well as its globally unique id.
1539    pub(crate) async fn stream_member_error(
1540        &self,
1541        note: &Note,
1542    ) -> RuntimeResult<Option<RuntimeError>> {
1543        let row = self
1544            .sql()
1545            .reader()
1546            .await?
1547            .query_row(
1548                SqlStatement::new(
1549                    "SELECT stream,seq FROM note_streams WHERE namespace=?1 AND note_id=?2",
1550                    vec![
1551                        SqlValue::Text(note.namespace.clone()),
1552                        SqlValue::Text(note.id.to_string()),
1553                    ],
1554                )
1555                .labelled("stream"),
1556            )
1557            .await?;
1558        row.map(|row| {
1559            Ok(KhiveError::conflict("stream entries are immutable")
1560                .with_details(Details::new_owned([
1561                    ("reason", "stream_member".into()),
1562                    ("id", note.id.to_string()),
1563                    (
1564                        "stream",
1565                        row.text("stream")
1566                            .map_err(|_| {
1567                                RuntimeError::Internal(
1568                                    "stream query missing text stream".to_owned(),
1569                                )
1570                            })?
1571                            .into(),
1572                    ),
1573                    (
1574                        "seq",
1575                        row.i64("seq")
1576                            .map_err(|_| {
1577                                RuntimeError::Internal(
1578                                    "stream query missing integer seq".to_owned(),
1579                                )
1580                            })?
1581                            .to_string(),
1582                    ),
1583                ]))
1584                .into())
1585        })
1586        .transpose()
1587    }
1588}
1589
1590#[cfg(test)]
1591#[path = "streams_batch_tests.rs"]
1592mod batch_tests;
1593
1594#[cfg(test)]
1595#[path = "streams_unit_tests.rs"]
1596mod tests;