fakecloud_dynamodb/
streams_dataplane.rs

1//! DynamoDB Streams data plane (`DynamoDBStreams_20120810`).
2//!
3//! Lambda event source mappings against `arn:aws:dynamodb:.../stream/...`
4//! depend on `ListStreams`, `DescribeStream`, `GetShardIterator`, and
5//! `GetRecords`. The control plane's `EnableStream` / `DescribeTable`
6//! already populate `DynamoTable::stream_records` on every mutation;
7//! this module is the consumer side that surfaces those records.
8
9use std::sync::Arc;
10
11use async_trait::async_trait;
12use http::StatusCode;
13use serde_json::{json, Value};
14
15use fakecloud_core::service::{AwsRequest, AwsResponse, AwsService, AwsServiceError};
16
17use crate::state::{DynamoTable, SharedDynamoDbState};
18
19pub struct DynamoDbStreamsService {
20    state: SharedDynamoDbState,
21}
22
23impl DynamoDbStreamsService {
24    pub fn new(state: SharedDynamoDbState) -> Self {
25        Self { state }
26    }
27}
28
29#[async_trait]
30impl AwsService for DynamoDbStreamsService {
31    fn service_name(&self) -> &str {
32        "dynamodbstreams"
33    }
34
35    async fn handle(&self, req: AwsRequest) -> Result<AwsResponse, AwsServiceError> {
36        let body: Value = serde_json::from_slice(&req.body).unwrap_or_default();
37        match req.action.as_str() {
38            "ListStreams" => self.list_streams(&req, &body),
39            "DescribeStream" => self.describe_stream(&req, &body),
40            "GetShardIterator" => self.get_shard_iterator(&req, &body),
41            "GetRecords" => self.get_records(&req, &body),
42            _ => Err(AwsServiceError::action_not_implemented(
43                "dynamodbstreams",
44                &req.action,
45            )),
46        }
47    }
48
49    fn supported_actions(&self) -> &[&str] {
50        &[
51            "ListStreams",
52            "DescribeStream",
53            "GetShardIterator",
54            "GetRecords",
55        ]
56    }
57}
58
59impl DynamoDbStreamsService {
60    fn list_streams(&self, req: &AwsRequest, body: &Value) -> Result<AwsResponse, AwsServiceError> {
61        let table_filter = body["TableName"].as_str();
62        let accounts = self.state.read();
63        let state = match accounts.get(&req.account_id) {
64            Some(s) => s,
65            None => return Ok(AwsResponse::ok_json(json!({ "Streams": [] }))),
66        };
67        let mut streams = Vec::new();
68        for table in state.tables.values() {
69            if let Some(name) = table_filter {
70                if table.name != name {
71                    continue;
72                }
73            }
74            if !table.stream_enabled {
75                continue;
76            }
77            let Some(arn) = table.stream_arn.as_ref() else {
78                continue;
79            };
80            let label = stream_label(arn);
81            streams.push(json!({
82                "StreamArn": arn,
83                "TableName": table.name,
84                "StreamLabel": label,
85            }));
86        }
87        Ok(AwsResponse::ok_json(json!({ "Streams": streams })))
88    }
89
90    fn describe_stream(
91        &self,
92        req: &AwsRequest,
93        body: &Value,
94    ) -> Result<AwsResponse, AwsServiceError> {
95        let stream_arn = require_string(body, "StreamArn")?;
96        let accounts = self.state.read();
97        let state = accounts
98            .get(&req.account_id)
99            .ok_or_else(|| not_found("Stream", &stream_arn))?;
100        let table = state
101            .tables
102            .values()
103            .find(|t| t.stream_arn.as_deref() == Some(stream_arn.as_str()))
104            .ok_or_else(|| not_found("Stream", &stream_arn))?;
105
106        let view_type = table
107            .stream_view_type
108            .clone()
109            .unwrap_or_else(|| "NEW_AND_OLD_IMAGES".to_string());
110        let label = stream_label(&stream_arn);
111        let key_schema: Vec<Value> = table
112            .key_schema
113            .iter()
114            .map(|k| {
115                json!({
116                    "AttributeName": k.attribute_name,
117                    "KeyType": k.key_type,
118                })
119            })
120            .collect();
121
122        let body = json!({
123            "StreamDescription": {
124                "StreamArn": stream_arn,
125                "StreamLabel": label,
126                "StreamStatus": "ENABLED",
127                "StreamViewType": view_type,
128                "CreationRequestDateTime": table.created_at.timestamp() as f64,
129                "TableName": table.name,
130                "KeySchema": key_schema,
131                "Shards": [{
132                    "ShardId": "shardId-00000000000000000000-00000001",
133                    "SequenceNumberRange": {
134                        "StartingSequenceNumber": "0",
135                    },
136                }],
137            }
138        });
139        Ok(AwsResponse::ok_json(body))
140    }
141
142    fn get_shard_iterator(
143        &self,
144        req: &AwsRequest,
145        body: &Value,
146    ) -> Result<AwsResponse, AwsServiceError> {
147        let stream_arn = require_string(body, "StreamArn")?;
148        let shard_id = require_string(body, "ShardId")?;
149        let iterator_type = require_string(body, "ShardIteratorType")?;
150
151        let accounts = self.state.read();
152        let state = accounts
153            .get(&req.account_id)
154            .ok_or_else(|| not_found("Stream", &stream_arn))?;
155        let table = state
156            .tables
157            .values()
158            .find(|t| t.stream_arn.as_deref() == Some(stream_arn.as_str()))
159            .ok_or_else(|| not_found("Stream", &stream_arn))?;
160
161        // The iterator is anchored to an *exclusive start sequence number*,
162        // not a positional index into the records Vec. `add_stream_record`
163        // physically trims expired records off the front (`records.retain`),
164        // which would shift every index; a sequence-number anchor is stable
165        // across trims (like native Kinesis advancing by stored seq, not by
166        // Vec position). GetRecords returns records whose sequence_number is
167        // strictly greater than this anchor. bug-audit 2026-06-15, 4.4.
168        let records = table.stream_records.read();
169        let after_seq: String = match iterator_type.as_str() {
170            // Oldest retained record: anchor below every sequence number so the
171            // first GetRecords returns the whole retained window.
172            "TRIM_HORIZON" => "0".to_string(),
173            // After the newest record: anchor at the max current sequence so
174            // only records added later are returned.
175            "LATEST" => records
176                .iter()
177                .map(|r| r.dynamodb.sequence_number.clone())
178                .max_by(|a, b| cmp_seq(a, b))
179                .unwrap_or_else(|| "0".to_string()),
180            // Inclusive of the named record: anchor just below it.
181            "AT_SEQUENCE_NUMBER" => {
182                let seq = require_string(body, "SequenceNumber")?;
183                if !records.iter().any(|r| r.dynamodb.sequence_number == seq) {
184                    return Err(invalid_argument("SequenceNumber not found"));
185                }
186                exclusive_before(&seq)
187            }
188            // Exclusive of the named record: anchor exactly at it.
189            "AFTER_SEQUENCE_NUMBER" => {
190                let seq = require_string(body, "SequenceNumber")?;
191                if !records.iter().any(|r| r.dynamodb.sequence_number == seq) {
192                    return Err(invalid_argument("SequenceNumber not found"));
193                }
194                seq
195            }
196            other => {
197                return Err(invalid_argument(&format!(
198                    "Unsupported ShardIteratorType: {other}",
199                )))
200            }
201        };
202
203        let token = format!("{stream_arn}|{shard_id}|{after_seq}");
204        Ok(AwsResponse::ok_json(json!({ "ShardIterator": token })))
205    }
206
207    fn get_records(&self, req: &AwsRequest, body: &Value) -> Result<AwsResponse, AwsServiceError> {
208        let iterator = require_string(body, "ShardIterator")?;
209        let limit = body["Limit"].as_u64().unwrap_or(1000) as usize;
210
211        let parts: Vec<&str> = iterator.splitn(3, '|').collect();
212        if parts.len() != 3 {
213            return Err(invalid_argument("ShardIterator is invalid"));
214        }
215        let stream_arn = parts[0].to_string();
216        let shard_id = parts[1].to_string();
217        // Exclusive start sequence number (see get_shard_iterator). Index-based
218        // tokens minted by older builds would parse as a number too, but they
219        // are positions; since we now compare by sequence number this is
220        // self-correcting after one GetShardIterator. bug-audit 2026-06-15, 4.4.
221        let after_seq = parts[2].to_string();
222
223        let accounts = self.state.read();
224        let state = accounts
225            .get(&req.account_id)
226            .ok_or_else(|| not_found("Stream", &stream_arn))?;
227        let table = state
228            .tables
229            .values()
230            .find(|t| t.stream_arn.as_deref() == Some(stream_arn.as_str()))
231            .ok_or_else(|| not_found("Stream", &stream_arn))?;
232
233        // Records whose sequence number is strictly greater than the anchor,
234        // in stored (arrival) order. Front-trimming by `records.retain` cannot
235        // make us skip or replay: the anchor moves only by what we actually
236        // returned, never by physical position.
237        let records = table.stream_records.read();
238        let selected: Vec<&crate::state::StreamRecord> = records
239            .iter()
240            .filter(|r| {
241                cmp_seq(&r.dynamodb.sequence_number, &after_seq) == std::cmp::Ordering::Greater
242            })
243            .take(limit)
244            .collect();
245
246        let next_seq = selected
247            .last()
248            .map(|r| r.dynamodb.sequence_number.clone())
249            .unwrap_or(after_seq);
250        let records_json: Vec<Value> = selected
251            .iter()
252            .map(|r| stream_record_to_json(r, table))
253            .collect();
254
255        let next_token = format!("{stream_arn}|{shard_id}|{next_seq}");
256        Ok(AwsResponse::ok_json(json!({
257            "Records": records_json,
258            "NextShardIterator": next_token,
259        })))
260    }
261}
262
263fn stream_record_to_json(r: &crate::state::StreamRecord, table: &DynamoTable) -> Value {
264    let mut dynamodb = json!({
265        "ApproximateCreationDateTime": r.timestamp.timestamp() as f64,
266        "Keys": &r.dynamodb.keys,
267        "SequenceNumber": r.dynamodb.sequence_number,
268        "SizeBytes": r.dynamodb.size_bytes,
269        "StreamViewType": r.dynamodb.stream_view_type,
270    });
271    if let Some(ni) = r.dynamodb.new_image.as_ref() {
272        dynamodb["NewImage"] = json!(ni);
273    }
274    if let Some(oi) = r.dynamodb.old_image.as_ref() {
275        dynamodb["OldImage"] = json!(oi);
276    }
277    json!({
278        "eventID": r.event_id,
279        "eventName": r.event_name,
280        "eventVersion": r.event_version,
281        "eventSource": r.event_source,
282        "awsRegion": r.aws_region,
283        "eventSourceARN": table.stream_arn.clone().unwrap_or_default(),
284        "dynamodb": dynamodb,
285    })
286}
287
288fn stream_label(stream_arn: &str) -> String {
289    stream_arn.rsplit('/').next().unwrap_or("").to_string()
290}
291
292/// Compare two DynamoDB stream sequence numbers numerically. Sequence numbers
293/// are minted by an atomic counter and zero-padded to a fixed width, so they
294/// are also lexicographically ordered; we still parse to `u128` so that an
295/// un-padded legacy value (or one of a different width) compares correctly.
296fn cmp_seq(a: &str, b: &str) -> std::cmp::Ordering {
297    match (a.parse::<u128>(), b.parse::<u128>()) {
298        (Ok(x), Ok(y)) => x.cmp(&y),
299        // Non-numeric values fall back to byte order (deterministic, total).
300        _ => a.cmp(b),
301    }
302}
303
304/// The largest sequence number strictly less than `seq`, used to build an
305/// exclusive anchor for `AT_SEQUENCE_NUMBER` (which is inclusive of the named
306/// record). For the numeric counter this is `seq - 1`; if `seq` is `0` or
307/// non-numeric we anchor at `"0"` so nothing earlier is skipped.
308fn exclusive_before(seq: &str) -> String {
309    match seq.parse::<u128>() {
310        Ok(n) if n > 0 => {
311            // Preserve the original zero-padded width so lexicographic order
312            // continues to match numeric order for downstream string compares.
313            format!("{:0width$}", n - 1, width = seq.len())
314        }
315        _ => "0".to_string(),
316    }
317}
318
319fn require_string(body: &Value, field: &str) -> Result<String, AwsServiceError> {
320    body[field]
321        .as_str()
322        .map(|s| s.to_string())
323        .ok_or_else(|| invalid_argument(&format!("{field} is required")))
324}
325
326fn invalid_argument(msg: &str) -> AwsServiceError {
327    AwsServiceError::aws_error(StatusCode::BAD_REQUEST, "ValidationException", msg)
328}
329
330fn not_found(kind: &str, target: &str) -> AwsServiceError {
331    AwsServiceError::aws_error(
332        StatusCode::BAD_REQUEST,
333        "ResourceNotFoundException",
334        format!("{kind} not found: {target}"),
335    )
336}
337
338pub fn shared(state: SharedDynamoDbState) -> Arc<dyn AwsService> {
339    Arc::new(DynamoDbStreamsService::new(state))
340}
341
342#[cfg(test)]
343mod tests {
344    use super::*;
345    use crate::state::{DynamoDbStreamRecord, DynamoTable, ProvisionedThroughput, StreamRecord};
346    use bytes::Bytes;
347    use chrono::Utc;
348    use http::{HeaderMap, Method};
349    use parking_lot::RwLock;
350    use std::collections::{BTreeMap, HashMap};
351    use std::sync::Arc;
352
353    fn make_state() -> SharedDynamoDbState {
354        Arc::new(RwLock::new(
355            fakecloud_core::multi_account::MultiAccountState::new("123456789012", "us-east-1", ""),
356        ))
357    }
358
359    fn req(action: &str, body: Value) -> AwsRequest {
360        AwsRequest {
361            service: "dynamodbstreams".into(),
362            action: action.into(),
363            region: "us-east-1".into(),
364            account_id: "123456789012".into(),
365            request_id: "r".into(),
366            headers: HeaderMap::new(),
367            query_params: HashMap::new(),
368            body: Bytes::from(serde_json::to_vec(&body).unwrap()),
369            body_stream: parking_lot::Mutex::new(None),
370            path_segments: vec![],
371            raw_path: "/".into(),
372            raw_query: String::new(),
373            method: Method::POST,
374            is_query_protocol: false,
375            access_key_id: None,
376            principal: None,
377        }
378    }
379
380    fn seed_table(state: &SharedDynamoDbState) -> String {
381        let mut accts = state.write();
382        let s = accts.get_or_create("123456789012");
383        let arn =
384            "arn:aws:dynamodb:us-east-1:123456789012:table/widgets/stream/2026-05-03T00:00:00.000"
385                .to_string();
386        let table = DynamoTable {
387            name: "widgets".to_string(),
388            arn: "arn:aws:dynamodb:us-east-1:123456789012:table/widgets".to_string(),
389            table_id: "id".to_string(),
390            key_schema: Vec::new(),
391            attribute_definitions: Vec::new(),
392            provisioned_throughput: ProvisionedThroughput {
393                read_capacity_units: 0,
394                write_capacity_units: 0,
395            },
396            items: Vec::new(),
397            gsi: Vec::new(),
398            lsi: Vec::new(),
399            tags: BTreeMap::new(),
400            created_at: Utc::now(),
401            status: "ACTIVE".to_string(),
402            item_count: 0,
403            size_bytes: 0,
404            billing_mode: "PAY_PER_REQUEST".to_string(),
405            ttl_attribute: None,
406            ttl_enabled: false,
407            resource_policy: None,
408            pitr_enabled: false,
409            kinesis_destinations: Vec::new(),
410            contributor_insights_status: "DISABLED".to_string(),
411            contributor_insights_counters: BTreeMap::new(),
412            stream_enabled: true,
413            stream_view_type: Some("NEW_AND_OLD_IMAGES".to_string()),
414            stream_arn: Some(arn.clone()),
415            stream_records: Arc::new(RwLock::new(Vec::new())),
416            sse_type: None,
417            sse_kms_key_arn: None,
418            deletion_protection_enabled: false,
419            on_demand_throughput: None,
420        };
421        let rec = StreamRecord {
422            event_id: "e1".into(),
423            event_name: "INSERT".into(),
424            event_version: "1.1".into(),
425            event_source: "aws:dynamodb".into(),
426            aws_region: "us-east-1".into(),
427            event_source_arn: arn.clone(),
428            timestamp: Utc::now(),
429            dynamodb: DynamoDbStreamRecord {
430                keys: HashMap::new(),
431                new_image: Some(HashMap::new()),
432                old_image: None,
433                sequence_number: "1".into(),
434                size_bytes: 16,
435                stream_view_type: "NEW_AND_OLD_IMAGES".into(),
436            },
437        };
438        table.stream_records.write().push(rec);
439        s.tables.insert("widgets".to_string(), table);
440        arn
441    }
442
443    #[tokio::test]
444    async fn list_streams_returns_enabled_streams() {
445        let state = make_state();
446        let arn = seed_table(&state);
447        let svc = DynamoDbStreamsService::new(state);
448        let resp = svc.handle(req("ListStreams", json!({}))).await.unwrap();
449        let body: Value = serde_json::from_slice(resp.body.expect_bytes()).unwrap();
450        let streams = body["Streams"].as_array().unwrap();
451        assert_eq!(streams.len(), 1);
452        assert_eq!(streams[0]["StreamArn"].as_str().unwrap(), arn);
453    }
454
455    #[tokio::test]
456    async fn describe_stream_returns_shard() {
457        let state = make_state();
458        let arn = seed_table(&state);
459        let svc = DynamoDbStreamsService::new(state);
460        let resp = svc
461            .handle(req("DescribeStream", json!({"StreamArn": arn})))
462            .await
463            .unwrap();
464        let body: Value = serde_json::from_slice(resp.body.expect_bytes()).unwrap();
465        let desc = &body["StreamDescription"];
466        assert_eq!(desc["StreamStatus"].as_str().unwrap(), "ENABLED");
467        assert_eq!(desc["Shards"].as_array().unwrap().len(), 1);
468    }
469
470    #[tokio::test]
471    async fn get_records_round_trip_via_shard_iterator() {
472        let state = make_state();
473        let arn = seed_table(&state);
474        let svc = DynamoDbStreamsService::new(state);
475        let it_resp = svc
476            .handle(req(
477                "GetShardIterator",
478                json!({
479                    "StreamArn": arn,
480                    "ShardId": "shardId-00000000000000000000-00000001",
481                    "ShardIteratorType": "TRIM_HORIZON",
482                }),
483            ))
484            .await
485            .unwrap();
486        let it_body: Value = serde_json::from_slice(it_resp.body.expect_bytes()).unwrap();
487        let iterator = it_body["ShardIterator"].as_str().unwrap().to_string();
488        let resp = svc
489            .handle(req("GetRecords", json!({"ShardIterator": iterator})))
490            .await
491            .unwrap();
492        let body: Value = serde_json::from_slice(resp.body.expect_bytes()).unwrap();
493        let recs = body["Records"].as_array().unwrap();
494        assert_eq!(recs.len(), 1);
495        assert_eq!(recs[0]["eventName"].as_str().unwrap(), "INSERT");
496    }
497
498    fn push_record(state: &SharedDynamoDbState, seq: &str, age_hours: i64, event_id: &str) {
499        let mut accts = state.write();
500        let s = accts.get_or_create("123456789012");
501        let table = s.tables.get_mut("widgets").unwrap();
502        let rec = StreamRecord {
503            event_id: event_id.into(),
504            event_name: "INSERT".into(),
505            event_version: "1.1".into(),
506            event_source: "aws:dynamodb".into(),
507            aws_region: "us-east-1".into(),
508            event_source_arn: table.stream_arn.clone().unwrap(),
509            timestamp: Utc::now() - chrono::Duration::hours(age_hours),
510            dynamodb: DynamoDbStreamRecord {
511                keys: HashMap::new(),
512                new_image: Some(HashMap::new()),
513                old_image: None,
514                sequence_number: seq.into(),
515                size_bytes: 16,
516                stream_view_type: "NEW_AND_OLD_IMAGES".into(),
517            },
518        };
519        table.stream_records.write().push(rec);
520    }
521
522    fn trim_front(state: &SharedDynamoDbState, n: usize) {
523        let accts = state.read();
524        let s = accts.get("123456789012").unwrap();
525        let table = s.tables.get("widgets").unwrap();
526        let mut recs = table.stream_records.write();
527        for _ in 0..n {
528            if !recs.is_empty() {
529                recs.remove(0);
530            }
531        }
532    }
533
534    // bug-audit 2026-06-15, 4.4: the iterator is anchored to a sequence number,
535    // not a Vec index. A consumer that has read up to record N must, after the
536    // front of the records Vec is physically trimmed (24h retention), continue
537    // exactly where it left off — no skipped or replayed records.
538    #[tokio::test]
539    async fn iterator_survives_front_trim_without_skip_or_replay() {
540        let state = make_state();
541        let arn = seed_table(&state); // seeds one record seq "1"
542                                      // Replace the seeded record set with a clean, ordered set seq 1..=5.
543        {
544            let accts = state.read();
545            let s = accts.get("123456789012").unwrap();
546            s.tables
547                .get("widgets")
548                .unwrap()
549                .stream_records
550                .write()
551                .clear();
552        }
553        for i in 1..=5u64 {
554            // First two records are aged so a later trim removes them.
555            let age = if i <= 2 { 30 } else { 0 };
556            push_record(&state, &format!("{i:021}"), age, &format!("e{i}"));
557        }
558        let svc = DynamoDbStreamsService::new(state.clone());
559
560        // Start at TRIM_HORIZON, read 3 records (seq 1,2,3).
561        let it_resp = svc
562            .handle(req(
563                "GetShardIterator",
564                json!({
565                    "StreamArn": arn,
566                    "ShardId": "shardId-00000000000000000000-00000001",
567                    "ShardIteratorType": "TRIM_HORIZON",
568                }),
569            ))
570            .await
571            .unwrap();
572        let it: Value = serde_json::from_slice(it_resp.body.expect_bytes()).unwrap();
573        let iterator = it["ShardIterator"].as_str().unwrap().to_string();
574
575        let r1 = svc
576            .handle(req(
577                "GetRecords",
578                json!({"ShardIterator": iterator, "Limit": 3}),
579            ))
580            .await
581            .unwrap();
582        let b1: Value = serde_json::from_slice(r1.body.expect_bytes()).unwrap();
583        let recs1 = b1["Records"].as_array().unwrap();
584        assert_eq!(recs1.len(), 3);
585        assert_eq!(recs1[0]["eventID"].as_str().unwrap(), "e1");
586        assert_eq!(recs1[2]["eventID"].as_str().unwrap(), "e3");
587        let next = b1["NextShardIterator"].as_str().unwrap().to_string();
588
589        // Now retention trims the two aged front records (seq 1,2) off the Vec.
590        // An index-based iterator would now mis-resolve and skip/replay; a
591        // sequence-anchored one continues correctly with seq 4,5.
592        trim_front(&state, 2);
593
594        let r2 = svc
595            .handle(req("GetRecords", json!({"ShardIterator": next})))
596            .await
597            .unwrap();
598        let b2: Value = serde_json::from_slice(r2.body.expect_bytes()).unwrap();
599        let recs2 = b2["Records"].as_array().unwrap();
600        assert_eq!(
601            recs2.len(),
602            2,
603            "must return exactly the un-consumed records after a front trim"
604        );
605        assert_eq!(recs2[0]["eventID"].as_str().unwrap(), "e4");
606        assert_eq!(recs2[1]["eventID"].as_str().unwrap(), "e5");
607    }
608
609    #[tokio::test]
610    async fn describe_stream_unknown_arn_404s() {
611        let state = make_state();
612        let _ = seed_table(&state);
613        let svc = DynamoDbStreamsService::new(state);
614        let err = svc
615            .handle(req(
616                "DescribeStream",
617                json!({"StreamArn": "arn:aws:dynamodb:us-east-1:123456789012:table/missing/stream/x"}),
618            ))
619            .await
620            .err()
621            .expect("expected ResourceNotFound");
622        assert!(format!("{:?}", err).contains("ResourceNotFoundException"));
623    }
624}
fakecloud_dynamodb/streams_dataplane.rs

fakecloud_dynamodb/
streams_dataplane.rs
