Skip to main content

mysql_replication/events/rows/
decoder.rs

1use super::{
2    BinlogMode, ColumnInfo, ColumnType, DecimalValue, Field, IntegerValue, RowImages, RowValue,
3    Rows, RowsEvent, RowsKind, TableDefinition,
4};
5use crate::{Error, Result, error::invalid, events::TableMapEvent, wire::Cursor};
6use std::{iter::FusedIterator, sync::Arc};
7
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9enum ReadOp {
10    Fixed(usize),
11    Prefixed { width: usize, maximum: u64 },
12}
13impl ReadOp {
14    fn read<'a>(self, input: &mut Cursor<'a>) -> Result<&'a [u8]> {
15        let length = match self {
16            Self::Fixed(n) => n,
17            Self::Prefixed { width, maximum } => {
18                let n = input.le(width)?;
19                if n > maximum {
20                    return Err(invalid("column length exceeds TABLE_MAP maximum"));
21                }
22                usize::try_from(n).map_err(|_| invalid("column length overflows usize"))?
23            }
24        };
25        input.take(length)
26    }
27}
28
29/// A validated column layout. SQL names, signedness and character sets are not guessed.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub struct ColumnDefinition {
32    wire_type: ColumnType,
33    column_type: ColumnType,
34    nullable: bool,
35    metadata: [u8; 2],
36    metadata_len: usize,
37    read: ReadOp,
38}
39impl ColumnDefinition {
40    pub fn wire_type(&self) -> ColumnType {
41        self.wire_type
42    }
43    /// Resolves ENUM/SET encoded under MYSQL_TYPE_STRING.
44    pub fn column_type(&self) -> ColumnType {
45        self.column_type
46    }
47    pub fn nullable(&self) -> bool {
48        self.nullable
49    }
50    pub fn metadata(&self) -> &[u8] {
51        &self.metadata[..self.metadata_len]
52    }
53    fn value<'a>(&self, input: &mut Cursor<'a>) -> Result<RowValue<'a>> {
54        use ColumnType::*;
55        let data = self.read.read(input)?;
56        Ok(match self.column_type {
57            MYSQL_TYPE_TINY | MYSQL_TYPE_SHORT | MYSQL_TYPE_LONG | MYSQL_TYPE_INT24
58            | MYSQL_TYPE_LONGLONG => RowValue::Integer(IntegerValue { data }),
59            MYSQL_TYPE_FLOAT => RowValue::Float(f32::from_le_bytes(
60                data.try_into().map_err(|_| invalid("FLOAT width"))?,
61            )),
62            MYSQL_TYPE_DOUBLE => RowValue::Double(f64::from_le_bytes(
63                data.try_into().map_err(|_| invalid("DOUBLE width"))?,
64            )),
65            MYSQL_TYPE_NEWDECIMAL => {
66                RowValue::Decimal(DecimalValue::new(data, self.metadata[0], self.metadata[1])?)
67            }
68            MYSQL_TYPE_VARCHAR
69            | MYSQL_TYPE_VAR_STRING
70            | MYSQL_TYPE_STRING
71            | MYSQL_TYPE_TINY_BLOB
72            | MYSQL_TYPE_MEDIUM_BLOB
73            | MYSQL_TYPE_LONG_BLOB
74            | MYSQL_TYPE_BLOB => RowValue::Bytes(data),
75            MYSQL_TYPE_BIT => {
76                let bits = usize::from(self.metadata[0]) + usize::from(self.metadata[1]) * 8;
77                if bits % 8 != 0 && data[0] >> (bits % 8) != 0 {
78                    return Err(invalid("nonzero BIT padding"));
79                }
80                RowValue::Bit(data.iter().fold(0, |n, &b| (n << 8) | u64::from(b)))
81            }
82            MYSQL_TYPE_ENUM => RowValue::Enum(Cursor::new(data).le(data.len())? as u16),
83            MYSQL_TYPE_SET => RowValue::Set(Cursor::new(data).le(data.len())?),
84            MYSQL_TYPE_YEAR => RowValue::Year(if data[0] == 0 {
85                0
86            } else {
87                1900 + u16::from(data[0])
88            }),
89            _ => RowValue::Encoded(data),
90        })
91    }
92}
93
94/// Reusable TABLE_MAP plan with historical protocol column metadata.
95/// Typed parsers bind it automatically; manual adapters can also construct it.
96/// It performs no schema queries, filtering or CDC policy checks.
97#[derive(Debug, Clone)]
98pub struct RowsDecoder<M: BinlogMode> {
99    mode: std::marker::PhantomData<M>,
100    table_id: u64,
101    table: Arc<TableDefinition>,
102}
103impl<M: BinlogMode> RowsDecoder<M> {
104    pub fn new(table: &TableMapEvent<'_>) -> Result<Self> {
105        let layout = table.columns()?;
106        validate_bitmap(layout.nullable, layout.count)?;
107        let mut metadata = Cursor::new(layout.metadata);
108        let mut columns = Vec::with_capacity(layout.count);
109        for (index, &ty) in layout.types.iter().enumerate() {
110            columns.push(compile_column(
111                ty,
112                bit(layout.nullable, index),
113                &mut metadata,
114            )?);
115        }
116        metadata.finish()?;
117        Ok(Self {
118            mode: std::marker::PhantomData,
119            table_id: table.table_id,
120            table: Arc::new(TableDefinition::new(
121                table,
122                columns,
123                layout.optional_metadata,
124            )?),
125        })
126    }
127    pub fn table_id(&self) -> u64 {
128        self.table_id
129    }
130    pub fn table(&self) -> &TableDefinition {
131        &self.table
132    }
133    pub fn columns(&self) -> &[ColumnInfo] {
134        &self.table.columns
135    }
136    /// The caller selects the current TABLE_MAP for this source and table ID.
137    /// A remap requires a new decoder, even when the table ID is unchanged.
138    pub fn rows<'a>(&self, event: &RowsEvent<'a, M>) -> Result<Rows<'a, M>> {
139        if event.table_id != self.table_id {
140            return Err(invalid("ROWS table ID differs from TABLE_MAP"));
141        }
142        self.rows_from_images(event.kind, event.images()?)
143    }
144
145    /// Decodes normalized images with this TABLE_MAP and a statically fixed mode.
146    /// Callers adapting a different event layout retain responsibility for table IDs.
147    pub fn rows_from_images<'a>(
148        &self,
149        kind: RowsKind,
150        images: RowImages<'a>,
151    ) -> Result<Rows<'a, M>> {
152        M::validate_images(&images)?;
153        if images.column_count != self.columns().len() {
154            return Err(invalid("ROWS column count differs from TABLE_MAP"));
155        }
156        let first = ImagePlan::new(&self.table, images.before_columns)?;
157        let (before, after) = match kind {
158            RowsKind::Write => (None, Some(first)),
159            RowsKind::Delete => (Some(first), None),
160            RowsKind::Update => (
161                Some(first),
162                Some(ImagePlan::new(&self.table, images.after_columns)?),
163            ),
164        };
165        if images.rows.is_empty() {
166            return Err(invalid("ROWS event contains no rows"));
167        }
168        Ok(Rows::new(DecodedRows {
169            input: Cursor::new(images.rows),
170            before,
171            after,
172            done: false,
173        }))
174    }
175}
176
177#[derive(Debug)]
178struct ImagePlan {
179    table: Arc<TableDefinition>,
180    indices: Arc<[u16]>,
181    columns: Vec<ColumnDefinition>,
182}
183impl ImagePlan {
184    fn new(table: &Arc<TableDefinition>, bitmap: &[u8]) -> Result<Arc<Self>> {
185        let columns = &table.columns;
186        validate_bitmap(bitmap, columns.len())?;
187        let mut indices = Vec::new();
188        let mut present = Vec::new();
189        for (byte_index, &byte) in bitmap.iter().enumerate() {
190            let mut bits = byte;
191            while bits != 0 {
192                let index = byte_index * 8 + bits.trailing_zeros() as usize;
193                if index >= columns.len() {
194                    break;
195                }
196                indices.push(index as u16);
197                present.push(columns[index].definition);
198                bits &= bits - 1;
199            }
200        }
201        if present.is_empty() {
202            return Err(invalid("empty row image"));
203        }
204        Ok(Arc::new(Self {
205            table: Arc::clone(table),
206            indices: indices.into(),
207            columns: present,
208        }))
209    }
210    fn read<'a>(self: &Arc<Self>, input: &mut Cursor<'a>) -> Result<RowImage<'a>> {
211        let nulls = input.take(self.columns.len().div_ceil(8))?;
212        let start = input.remaining();
213        for (i, column) in self.columns.iter().enumerate() {
214            if bit(nulls, i) {
215                if !column.nullable {
216                    return Err(invalid("NULL in a NOT NULL column"));
217                }
218            } else {
219                column.value(input)?;
220            }
221        }
222        Ok(RowImage {
223            plan: Arc::clone(self),
224            nulls,
225            data: &start[..start.len() - input.remaining().len()],
226        })
227    }
228}
229
230/// One row operation. Missing images and absent columns are distinct from SQL NULL.
231#[derive(Debug, Clone)]
232pub(crate) struct DecodedRow<'a> {
233    pub before: Option<RowImage<'a>>,
234    pub after: Option<RowImage<'a>>,
235}
236
237/// An image borrowing the event and sharing its sparse column plan across rows.
238#[derive(Debug, Clone)]
239pub struct RowImage<'a> {
240    plan: Arc<ImagePlan>,
241    nulls: &'a [u8],
242    data: &'a [u8],
243}
244impl<'a> RowImage<'a> {
245    /// Total number of table columns, including columns absent from this image.
246    pub fn column_count(&self) -> usize {
247        self.plan.table.columns.len()
248    }
249    pub fn table(&self) -> &TableDefinition {
250        &self.plan.table
251    }
252    pub fn columns(&self) -> &[ColumnInfo] {
253        &self.plan.table.columns
254    }
255    /// Absolute zero-based table column indices, in wire order.
256    pub fn column_indices(&self) -> &[u16] {
257        &self.plan.indices
258    }
259    /// Fields contain only columns present in this image; bytes remain borrowed.
260    pub fn fields(&self) -> Fields<'_, 'a> {
261        Fields {
262            plan: &self.plan,
263            nulls: self.nulls,
264            input: Cursor::new(self.data),
265            index: 0,
266            done: false,
267        }
268    }
269
270    /// Size of a retained copy, conservatively charging the shared image layout.
271    pub fn retained_bytes(&self) -> usize {
272        retained_size(&self.plan, self.nulls.len() + self.data.len())
273    }
274    /// Copies only this image's compact NULL bitmap and encoded values.
275    /// The immutable historical column plan remains shared across rows.
276    pub fn into_owned(self) -> OwnedRowImage {
277        let null_bytes = self.nulls.len();
278        let mut data = Vec::with_capacity(null_bytes + self.data.len());
279        data.extend_from_slice(self.nulls);
280        data.extend_from_slice(self.data);
281        OwnedRowImage {
282            plan: self.plan,
283            data: data.into_boxed_slice(),
284            null_bytes,
285        }
286    }
287}
288
289/// An owned row image with no self references; values borrow it when accessed.
290#[derive(Debug, Clone)]
291pub struct OwnedRowImage {
292    plan: Arc<ImagePlan>,
293    data: Box<[u8]>,
294    null_bytes: usize,
295}
296impl OwnedRowImage {
297    pub fn view(&self) -> RowImage<'_> {
298        RowImage {
299            plan: Arc::clone(&self.plan),
300            nulls: &self.data[..self.null_bytes],
301            data: &self.data[self.null_bytes..],
302        }
303    }
304    pub fn column_indices(&self) -> &[u16] {
305        &self.plan.indices
306    }
307    pub fn columns(&self) -> &[ColumnInfo] {
308        &self.plan.table.columns
309    }
310    pub fn fields(&self) -> Fields<'_, '_> {
311        Fields {
312            plan: &self.plan,
313            nulls: &self.data[..self.null_bytes],
314            input: Cursor::new(&self.data[self.null_bytes..]),
315            index: 0,
316            done: false,
317        }
318    }
319    /// Charges the buffer and shared image layout conservatively per retained image.
320    /// The TABLE_MAP definition is shared separately and excluded from this count.
321    pub fn retained_bytes(&self) -> usize {
322        retained_size(&self.plan, self.data.len())
323    }
324}
325
326fn retained_size(plan: &ImagePlan, data_len: usize) -> usize {
327    data_len
328        + std::mem::size_of::<ImagePlan>()
329        + 4 * std::mem::size_of::<usize>()
330        + plan.indices.len() * std::mem::size_of::<u16>()
331        + plan.columns.capacity() * std::mem::size_of::<ColumnDefinition>()
332}
333
334/// Streaming fields of a previously validated image.
335pub struct Fields<'image, 'event> {
336    plan: &'image ImagePlan,
337    nulls: &'event [u8],
338    input: Cursor<'event>,
339    index: usize,
340    done: bool,
341}
342impl<'image, 'event> Iterator for Fields<'image, 'event> {
343    type Item = Result<Field<'image, 'event>>;
344    fn next(&mut self) -> Option<Self::Item> {
345        if self.done {
346            return None;
347        }
348        let column = self.plan.columns.get(self.index)?;
349        let value = if bit(self.nulls, self.index) {
350            Ok(RowValue::Null)
351        } else {
352            column.value(&mut self.input)
353        };
354        let result = value.map(|value| Field {
355            column: &self.plan.table.columns[usize::from(self.plan.indices[self.index])],
356            value,
357        });
358        self.index += 1;
359        self.done = result.is_err();
360        Some(result)
361    }
362}
363impl FusedIterator for Fields<'_, '_> {}
364
365/// Streaming row operations. An invalid image yields one error, then ends iteration.
366/// Each operation is validated completely before it becomes visible to the caller.
367pub(crate) struct DecodedRows<'a> {
368    input: Cursor<'a>,
369    before: Option<Arc<ImagePlan>>,
370    after: Option<Arc<ImagePlan>>,
371    done: bool,
372}
373impl<'a> Iterator for DecodedRows<'a> {
374    type Item = Result<DecodedRow<'a>>;
375    fn next(&mut self) -> Option<Self::Item> {
376        if self.done || self.input.remaining().is_empty() {
377            return None;
378        }
379        let result = (|| {
380            Ok(DecodedRow {
381                before: self
382                    .before
383                    .as_ref()
384                    .map(|p| p.read(&mut self.input))
385                    .transpose()?,
386                after: self
387                    .after
388                    .as_ref()
389                    .map(|p| p.read(&mut self.input))
390                    .transpose()?,
391            })
392        })();
393        self.done = result.is_err();
394        Some(result)
395    }
396}
397impl FusedIterator for DecodedRows<'_> {}
398
399fn bit(bitmap: &[u8], index: usize) -> bool {
400    bitmap[index / 8] & (1 << (index % 8)) != 0
401}
402fn validate_bitmap(bitmap: &[u8], count: usize) -> Result<()> {
403    if bitmap.len() != count.div_ceil(8) {
404        return Err(invalid("column bitmap length"));
405    }
406    Ok(())
407}
408
409fn compile_column(ty: u8, nullable: bool, metadata: &mut Cursor<'_>) -> Result<ColumnDefinition> {
410    use ColumnType::*;
411    use ReadOp::*;
412    let wire_type = ColumnType::try_from(ty)
413        .map_err(|_| Error::Unsupported(format!("wire column type {ty}")))?;
414    let start = metadata.remaining();
415    let mut column_type = wire_type;
416    let mut m = [0; 2];
417    let string = |maximum| Prefixed {
418        width: if maximum < 256 { 1 } else { 2 },
419        maximum,
420    };
421    let read = match wire_type {
422        MYSQL_TYPE_TINY | MYSQL_TYPE_YEAR => Fixed(1),
423        MYSQL_TYPE_SHORT => Fixed(2),
424        MYSQL_TYPE_LONG | MYSQL_TYPE_TIMESTAMP => Fixed(4),
425        MYSQL_TYPE_LONGLONG | MYSQL_TYPE_DATETIME => Fixed(8),
426        MYSQL_TYPE_INT24 | MYSQL_TYPE_DATE | MYSQL_TYPE_NEWDATE | MYSQL_TYPE_TIME => Fixed(3),
427        MYSQL_TYPE_FLOAT | MYSQL_TYPE_DOUBLE => {
428            m[0] = metadata.byte()?;
429            let width = if wire_type == MYSQL_TYPE_FLOAT { 4 } else { 8 };
430            if usize::from(m[0]) != width {
431                return Err(invalid("FLOAT/DOUBLE metadata width"));
432            }
433            Fixed(width)
434        }
435        MYSQL_TYPE_VARCHAR | MYSQL_TYPE_VAR_STRING => {
436            m.copy_from_slice(metadata.take(2)?);
437            string(u64::from(u16::from_le_bytes(m)))
438        }
439        MYSQL_TYPE_BIT => {
440            m.copy_from_slice(metadata.take(2)?);
441            let bits = usize::from(m[0]) + usize::from(m[1]) * 8;
442            if m[0] > 7 || bits == 0 || bits > 64 {
443                return Err(invalid("BIT metadata width"));
444            }
445            Fixed(bits.div_ceil(8))
446        }
447        MYSQL_TYPE_TIMESTAMP2 | MYSQL_TYPE_DATETIME2 | MYSQL_TYPE_TIME2 => {
448            m[0] = metadata.byte()?;
449            if m[0] > 6 {
450                return Err(invalid("temporal precision exceeds 6"));
451            }
452            let width = match wire_type {
453                MYSQL_TYPE_TIMESTAMP2 => 4,
454                MYSQL_TYPE_DATETIME2 => 5,
455                _ => 3,
456            };
457            Fixed(width + usize::from(m[0]).div_ceil(2))
458        }
459        MYSQL_TYPE_NEWDECIMAL => {
460            m.copy_from_slice(metadata.take(2)?);
461            if m[0] == 0 || m[0] > 65 || m[1] > m[0] || m[1] > 30 {
462                return Err(invalid("DECIMAL precision/scale"));
463            }
464            let integral = usize::from(m[0] - m[1]);
465            let scale = usize::from(m[1]);
466            Fixed(
467                integral / 9 * 4
468                    + super::value::decimal_group_size(integral % 9)
469                    + scale / 9 * 4
470                    + super::value::decimal_group_size(scale % 9),
471            )
472        }
473        MYSQL_TYPE_ENUM | MYSQL_TYPE_SET | MYSQL_TYPE_STRING => {
474            m.copy_from_slice(metadata.take(2)?);
475            if wire_type == MYSQL_TYPE_STRING {
476                column_type = match m[0] | 0x30 {
477                    247 => MYSQL_TYPE_ENUM,
478                    248 => MYSQL_TYPE_SET,
479                    254 => MYSQL_TYPE_STRING,
480                    _ if m[0] == 0 => MYSQL_TYPE_STRING,
481                    _ => return Err(invalid("invalid STRING real type")),
482                };
483            }
484            if matches!(column_type, MYSQL_TYPE_ENUM | MYSQL_TYPE_SET) {
485                let width = usize::from(m[1]);
486                if width == 0 || width > if column_type == MYSQL_TYPE_ENUM { 2 } else { 8 } {
487                    return Err(invalid("ENUM/SET width"));
488                }
489                Fixed(width)
490            } else {
491                let maximum = if m[0] == 0 {
492                    u64::from(m[1]) << 8
493                } else {
494                    u64::from(m[1]) | (u64::from((m[0] & 0x30) ^ 0x30) << 4)
495                };
496                string(maximum)
497            }
498        }
499        MYSQL_TYPE_JSON
500        | MYSQL_TYPE_TINY_BLOB
501        | MYSQL_TYPE_MEDIUM_BLOB
502        | MYSQL_TYPE_LONG_BLOB
503        | MYSQL_TYPE_BLOB
504        | MYSQL_TYPE_GEOMETRY
505        | MYSQL_TYPE_VECTOR => {
506            m[0] = metadata.byte()?;
507            let width = usize::from(m[0]);
508            if !(1..=4).contains(&width) || wire_type == MYSQL_TYPE_JSON && width != 4 {
509                return Err(invalid("BLOB/JSON length-prefix width"));
510            }
511            Prefixed {
512                width,
513                maximum: (1u64 << (width * 8)) - 1,
514            }
515        }
516        _ => return Err(Error::Unsupported(format!("wire column type {ty}"))),
517    };
518    Ok(ColumnDefinition {
519        wire_type,
520        column_type,
521        nullable,
522        metadata: m,
523        metadata_len: start.len() - metadata.remaining().len(),
524        read,
525    })
526}