Skip to main content

mysql_replication/events/rows/
decoder.rs

1use super::{
2    BinlogMode, ColumnInfo, ColumnType, DecimalValue, Field, IntegerValue, RowImages, RowValue,
3    Rows, RowsEvent, RowsKind, TableDefinition,
4};
5use crate::{Error, Result, error::invalid, events::TableMapEvent, wire::Cursor};
6use std::{iter::FusedIterator, sync::Arc};
7
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9enum ReadOp {
10    Fixed(usize),
11    Prefixed { width: usize, maximum: u64 },
12}
13impl ReadOp {
14    fn read<'a>(self, input: &mut Cursor<'a>) -> Result<&'a [u8]> {
15        let length = match self {
16            Self::Fixed(n) => n,
17            Self::Prefixed { width, maximum } => {
18                let n = input.le(width)?;
19                if n > maximum {
20                    return Err(invalid("column length exceeds TABLE_MAP maximum"));
21                }
22                usize::try_from(n).map_err(|_| invalid("column length overflows usize"))?
23            }
24        };
25        input.take(length)
26    }
27}
28
29/// A validated column layout. SQL names, signedness and character sets are not guessed.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub struct ColumnDefinition {
32    wire_type: ColumnType,
33    column_type: ColumnType,
34    nullable: bool,
35    metadata: [u8; 2],
36    metadata_len: usize,
37    read: ReadOp,
38}
39impl ColumnDefinition {
40    pub fn wire_type(&self) -> ColumnType {
41        self.wire_type
42    }
43    /// Resolves ENUM/SET encoded under MYSQL_TYPE_STRING.
44    pub fn column_type(&self) -> ColumnType {
45        self.column_type
46    }
47    pub fn nullable(&self) -> bool {
48        self.nullable
49    }
50    pub fn metadata(&self) -> &[u8] {
51        &self.metadata[..self.metadata_len]
52    }
53    /// Maximum encoded byte length for a length-prefixed column.
54    pub fn maximum_length(&self) -> Option<u64> {
55        match self.read {
56            ReadOp::Prefixed { maximum, .. } => Some(maximum),
57            ReadOp::Fixed(_) => None,
58        }
59    }
60    fn value<'a>(&self, input: &mut Cursor<'a>) -> Result<RowValue<'a>> {
61        use ColumnType::*;
62        let data = self.read.read(input)?;
63        Ok(match self.column_type {
64            MYSQL_TYPE_TINY | MYSQL_TYPE_SHORT | MYSQL_TYPE_LONG | MYSQL_TYPE_INT24
65            | MYSQL_TYPE_LONGLONG => RowValue::Integer(IntegerValue { data }),
66            MYSQL_TYPE_FLOAT => RowValue::Float(f32::from_le_bytes(
67                data.try_into().map_err(|_| invalid("FLOAT width"))?,
68            )),
69            MYSQL_TYPE_DOUBLE => RowValue::Double(f64::from_le_bytes(
70                data.try_into().map_err(|_| invalid("DOUBLE width"))?,
71            )),
72            MYSQL_TYPE_NEWDECIMAL => {
73                RowValue::Decimal(DecimalValue::new(data, self.metadata[0], self.metadata[1])?)
74            }
75            MYSQL_TYPE_VARCHAR
76            | MYSQL_TYPE_VAR_STRING
77            | MYSQL_TYPE_STRING
78            | MYSQL_TYPE_TINY_BLOB
79            | MYSQL_TYPE_MEDIUM_BLOB
80            | MYSQL_TYPE_LONG_BLOB
81            | MYSQL_TYPE_BLOB => RowValue::Bytes(data),
82            MYSQL_TYPE_BIT => {
83                let bits = usize::from(self.metadata[0]) + usize::from(self.metadata[1]) * 8;
84                if bits % 8 != 0 && data[0] >> (bits % 8) != 0 {
85                    return Err(invalid("nonzero BIT padding"));
86                }
87                RowValue::Bit(data.iter().fold(0, |n, &b| (n << 8) | u64::from(b)))
88            }
89            MYSQL_TYPE_ENUM => RowValue::Enum(Cursor::new(data).le(data.len())? as u16),
90            MYSQL_TYPE_SET => RowValue::Set(Cursor::new(data).le(data.len())?),
91            MYSQL_TYPE_YEAR => RowValue::Year(if data[0] == 0 {
92                0
93            } else {
94                1900 + u16::from(data[0])
95            }),
96            _ => RowValue::Encoded(data),
97        })
98    }
99}
100
101/// Reusable TABLE_MAP plan with historical protocol column metadata.
102/// Typed parsers bind it automatically; manual adapters can also construct it.
103/// It performs no schema queries, filtering or CDC policy checks.
104#[derive(Debug, Clone)]
105pub struct RowsDecoder<M: BinlogMode> {
106    mode: std::marker::PhantomData<M>,
107    table_id: u64,
108    table: Arc<TableDefinition>,
109}
110impl<M: BinlogMode> RowsDecoder<M> {
111    pub fn new(table: &TableMapEvent<'_>) -> Result<Self> {
112        let layout = table.columns()?;
113        validate_bitmap(layout.nullable, layout.count)?;
114        let mut metadata = Cursor::new(layout.metadata);
115        let mut columns = Vec::with_capacity(layout.count);
116        for (index, &ty) in layout.types.iter().enumerate() {
117            columns.push(compile_column(
118                ty,
119                bit(layout.nullable, index),
120                &mut metadata,
121            )?);
122        }
123        metadata.finish()?;
124        Ok(Self {
125            mode: std::marker::PhantomData,
126            table_id: table.table_id,
127            table: Arc::new(TableDefinition::new(
128                table,
129                columns,
130                layout.optional_metadata,
131            )?),
132        })
133    }
134    pub fn table_id(&self) -> u64 {
135        self.table_id
136    }
137    pub fn table(&self) -> &TableDefinition {
138        &self.table
139    }
140    pub fn columns(&self) -> &[ColumnInfo] {
141        &self.table.columns
142    }
143    /// The caller selects the current TABLE_MAP for this source and table ID.
144    /// A remap requires a new decoder, even when the table ID is unchanged.
145    pub fn rows<'a>(&self, event: &RowsEvent<'a, M>) -> Result<Rows<'a, M>> {
146        if event.table_id != self.table_id {
147            return Err(invalid("ROWS table ID differs from TABLE_MAP"));
148        }
149        self.rows_from_images(event.kind, event.images()?)
150    }
151
152    /// Decodes normalized images with this TABLE_MAP and a statically fixed mode.
153    /// Callers adapting a different event layout retain responsibility for table IDs.
154    pub fn rows_from_images<'a>(
155        &self,
156        kind: RowsKind,
157        images: RowImages<'a>,
158    ) -> Result<Rows<'a, M>> {
159        M::validate_images(&images)?;
160        if images.column_count != self.columns().len() {
161            return Err(invalid("ROWS column count differs from TABLE_MAP"));
162        }
163        let first = ImagePlan::new(&self.table, images.before_columns)?;
164        let plans = match kind {
165            RowsKind::Write => super::RowOperation::Insert { after: first },
166            RowsKind::Delete => super::RowOperation::Delete { before: first },
167            RowsKind::Update => super::RowOperation::Update {
168                before: first,
169                after: ImagePlan::new(&self.table, images.after_columns)?,
170            },
171        };
172        if images.rows.is_empty() {
173            return Err(invalid("ROWS event contains no rows"));
174        }
175        Ok(Rows::new(DecodedRows {
176            input: Cursor::new(images.rows),
177            plans,
178            column_count: images.column_count,
179        }))
180    }
181}
182
183#[derive(Debug)]
184struct ImagePlan {
185    table: Arc<TableDefinition>,
186    indices: Arc<[u16]>,
187    columns: Vec<ColumnDefinition>,
188}
189impl ImagePlan {
190    fn new(table: &Arc<TableDefinition>, bitmap: &[u8]) -> Result<Arc<Self>> {
191        let columns = &table.columns;
192        validate_bitmap(bitmap, columns.len())?;
193        let mut indices = Vec::new();
194        let mut present = Vec::new();
195        for (byte_index, &byte) in bitmap.iter().enumerate() {
196            let mut bits = byte;
197            while bits != 0 {
198                let index = byte_index * 8 + bits.trailing_zeros() as usize;
199                if index >= columns.len() {
200                    break;
201                }
202                indices.push(index as u16);
203                present.push(columns[index].definition);
204                bits &= bits - 1;
205            }
206        }
207        if present.is_empty() {
208            return Err(invalid("empty row image"));
209        }
210        Ok(Arc::new(Self {
211            table: Arc::clone(table),
212            indices: indices.into(),
213            columns: present,
214        }))
215    }
216    fn read<'a>(
217        self: &Arc<Self>,
218        input: &mut Cursor<'a>,
219        selected: Option<u16>,
220    ) -> std::result::Result<(RowImage<'a>, Option<RowValue<'a>>), super::RowReadError> {
221        let nulls = input.take(self.columns.len().div_ceil(8))?;
222        let start = input.remaining();
223        let mut selected_value = None;
224        for (i, column) in self.columns.iter().enumerate() {
225            let value = if bit(nulls, i) {
226                if !column.nullable {
227                    return Err(super::RowReadError::UnexpectedNull(self.indices[i]));
228                }
229                RowValue::Null
230            } else {
231                column.value(input)?
232            };
233            if selected.is_some_and(|index| index == self.indices[i]) {
234                selected_value = Some(value);
235            }
236        }
237        Ok((
238            RowImage {
239                plan: Arc::clone(self),
240                nulls,
241                data: &start[..start.len() - input.remaining().len()],
242            },
243            selected_value,
244        ))
245    }
246}
247
248/// An image borrowing the event and sharing its sparse column plan across rows.
249#[derive(Debug, Clone)]
250pub struct RowImage<'a> {
251    plan: Arc<ImagePlan>,
252    nulls: &'a [u8],
253    data: &'a [u8],
254}
255impl<'a> RowImage<'a> {
256    /// Total number of table columns, including columns absent from this image.
257    pub fn column_count(&self) -> usize {
258        self.plan.table.columns.len()
259    }
260    pub fn table(&self) -> &TableDefinition {
261        &self.plan.table
262    }
263    pub fn columns(&self) -> &[ColumnInfo] {
264        &self.plan.table.columns
265    }
266    /// Absolute zero-based table column indices, in wire order.
267    pub fn column_indices(&self) -> &[u16] {
268        &self.plan.indices
269    }
270    /// Fields contain only columns present in this image; bytes remain borrowed.
271    pub fn fields(&self) -> Fields<'_, 'a> {
272        Fields {
273            plan: &self.plan,
274            nulls: self.nulls,
275            input: Cursor::new(self.data),
276            index: 0,
277            done: false,
278        }
279    }
280
281    /// Size of a retained copy, conservatively charging the shared image layout.
282    pub fn retained_bytes(&self) -> usize {
283        retained_size(&self.plan, self.nulls.len() + self.data.len())
284    }
285    /// Copies only this image's compact NULL bitmap and encoded values.
286    /// The immutable historical column plan remains shared across rows.
287    pub fn into_owned(self) -> OwnedRowImage {
288        let null_bytes = self.nulls.len();
289        let mut data = Vec::with_capacity(null_bytes + self.data.len());
290        data.extend_from_slice(self.nulls);
291        data.extend_from_slice(self.data);
292        OwnedRowImage {
293            plan: self.plan,
294            data: data.into_boxed_slice(),
295            null_bytes,
296        }
297    }
298}
299
300/// An owned row image with no self references; values borrow it when accessed.
301#[derive(Debug, Clone)]
302pub struct OwnedRowImage {
303    plan: Arc<ImagePlan>,
304    data: Box<[u8]>,
305    null_bytes: usize,
306}
307impl OwnedRowImage {
308    pub fn view(&self) -> RowImage<'_> {
309        RowImage {
310            plan: Arc::clone(&self.plan),
311            nulls: &self.data[..self.null_bytes],
312            data: &self.data[self.null_bytes..],
313        }
314    }
315    pub fn column_indices(&self) -> &[u16] {
316        &self.plan.indices
317    }
318    pub fn columns(&self) -> &[ColumnInfo] {
319        &self.plan.table.columns
320    }
321    pub fn fields(&self) -> Fields<'_, '_> {
322        Fields {
323            plan: &self.plan,
324            nulls: &self.data[..self.null_bytes],
325            input: Cursor::new(&self.data[self.null_bytes..]),
326            index: 0,
327            done: false,
328        }
329    }
330    /// Charges the buffer and shared image layout conservatively per retained image.
331    /// The TABLE_MAP definition is shared separately and excluded from this count.
332    pub fn retained_bytes(&self) -> usize {
333        retained_size(&self.plan, self.data.len())
334    }
335}
336
337fn retained_size(plan: &ImagePlan, data_len: usize) -> usize {
338    data_len
339        + std::mem::size_of::<ImagePlan>()
340        + 4 * std::mem::size_of::<usize>()
341        + plan.indices.len() * std::mem::size_of::<u16>()
342        + plan.columns.capacity() * std::mem::size_of::<ColumnDefinition>()
343}
344
345/// Streaming fields of a previously validated image.
346pub struct Fields<'image, 'event> {
347    plan: &'image ImagePlan,
348    nulls: &'event [u8],
349    input: Cursor<'event>,
350    index: usize,
351    done: bool,
352}
353impl<'image, 'event> Iterator for Fields<'image, 'event> {
354    type Item = Result<Field<'image, 'event>>;
355    fn next(&mut self) -> Option<Self::Item> {
356        if self.done {
357            return None;
358        }
359        let column = self.plan.columns.get(self.index)?;
360        let value = if bit(self.nulls, self.index) {
361            Ok(RowValue::Null)
362        } else {
363            column.value(&mut self.input)
364        };
365        let result = value.map(|value| Field {
366            column: &self.plan.table.columns[usize::from(self.plan.indices[self.index])],
367            value,
368        });
369        self.index += 1;
370        self.done = result.is_err();
371        Some(result)
372    }
373}
374impl FusedIterator for Fields<'_, '_> {}
375
376pub(crate) struct DecodedRows<'a> {
377    input: Cursor<'a>,
378    plans: super::RowOperation<Arc<ImagePlan>>,
379    column_count: usize,
380}
381
382impl<'a> DecodedRows<'a> {
383    pub(crate) fn read<T>(
384        &mut self,
385        selected: Option<u16>,
386        map: impl Fn(RowImage<'a>, Option<RowValue<'a>>) -> Result<T>,
387    ) -> Option<std::result::Result<super::RowOperation<T>, super::RowReadError>> {
388        if self.input.remaining().is_empty() {
389            return None;
390        }
391        Some((|| {
392            if selected.is_some_and(|i| usize::from(i) >= self.column_count) {
393                return Err(invalid("selected column outside row image").into());
394            }
395            let mut read = |plan: &Arc<ImagePlan>| {
396                let (image, value) = plan.read(&mut self.input, selected)?;
397                map(image, value).map_err(super::RowReadError::from)
398            };
399            use super::RowOperation::*;
400            Ok(match &self.plans {
401                Insert { after } => Insert {
402                    after: read(after)?,
403                },
404                Update { before, after } => Update {
405                    before: read(before)?,
406                    after: read(after)?,
407                },
408                Delete { before } => Delete {
409                    before: read(before)?,
410                },
411            })
412        })())
413    }
414}
415
416fn bit(bitmap: &[u8], index: usize) -> bool {
417    bitmap[index / 8] & (1 << (index % 8)) != 0
418}
419fn validate_bitmap(bitmap: &[u8], count: usize) -> Result<()> {
420    if bitmap.len() != count.div_ceil(8) {
421        return Err(invalid("column bitmap length"));
422    }
423    Ok(())
424}
425
426fn compile_column(ty: u8, nullable: bool, metadata: &mut Cursor<'_>) -> Result<ColumnDefinition> {
427    use ColumnType::*;
428    use ReadOp::*;
429    let wire_type = ColumnType::try_from(ty)
430        .map_err(|_| Error::Unsupported(format!("wire column type {ty}")))?;
431    let start = metadata.remaining();
432    let mut column_type = wire_type;
433    let mut m = [0; 2];
434    let string = |maximum| Prefixed {
435        width: if maximum < 256 { 1 } else { 2 },
436        maximum,
437    };
438    let read = match wire_type {
439        MYSQL_TYPE_TINY | MYSQL_TYPE_YEAR => Fixed(1),
440        MYSQL_TYPE_SHORT => Fixed(2),
441        MYSQL_TYPE_LONG | MYSQL_TYPE_TIMESTAMP => Fixed(4),
442        MYSQL_TYPE_LONGLONG | MYSQL_TYPE_DATETIME => Fixed(8),
443        MYSQL_TYPE_INT24 | MYSQL_TYPE_DATE | MYSQL_TYPE_NEWDATE | MYSQL_TYPE_TIME => Fixed(3),
444        MYSQL_TYPE_FLOAT | MYSQL_TYPE_DOUBLE => {
445            m[0] = metadata.byte()?;
446            let width = if wire_type == MYSQL_TYPE_FLOAT { 4 } else { 8 };
447            if usize::from(m[0]) != width {
448                return Err(invalid("FLOAT/DOUBLE metadata width"));
449            }
450            Fixed(width)
451        }
452        MYSQL_TYPE_VARCHAR | MYSQL_TYPE_VAR_STRING => {
453            m.copy_from_slice(metadata.take(2)?);
454            string(u64::from(u16::from_le_bytes(m)))
455        }
456        MYSQL_TYPE_BIT => {
457            m.copy_from_slice(metadata.take(2)?);
458            let bits = usize::from(m[0]) + usize::from(m[1]) * 8;
459            if m[0] > 7 || bits == 0 || bits > 64 {
460                return Err(invalid("BIT metadata width"));
461            }
462            Fixed(bits.div_ceil(8))
463        }
464        MYSQL_TYPE_TIMESTAMP2 | MYSQL_TYPE_DATETIME2 | MYSQL_TYPE_TIME2 => {
465            m[0] = metadata.byte()?;
466            if m[0] > 6 {
467                return Err(invalid("temporal precision exceeds 6"));
468            }
469            let width = match wire_type {
470                MYSQL_TYPE_TIMESTAMP2 => 4,
471                MYSQL_TYPE_DATETIME2 => 5,
472                _ => 3,
473            };
474            Fixed(width + usize::from(m[0]).div_ceil(2))
475        }
476        MYSQL_TYPE_NEWDECIMAL => {
477            m.copy_from_slice(metadata.take(2)?);
478            if m[0] == 0 || m[0] > 65 || m[1] > m[0] || m[1] > 30 {
479                return Err(invalid("DECIMAL precision/scale"));
480            }
481            let integral = usize::from(m[0] - m[1]);
482            let scale = usize::from(m[1]);
483            Fixed(
484                integral / 9 * 4
485                    + super::value::decimal_group_size(integral % 9)
486                    + scale / 9 * 4
487                    + super::value::decimal_group_size(scale % 9),
488            )
489        }
490        MYSQL_TYPE_ENUM | MYSQL_TYPE_SET | MYSQL_TYPE_STRING => {
491            m.copy_from_slice(metadata.take(2)?);
492            if wire_type == MYSQL_TYPE_STRING {
493                column_type = match m[0] | 0x30 {
494                    247 => MYSQL_TYPE_ENUM,
495                    248 => MYSQL_TYPE_SET,
496                    254 => MYSQL_TYPE_STRING,
497                    _ if m[0] == 0 => MYSQL_TYPE_STRING,
498                    _ => return Err(invalid("invalid STRING real type")),
499                };
500            }
501            if matches!(column_type, MYSQL_TYPE_ENUM | MYSQL_TYPE_SET) {
502                let width = usize::from(m[1]);
503                if width == 0 || width > if column_type == MYSQL_TYPE_ENUM { 2 } else { 8 } {
504                    return Err(invalid("ENUM/SET width"));
505                }
506                Fixed(width)
507            } else {
508                let maximum = if m[0] == 0 {
509                    u64::from(m[1]) << 8
510                } else {
511                    u64::from(m[1]) | (u64::from((m[0] & 0x30) ^ 0x30) << 4)
512                };
513                string(maximum)
514            }
515        }
516        MYSQL_TYPE_JSON
517        | MYSQL_TYPE_TINY_BLOB
518        | MYSQL_TYPE_MEDIUM_BLOB
519        | MYSQL_TYPE_LONG_BLOB
520        | MYSQL_TYPE_BLOB
521        | MYSQL_TYPE_GEOMETRY
522        | MYSQL_TYPE_VECTOR => {
523            m[0] = metadata.byte()?;
524            let width = usize::from(m[0]);
525            if !(1..=4).contains(&width) || wire_type == MYSQL_TYPE_JSON && width != 4 {
526                return Err(invalid("BLOB/JSON length-prefix width"));
527            }
528            Prefixed {
529                width,
530                maximum: (1u64 << (width * 8)) - 1,
531            }
532        }
533        _ => return Err(Error::Unsupported(format!("wire column type {ty}"))),
534    };
535    Ok(ColumnDefinition {
536        wire_type,
537        column_type,
538        nullable,
539        metadata: m,
540        metadata_len: start.len() - metadata.remaining().len(),
541        read,
542    })
543}