Skip to main content

mysql_replication/events/rows/
decoder.rs

1use super::{ColumnType, DecimalValue, IntegerValue, RowValue, RowsEvent, RowsKind};
2use crate::{Error, Result, error::invalid, events::TableMapEvent, wire::Cursor};
3use std::{iter::FusedIterator, sync::Arc};
4
5#[derive(Debug, Clone, Copy)]
6enum ReadOp {
7    Fixed(usize),
8    Prefixed { width: usize, maximum: u64 },
9}
10impl ReadOp {
11    fn read<'a>(self, input: &mut Cursor<'a>) -> Result<&'a [u8]> {
12        let length = match self {
13            Self::Fixed(n) => n,
14            Self::Prefixed { width, maximum } => {
15                let n = input.le(width)?;
16                if n > maximum {
17                    return Err(invalid("column length exceeds TABLE_MAP maximum"));
18                }
19                usize::try_from(n).map_err(|_| invalid("column length overflows usize"))?
20            }
21        };
22        input.take(length)
23    }
24}
25
26/// A validated column layout. SQL names, signedness and character sets are not guessed.
27#[derive(Debug, Clone, Copy)]
28pub struct ColumnDefinition {
29    wire_type: ColumnType,
30    column_type: ColumnType,
31    nullable: bool,
32    metadata: [u8; 2],
33    metadata_len: usize,
34    read: ReadOp,
35}
36impl ColumnDefinition {
37    pub fn wire_type(&self) -> ColumnType {
38        self.wire_type
39    }
40    /// Resolves ENUM/SET encoded under MYSQL_TYPE_STRING.
41    pub fn column_type(&self) -> ColumnType {
42        self.column_type
43    }
44    pub fn nullable(&self) -> bool {
45        self.nullable
46    }
47    pub fn metadata(&self) -> &[u8] {
48        &self.metadata[..self.metadata_len]
49    }
50    fn value<'a>(&self, input: &mut Cursor<'a>) -> Result<RowValue<'a>> {
51        use ColumnType::*;
52        let data = self.read.read(input)?;
53        Ok(match self.column_type {
54            MYSQL_TYPE_TINY | MYSQL_TYPE_SHORT | MYSQL_TYPE_LONG | MYSQL_TYPE_INT24
55            | MYSQL_TYPE_LONGLONG => RowValue::Integer(IntegerValue { data }),
56            MYSQL_TYPE_FLOAT => RowValue::Float(f32::from_le_bytes(
57                data.try_into().map_err(|_| invalid("FLOAT width"))?,
58            )),
59            MYSQL_TYPE_DOUBLE => RowValue::Double(f64::from_le_bytes(
60                data.try_into().map_err(|_| invalid("DOUBLE width"))?,
61            )),
62            MYSQL_TYPE_NEWDECIMAL => {
63                RowValue::Decimal(DecimalValue::new(data, self.metadata[0], self.metadata[1])?)
64            }
65            MYSQL_TYPE_VARCHAR
66            | MYSQL_TYPE_VAR_STRING
67            | MYSQL_TYPE_STRING
68            | MYSQL_TYPE_TINY_BLOB
69            | MYSQL_TYPE_MEDIUM_BLOB
70            | MYSQL_TYPE_LONG_BLOB
71            | MYSQL_TYPE_BLOB => RowValue::Bytes(data),
72            MYSQL_TYPE_BIT => {
73                let bits = usize::from(self.metadata[0]) + usize::from(self.metadata[1]) * 8;
74                if bits % 8 != 0 && data[0] >> (bits % 8) != 0 {
75                    return Err(invalid("nonzero BIT padding"));
76                }
77                RowValue::Bit(data.iter().fold(0, |n, &b| (n << 8) | u64::from(b)))
78            }
79            MYSQL_TYPE_ENUM => RowValue::Enum(Cursor::new(data).le(data.len())? as u16),
80            MYSQL_TYPE_SET => RowValue::Set(Cursor::new(data).le(data.len())?),
81            MYSQL_TYPE_YEAR => RowValue::Year(if data[0] == 0 {
82                0
83            } else {
84                1900 + u16::from(data[0])
85            }),
86            _ => RowValue::Encoded(data),
87        })
88    }
89}
90
91/// Reusable TABLE_MAP plan for opt-in raw row decoding.
92/// This owns only column layouts, with no schema queries, filtering or CDC policy.
93#[derive(Debug)]
94pub struct RowsDecoder {
95    table_id: u64,
96    columns: Vec<ColumnDefinition>,
97}
98impl RowsDecoder {
99    pub fn new(table: &TableMapEvent<'_>) -> Result<Self> {
100        let layout = table.columns()?;
101        validate_bitmap(layout.nullable, layout.count)?;
102        let mut metadata = Cursor::new(layout.metadata);
103        let mut columns = Vec::with_capacity(layout.count);
104        for (index, &ty) in layout.types.iter().enumerate() {
105            columns.push(compile_column(
106                ty,
107                bit(layout.nullable, index),
108                &mut metadata,
109            )?);
110        }
111        metadata.finish()?;
112        // Unknown optional TLVs remain opaque, but their envelopes must be complete.
113        let mut optional = Cursor::new(layout.optional_metadata);
114        while !optional.remaining().is_empty() {
115            optional.byte()?;
116            let length = optional.lenenc()?;
117            optional.take(length)?;
118        }
119        Ok(Self {
120            table_id: table.table_id,
121            columns,
122        })
123    }
124    pub fn table_id(&self) -> u64 {
125        self.table_id
126    }
127    pub fn columns(&self) -> &[ColumnDefinition] {
128        &self.columns
129    }
130    /// The caller selects the current TABLE_MAP for this source and table ID.
131    /// A remap requires a new decoder, even when the table ID is unchanged.
132    pub fn rows<'a>(&self, event: &RowsEvent<'a>) -> Result<Rows<'a>> {
133        if event.table_id != self.table_id {
134            return Err(invalid("ROWS table ID differs from TABLE_MAP"));
135        }
136        let images = event.images()?;
137        if images.column_count != self.columns.len() {
138            return Err(invalid("ROWS column count differs from TABLE_MAP"));
139        }
140        let first = ImagePlan::new(&self.columns, images.before_columns)?;
141        let (before, after) = match event.kind {
142            RowsKind::Write => (None, Some(first)),
143            RowsKind::Delete => (Some(first), None),
144            RowsKind::Update => (
145                Some(first),
146                Some(ImagePlan::new(&self.columns, images.after_columns)?),
147            ),
148        };
149        if images.rows.is_empty() {
150            return Err(invalid("ROWS event contains no rows"));
151        }
152        Ok(Rows {
153            input: Cursor::new(images.rows),
154            before,
155            after,
156            done: false,
157        })
158    }
159}
160
161#[derive(Debug)]
162struct ImagePlan {
163    column_count: usize,
164    indices: Arc<[u16]>,
165    columns: Vec<ColumnDefinition>,
166}
167impl ImagePlan {
168    fn new(columns: &[ColumnDefinition], bitmap: &[u8]) -> Result<Arc<Self>> {
169        validate_bitmap(bitmap, columns.len())?;
170        let mut indices = Vec::new();
171        let mut present = Vec::new();
172        for (byte_index, &byte) in bitmap.iter().enumerate() {
173            let mut bits = byte;
174            while bits != 0 {
175                let index = byte_index * 8 + bits.trailing_zeros() as usize;
176                if index >= columns.len() {
177                    break;
178                }
179                indices.push(index as u16);
180                present.push(columns[index]);
181                bits &= bits - 1;
182            }
183        }
184        if present.is_empty() {
185            return Err(invalid("empty row image"));
186        }
187        Ok(Arc::new(Self {
188            column_count: columns.len(),
189            indices: indices.into(),
190            columns: present,
191        }))
192    }
193    fn read<'a>(self: &Arc<Self>, input: &mut Cursor<'a>) -> Result<RowImage<'a>> {
194        let nulls = input.take(self.columns.len().div_ceil(8))?;
195        let start = input.remaining();
196        for (i, column) in self.columns.iter().enumerate() {
197            if bit(nulls, i) {
198                if !column.nullable {
199                    return Err(invalid("NULL in a NOT NULL column"));
200                }
201            } else {
202                column.value(input)?;
203            }
204        }
205        Ok(RowImage {
206            plan: Arc::clone(self),
207            nulls,
208            data: &start[..start.len() - input.remaining().len()],
209        })
210    }
211}
212
213/// One row operation. Missing images and absent columns are distinct from SQL NULL.
214#[derive(Debug, Clone)]
215pub struct Row<'a> {
216    pub before: Option<RowImage<'a>>,
217    pub after: Option<RowImage<'a>>,
218}
219
220/// An image borrowing the event and sharing its sparse column plan across rows.
221#[derive(Debug, Clone)]
222pub struct RowImage<'a> {
223    plan: Arc<ImagePlan>,
224    nulls: &'a [u8],
225    data: &'a [u8],
226}
227impl<'a> RowImage<'a> {
228    /// Total number of table columns, including columns absent from this image.
229    pub fn column_count(&self) -> usize {
230        self.plan.column_count
231    }
232    /// Absolute zero-based table column indices, in wire order.
233    pub fn column_indices(&self) -> &[u16] {
234        &self.plan.indices
235    }
236    /// Fields contain only columns present in this image; bytes remain borrowed.
237    pub fn fields(&self) -> RowFields<'_, 'a> {
238        RowFields {
239            image: self,
240            input: Cursor::new(self.data),
241            index: 0,
242            done: false,
243        }
244    }
245}
246
247#[derive(Debug, Clone, Copy, PartialEq)]
248pub struct RowField<'a> {
249    pub column_index: u16,
250    pub value: RowValue<'a>,
251}
252
253/// Streaming fields of a previously validated image.
254pub struct RowFields<'image, 'event> {
255    image: &'image RowImage<'event>,
256    input: Cursor<'event>,
257    index: usize,
258    done: bool,
259}
260impl<'event> Iterator for RowFields<'_, 'event> {
261    type Item = Result<RowField<'event>>;
262    fn next(&mut self) -> Option<Self::Item> {
263        if self.done {
264            return None;
265        }
266        let column = self.image.plan.columns.get(self.index)?;
267        let value = if bit(self.image.nulls, self.index) {
268            Ok(RowValue::Null)
269        } else {
270            column.value(&mut self.input)
271        };
272        let result = value.map(|value| RowField {
273            column_index: self.image.plan.indices[self.index],
274            value,
275        });
276        self.index += 1;
277        self.done = result.is_err();
278        Some(result)
279    }
280}
281impl FusedIterator for RowFields<'_, '_> {}
282
283/// Streaming row operations. An invalid image yields one error, then ends iteration.
284/// Each operation is validated completely before it becomes visible to the caller.
285pub struct Rows<'a> {
286    input: Cursor<'a>,
287    before: Option<Arc<ImagePlan>>,
288    after: Option<Arc<ImagePlan>>,
289    done: bool,
290}
291impl<'a> Iterator for Rows<'a> {
292    type Item = Result<Row<'a>>;
293    fn next(&mut self) -> Option<Self::Item> {
294        if self.done || self.input.remaining().is_empty() {
295            return None;
296        }
297        let result = (|| {
298            Ok(Row {
299                before: self
300                    .before
301                    .as_ref()
302                    .map(|p| p.read(&mut self.input))
303                    .transpose()?,
304                after: self
305                    .after
306                    .as_ref()
307                    .map(|p| p.read(&mut self.input))
308                    .transpose()?,
309            })
310        })();
311        self.done = result.is_err();
312        Some(result)
313    }
314}
315impl FusedIterator for Rows<'_> {}
316
317impl Rows<'_> {
318    pub(super) fn require_complete(&self) -> Result<()> {
319        for plan in self.before.iter().chain(&self.after) {
320            if plan.indices.len() != plan.column_count {
321                return Err(Error::Unsupported(
322                    "FULL row images must contain every table column".into(),
323                ));
324            }
325        }
326        Ok(())
327    }
328}
329
330fn bit(bitmap: &[u8], index: usize) -> bool {
331    bitmap[index / 8] & (1 << (index % 8)) != 0
332}
333fn validate_bitmap(bitmap: &[u8], count: usize) -> Result<()> {
334    if bitmap.len() != count.div_ceil(8) {
335        return Err(invalid("column bitmap length"));
336    }
337    Ok(())
338}
339
340fn compile_column(ty: u8, nullable: bool, metadata: &mut Cursor<'_>) -> Result<ColumnDefinition> {
341    use ColumnType::*;
342    use ReadOp::*;
343    let wire_type = ColumnType::try_from(ty)
344        .map_err(|_| Error::Unsupported(format!("wire column type {ty}")))?;
345    let start = metadata.remaining();
346    let mut column_type = wire_type;
347    let mut m = [0; 2];
348    let string = |maximum| Prefixed {
349        width: if maximum < 256 { 1 } else { 2 },
350        maximum,
351    };
352    let read = match wire_type {
353        MYSQL_TYPE_TINY | MYSQL_TYPE_YEAR => Fixed(1),
354        MYSQL_TYPE_SHORT => Fixed(2),
355        MYSQL_TYPE_LONG | MYSQL_TYPE_TIMESTAMP => Fixed(4),
356        MYSQL_TYPE_LONGLONG | MYSQL_TYPE_DATETIME => Fixed(8),
357        MYSQL_TYPE_INT24 | MYSQL_TYPE_DATE | MYSQL_TYPE_NEWDATE | MYSQL_TYPE_TIME => Fixed(3),
358        MYSQL_TYPE_FLOAT | MYSQL_TYPE_DOUBLE => {
359            m[0] = metadata.byte()?;
360            let width = if wire_type == MYSQL_TYPE_FLOAT { 4 } else { 8 };
361            if usize::from(m[0]) != width {
362                return Err(invalid("FLOAT/DOUBLE metadata width"));
363            }
364            Fixed(width)
365        }
366        MYSQL_TYPE_VARCHAR | MYSQL_TYPE_VAR_STRING => {
367            m.copy_from_slice(metadata.take(2)?);
368            string(u64::from(u16::from_le_bytes(m)))
369        }
370        MYSQL_TYPE_BIT => {
371            m.copy_from_slice(metadata.take(2)?);
372            let bits = usize::from(m[0]) + usize::from(m[1]) * 8;
373            if m[0] > 7 || bits == 0 || bits > 64 {
374                return Err(invalid("BIT metadata width"));
375            }
376            Fixed(bits.div_ceil(8))
377        }
378        MYSQL_TYPE_TIMESTAMP2 | MYSQL_TYPE_DATETIME2 | MYSQL_TYPE_TIME2 => {
379            m[0] = metadata.byte()?;
380            if m[0] > 6 {
381                return Err(invalid("temporal precision exceeds 6"));
382            }
383            let width = match wire_type {
384                MYSQL_TYPE_TIMESTAMP2 => 4,
385                MYSQL_TYPE_DATETIME2 => 5,
386                _ => 3,
387            };
388            Fixed(width + usize::from(m[0]).div_ceil(2))
389        }
390        MYSQL_TYPE_NEWDECIMAL => {
391            m.copy_from_slice(metadata.take(2)?);
392            if m[0] == 0 || m[0] > 65 || m[1] > m[0] || m[1] > 30 {
393                return Err(invalid("DECIMAL precision/scale"));
394            }
395            let integral = usize::from(m[0] - m[1]);
396            let scale = usize::from(m[1]);
397            Fixed(
398                integral / 9 * 4
399                    + super::value::decimal_group_size(integral % 9)
400                    + scale / 9 * 4
401                    + super::value::decimal_group_size(scale % 9),
402            )
403        }
404        MYSQL_TYPE_ENUM | MYSQL_TYPE_SET | MYSQL_TYPE_STRING => {
405            m.copy_from_slice(metadata.take(2)?);
406            if wire_type == MYSQL_TYPE_STRING {
407                column_type = match m[0] | 0x30 {
408                    247 => MYSQL_TYPE_ENUM,
409                    248 => MYSQL_TYPE_SET,
410                    254 => MYSQL_TYPE_STRING,
411                    _ if m[0] == 0 => MYSQL_TYPE_STRING,
412                    _ => return Err(invalid("invalid STRING real type")),
413                };
414            }
415            if matches!(column_type, MYSQL_TYPE_ENUM | MYSQL_TYPE_SET) {
416                let width = usize::from(m[1]);
417                if width == 0 || width > if column_type == MYSQL_TYPE_ENUM { 2 } else { 8 } {
418                    return Err(invalid("ENUM/SET width"));
419                }
420                Fixed(width)
421            } else {
422                let maximum = if m[0] == 0 {
423                    u64::from(m[1]) << 8
424                } else {
425                    u64::from(m[1]) | (u64::from((m[0] & 0x30) ^ 0x30) << 4)
426                };
427                string(maximum)
428            }
429        }
430        MYSQL_TYPE_JSON
431        | MYSQL_TYPE_TINY_BLOB
432        | MYSQL_TYPE_MEDIUM_BLOB
433        | MYSQL_TYPE_LONG_BLOB
434        | MYSQL_TYPE_BLOB
435        | MYSQL_TYPE_GEOMETRY
436        | MYSQL_TYPE_VECTOR => {
437            m[0] = metadata.byte()?;
438            let width = usize::from(m[0]);
439            if !(1..=4).contains(&width) || wire_type == MYSQL_TYPE_JSON && width != 4 {
440                return Err(invalid("BLOB/JSON length-prefix width"));
441            }
442            Prefixed {
443                width,
444                maximum: (1u64 << (width * 8)) - 1,
445            }
446        }
447        _ => return Err(Error::Unsupported(format!("wire column type {ty}"))),
448    };
449    Ok(ColumnDefinition {
450        wire_type,
451        column_type,
452        nullable,
453        metadata: m,
454        metadata_len: start.len() - metadata.remaining().len(),
455        read,
456    })
457}