Skip to main content

mysql_replication/protocol/mysql/
decoder.rs

1use super::events::{
2    GENERATED_INVISIBLE_PK_F, MysqlEventParser, RowsEvent, STMT_END_F, TableMapEvent,
3};
4use crate::{
5    Change, ColumnMeta, Error, Operation, PrimaryKey, Result, TableMeta,
6    error::invalid,
7    model::schema::{KeyDecoder, key_decoder},
8    protocol::mysql::wire::{Cursor, bit, set_bits},
9};
10use std::{collections::BTreeMap, sync::Arc};
11
12#[derive(Clone, Copy, Debug)]
13enum ReadOp {
14    Fixed(usize),
15    Prefixed { width: usize, maximum: u64 },
16}
17impl ReadOp {
18    fn read<'a>(self, input: &mut Cursor<'a>) -> Result<&'a [u8]> {
19        let len = match self {
20            Self::Fixed(n) => n,
21            Self::Prefixed { width, maximum } => {
22                let n = input.le(width)?;
23                if n > maximum {
24                    return Err(invalid("column length exceeds TABLE_MAP maximum"));
25                }
26                usize::try_from(n).map_err(|_| invalid("column length overflows usize"))?
27            }
28        };
29        input.take(len)
30    }
31}
32
33struct TablePlan {
34    meta: Arc<TableMeta>,
35    pk_index: usize,
36    decode_pk: KeyDecoder,
37    columns: Vec<ReadOp>,
38    signature: Vec<u8>,
39}
40
41struct ReadColumn {
42    op: ReadOp,
43    is_pk: bool,
44}
45struct ImagePlan {
46    columns: Vec<ReadColumn>,
47}
48impl ImagePlan {
49    fn new(table: &TablePlan, present: &[u8]) -> Self {
50        Self {
51            columns: set_bits(present)
52                .take_while(|&i| i < table.columns.len())
53                .map(|i| ReadColumn {
54                    op: table.columns[i],
55                    is_pk: i == table.pk_index,
56                })
57                .collect(),
58        }
59    }
60    fn read<'a>(
61        &self,
62        input: &mut Cursor<'a>,
63        table: &TablePlan,
64    ) -> Result<Option<PrimaryKey<'a>>> {
65        let nulls = input.bitmap(self.columns.len())?;
66        let mut pk = None;
67        for (i, col) in self.columns.iter().enumerate() {
68            if bit(nulls, i) {
69                if col.is_pk {
70                    return Err(Error::MissingPrimaryKey);
71                }
72                continue;
73            }
74            let data = col.op.read(input)?;
75            if col.is_pk {
76                pk = Some((table.decode_pk)(
77                    data,
78                    &table.meta.columns[table.pk_index],
79                )?);
80            }
81        }
82        Ok(pk)
83    }
84}
85
86/// TABLE_MAP preprocessing + sparse ROWS decoding. The callback borrows event bytes.
87/// Only white-listed tables are compiled; other tables' ROWS payloads are skipped.
88pub struct Decoder {
89    tables: BTreeMap<String, BTreeMap<String, Arc<TableMeta>>>,
90    compiled: BTreeMap<String, BTreeMap<String, Arc<TablePlan>>>,
91    index: Vec<(u64, Option<Arc<TablePlan>>)>,
92}
93
94impl Decoder {
95    pub fn new(tables: Vec<TableMeta>) -> Result<Self> {
96        let mut out = Self {
97            tables: BTreeMap::new(),
98            compiled: BTreeMap::new(),
99            index: Vec::new(),
100        };
101        if tables.is_empty() {
102            return Err(Error::Schema("empty whitelist".into()));
103        }
104        for table in tables {
105            table.validate()?;
106            let schema = out.tables.entry(table.schema.clone()).or_default();
107            if schema
108                .insert(table.table.clone(), Arc::new(table))
109                .is_some()
110            {
111                return Err(Error::Schema("duplicate whitelist table".into()));
112            }
113        }
114        Ok(out)
115    }
116
117    pub fn clear_table_maps(&mut self) {
118        self.index.clear();
119    }
120
121    /// Input excludes the 19-byte common header and checksum. Modern 6-byte IDs only.
122    pub fn table_map(&mut self, data: &[u8]) -> Result<()> {
123        self.table_map_event(MysqlEventParser::standard().table_map(data)?)
124    }
125
126    pub fn table_map_event(&mut self, event: TableMapEvent<'_>) -> Result<()> {
127        let TableMapEvent {
128            table_id: id,
129            flags,
130            schema,
131            table: name,
132            column_data: signature,
133            ..
134        } = event;
135        let meta = self.tables.get(schema).and_then(|s| s.get(name));
136        let plan = if let Some(meta) = meta {
137            if flags & GENERATED_INVISIBLE_PK_F != 0 {
138                return Err(Error::Unsupported("generated invisible primary key".into()));
139            }
140            if let Some(existing) = self
141                .compiled
142                .get(schema)
143                .and_then(|s| s.get(name))
144                .filter(|p| p.signature == signature)
145            {
146                Some(Arc::clone(existing))
147            } else {
148                let plan = Arc::new(compile(Arc::clone(meta), signature)?);
149                self.compiled
150                    .entry(schema.into())
151                    .or_default()
152                    .insert(name.into(), Arc::clone(&plan));
153                Some(plan)
154            }
155        } else {
156            None
157        };
158        let i = self.index.partition_point(|(table_id, _)| *table_id < id);
159        if self
160            .index
161            .get(i)
162            .is_some_and(|(table_id, _)| *table_id == id)
163        {
164            self.index[i].1 = plan;
165        } else {
166            self.index.insert(i, (id, plan));
167        }
168        Ok(())
169    }
170
171    /// Supports WRITE_ROWS_EVENT, UPDATE_ROWS_EVENT and DELETE_ROWS_EVENT, v1/v2.
172    pub fn rows(
173        &mut self,
174        event_type: u8,
175        data: &[u8],
176        mut emit: impl FnMut(Change<'_>) -> Result<()>,
177    ) -> Result<()> {
178        self.rows_event(
179            MysqlEventParser::standard().rows(event_type, data)?,
180            &mut emit,
181        )
182    }
183
184    pub fn rows_event(
185        &mut self,
186        event: RowsEvent<'_>,
187        mut emit: impl FnMut(Change<'_>) -> Result<()>,
188    ) -> Result<()> {
189        let id = event.table_id;
190        let flags = event.flags;
191        let operation = event.operation;
192        if id == 0x00ff_ffff && flags & STMT_END_F != 0 {
193            self.clear_table_maps();
194            return Ok(());
195        }
196        let i = self.index.partition_point(|(table_id, _)| *table_id < id);
197        let table = self
198            .index
199            .get(i)
200            .filter(|(table_id, _)| *table_id == id)
201            .ok_or(Error::MissingTable(id))?;
202        // Excluded tables never enter column or row decoding.
203        let Some(table) = table.1.as_ref() else {
204            if flags & STMT_END_F != 0 {
205                self.clear_table_maps();
206            }
207            return Ok(());
208        };
209        let images = event.images()?;
210        let count = images.column_count;
211        if count != table.columns.len() {
212            return Err(Error::Schema(
213                "ROWS column count differs from TABLE_MAP".into(),
214            ));
215        }
216        let first = images.before_columns;
217        let second = images.after_columns;
218        let mut input = Cursor::new(images.rows);
219        let before = ImagePlan::new(table, first);
220        let after = ImagePlan::new(table, second);
221        if before.columns.is_empty() {
222            return Err(invalid("empty first row image"));
223        }
224        if operation == Operation::Update && after.columns.is_empty() {
225            return Err(invalid("empty after row image"));
226        }
227        // With MINIMAL + a single PK, the before image must contain that PK only.
228        if operation != Operation::Insert
229            && (before.columns.len() != 1 || !bit(first, table.pk_index))
230        {
231            return Err(Error::Unsupported(
232                "expected MINIMAL before image containing only the primary key".into(),
233            ));
234        }
235        let changed: Arc<[u16]> = if operation == Operation::Delete {
236            Arc::from([])
237        } else {
238            set_bits(second)
239                .take_while(|&i| i < count)
240                .map(|i| i as u16)
241                .collect()
242        };
243        if input.remaining().is_empty() {
244            return Err(invalid("ROWS event contains no rows"));
245        }
246        while !input.remaining().is_empty() {
247            let old = before
248                .read(&mut input, table)?
249                .ok_or(Error::MissingPrimaryKey)?;
250            let (pk, previous_pk) = if operation == Operation::Update {
251                match after.read(&mut input, table)? {
252                    Some(new) if new != old => (new, Some(old)),
253                    Some(new) => (new, None),
254                    None => (old, None),
255                }
256            } else {
257                (old, None)
258            };
259            emit(Change {
260                table: &table.meta,
261                operation,
262                pk,
263                previous_pk,
264                changed: &changed,
265            })?;
266        }
267        if flags & STMT_END_F != 0 {
268            self.clear_table_maps();
269        }
270        Ok(())
271    }
272}
273
274fn compile(meta: Arc<TableMeta>, data: &[u8]) -> Result<TablePlan> {
275    let pk_index = meta.validate()?;
276    let decode_pk = key_decoder(&meta.columns[pk_index])?;
277    let mut input = Cursor::new(data);
278    let count = input.lenenc()?;
279    if count != meta.columns.len() {
280        return Err(Error::Schema(format!(
281            "{}.{} column count differs",
282            meta.schema, meta.table
283        )));
284    }
285    let types = input.take(count)?;
286    let metadata_len = input.lenenc()?;
287    let mut metadata = Cursor::new(input.take(metadata_len)?);
288    let nullable = input.bitmap(count)?;
289    let mut columns = Vec::with_capacity(count);
290    for (i, (&ty, col)) in types.iter().zip(&meta.columns).enumerate() {
291        if bit(nullable, i) != col.nullable {
292            return Err(Error::Schema(format!("{} nullability differs", col.name)));
293        }
294        let (op, logical_type) = read_op(ty, &mut metadata, col)?;
295        if !compatible(&col.data_type, logical_type) {
296            return Err(Error::Schema(format!(
297                "{}: {} differs from wire type {logical_type}",
298                col.name, col.data_type
299            )));
300        }
301        columns.push(op);
302    }
303    if !metadata.remaining().is_empty() {
304        return Err(invalid("unused TABLE_MAP column metadata"));
305    }
306    // Validate optional column names when binlog_row_metadata=FULL.
307    while !input.remaining().is_empty() {
308        let ty = input.byte()?;
309        let len = input.lenenc()?;
310        let value = input.take(len)?;
311        if ty == 4 {
312            let mut names = Cursor::new(value);
313            for col in &meta.columns {
314                let len = names.lenenc()?;
315                if names.take(len)? != col.name.as_bytes() {
316                    return Err(Error::Schema(
317                        "optional TABLE_MAP column name differs".into(),
318                    ));
319                }
320            }
321            if !names.remaining().is_empty() {
322                return Err(invalid("extra optional column names"));
323            }
324        }
325    }
326    Ok(TablePlan {
327        meta,
328        pk_index,
329        decode_pk,
330        columns,
331        signature: data.to_vec(),
332    })
333}
334
335fn compatible(sql: &str, ty: u8) -> bool {
336    matches!(
337        (sql, ty),
338        ("tinyint", 1)
339            | ("smallint", 2)
340            | ("int", 3)
341            | ("float", 4)
342            | ("double", 5)
343            | ("timestamp", 7 | 17)
344            | ("bigint", 8)
345            | ("mediumint", 9)
346            | ("date", 10 | 14)
347            | ("time", 11 | 19)
348            | ("datetime", 12 | 18)
349            | ("year", 13)
350            | ("varchar" | "varbinary", 15)
351            | ("bit", 16)
352            | ("vector", 242)
353            | ("json", 245)
354            | ("decimal", 246)
355            | ("enum", 247)
356            | ("set", 248)
357            | (
358                "tinyblob"
359                    | "tinytext"
360                    | "blob"
361                    | "text"
362                    | "mediumblob"
363                    | "mediumtext"
364                    | "longblob"
365                    | "longtext",
366                249..=252
367            )
368            | ("char" | "binary", 254)
369            | (
370                "geometry"
371                    | "point"
372                    | "linestring"
373                    | "polygon"
374                    | "multipoint"
375                    | "multilinestring"
376                    | "multipolygon"
377                    | "geometrycollection",
378                255
379            )
380    )
381}
382
383fn read_op(ty: u8, m: &mut Cursor<'_>, col: &ColumnMeta) -> Result<(ReadOp, u8)> {
384    use ReadOp::*;
385    let fixed = |n| Ok((Fixed(n), ty));
386    match ty {
387        1 | 13 => fixed(1),
388        2 => fixed(2),
389        3 | 7 => fixed(4),
390        8 | 12 => fixed(8),
391        9..=11 | 14 => fixed(3),
392        4 | 5 => {
393            let width = if ty == 4 { 4 } else { 8 };
394            if usize::from(m.byte()?) != width {
395                return Err(invalid("FLOAT/DOUBLE metadata width"));
396            }
397            fixed(width)
398        }
399        15 => {
400            let maximum = m.le(2)?;
401            check_length(col, maximum)?;
402            Ok((
403                Prefixed {
404                    width: if maximum < 256 { 1 } else { 2 },
405                    maximum,
406                },
407                ty,
408            ))
409        }
410        16 => {
411            let bits = u64::from(m.byte()?) + u64::from(m.byte()?) * 8;
412            if bits == 0 || bits > 64 || col.numeric_precision.map(u64::from) != Some(bits) {
413                return Err(Error::Schema("BIT width differs".into()));
414            }
415            fixed(bits.div_ceil(8) as usize)
416        }
417        17..=19 => {
418            let fsp = m.byte()?;
419            if fsp > 6 || col.datetime_precision != Some(u32::from(fsp)) {
420                return Err(Error::Schema("temporal precision differs".into()));
421            }
422            fixed(
423                match ty {
424                    17 => 4,
425                    18 => 5,
426                    _ => 3,
427                } + usize::from(fsp).div_ceil(2),
428            )
429        }
430        246 => {
431            let precision = m.byte()?;
432            let scale = m.byte()?;
433            if precision == 0
434                || precision > 65
435                || scale > precision
436                || scale > 30
437                || col.numeric_precision != Some(u32::from(precision))
438                || col.numeric_scale != Some(u32::from(scale))
439            {
440                return Err(Error::Schema("DECIMAL precision/scale differs".into()));
441            }
442            const PART: [usize; 10] = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4];
443            let int = usize::from(precision - scale);
444            let frac = usize::from(scale);
445            fixed(int / 9 * 4 + PART[int % 9] + frac / 9 * 4 + PART[frac % 9])
446        }
447        247 | 248 => {
448            m.byte()?;
449            let width = usize::from(m.byte()?);
450            if width == 0 || width > if ty == 247 { 2 } else { 8 } {
451                return Err(invalid("ENUM/SET width"));
452            }
453            fixed(width)
454        }
455        254 => {
456            let a = m.byte()?;
457            let b = m.byte()?;
458            let logical = a | 0x30;
459            if logical == 247 || logical == 248 {
460                let width = usize::from(b);
461                if width == 0 || width > if logical == 247 { 2 } else { 8 } {
462                    return Err(invalid("ENUM/SET width"));
463                }
464                Ok((Fixed(width), logical))
465            } else if logical == 254 || a == 0 {
466                let maximum = if a == 0 {
467                    u64::from(b) << 8
468                } else {
469                    u64::from(b) | (u64::from((a & 0x30) ^ 0x30) << 4)
470                };
471                check_length(col, maximum)?;
472                Ok((
473                    Prefixed {
474                        width: if maximum < 256 { 1 } else { 2 },
475                        maximum,
476                    },
477                    254,
478                ))
479            } else {
480                Err(invalid("invalid STRING real type"))
481            }
482        }
483        242 | 245 | 249..=252 | 255 => {
484            let width = usize::from(m.byte()?);
485            if !(1..=4).contains(&width) || ty == 245 && width != 4 {
486                return Err(invalid("BLOB/JSON length-prefix width"));
487            }
488            Ok((
489                Prefixed {
490                    width,
491                    maximum: (1u64 << (width * 8)) - 1,
492                },
493                ty,
494            ))
495        }
496        other => Err(Error::Unsupported(format!("wire column type {other}"))),
497    }
498}
499
500fn check_length(col: &ColumnMeta, maximum: u64) -> Result<()> {
501    if col.octet_length != Some(maximum) {
502        return Err(Error::Schema(format!(
503            "{} maximum byte length differs",
504            col.name
505        )));
506    }
507    Ok(())
508}