Skip to main content

mysql_replication/
stream.rs

1//! Synchronous replication configuration without CDC schema or checkpoints.
2use crate::{
3    BinlogEvent, BinlogFrame, BinlogParser, Error, FrameDecoder, MAX_EVENT_BYTES, Result,
4    error::invalid,
5    event_type as code,
6    events::{BinlogMode, ImageMode, Unspecified},
7    parser::ParserConfig,
8};
9use mysql::{
10    BinlogDumpFlags, BinlogRequest, BinlogStream, Conn, Opts, OptsBuilder, SslOpts,
11    prelude::Queryable,
12};
13use mysql_common::binlog::{
14    consts::BinlogChecksumAlg,
15    events::{BinlogEventHeader, Event as DriverEvent},
16};
17use std::{path::PathBuf, time::Duration};
18
19#[derive(Debug, Clone)]
20pub struct ConnectionConfig {
21    pub url_env: String,
22    pub tls_ca: Option<PathBuf>,
23    pub connect_timeout: Duration,
24    pub read_timeout: Duration,
25}
26impl ConnectionConfig {
27    pub fn new(url_env: impl Into<String>) -> Self {
28        Self {
29            url_env: url_env.into(),
30            tls_ca: None,
31            connect_timeout: Duration::from_secs(10),
32            read_timeout: Duration::from_secs(90),
33        }
34    }
35}
36pub fn connect(config: &ConnectionConfig) -> Result<Conn> {
37    let url = std::env::var(&config.url_env)
38        .map_err(|_| invalid(format!("missing environment variable {}", config.url_env)))?;
39    let opts = Opts::from_url(&url).map_err(|_| invalid("invalid MySQL URL"))?;
40    let mut builder = OptsBuilder::from_opts(opts)
41        .tcp_connect_timeout(Some(config.connect_timeout))
42        .read_timeout(Some(config.read_timeout));
43    if let Some(ca) = &config.tls_ca {
44        builder = builder.ssl_opts(SslOpts::default().with_root_cert_path(Some(ca.clone())));
45    }
46    Ok(Conn::new(builder)?)
47}
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum RowImage {
50    Minimal,
51    Full,
52    NoBlob,
53}
54impl RowImage {
55    fn value(self) -> &'static str {
56        match self {
57            Self::Minimal => "MINIMAL",
58            Self::Full => "FULL",
59            Self::NoBlob => "NOBLOB",
60        }
61    }
62}
63#[derive(Debug, Clone)]
64pub struct ReplicationConfig {
65    pub server_id: u32,
66    pub file: String,
67    pub position: u64,
68    pub non_blocking: bool,
69    /// Verifies upstream logging; never changes a global server setting.
70    pub required_row_image: Option<RowImage>,
71    pub parser: ParserConfig,
72}
73impl ReplicationConfig {
74    pub fn new(server_id: u32, file: impl Into<String>, position: u64) -> Self {
75        Self {
76            server_id,
77            file: file.into(),
78            position,
79            non_blocking: false,
80            required_row_image: None,
81            parser: ParserConfig::default(),
82        }
83    }
84}
85
86pub struct ReplicationStream<M: BinlogMode = Unspecified> {
87    stream: BinlogStream,
88    current: Option<DriverEvent>,
89    buffer: Vec<u8>,
90    framing: FrameDecoder,
91    parser: BinlogParser<M>,
92}
93impl ReplicationStream {
94    /// Validates primary keys for selected tables before starting replication.
95    /// The table list controls validation only; it does not filter binlog events.
96    pub fn connect_with_tables<'a>(
97        connection: &ConnectionConfig,
98        replication: ReplicationConfig,
99        tables: impl IntoIterator<Item = (&'a str, &'a str)>,
100    ) -> Result<Self> {
101        Self::from_connection_with_tables(connect(connection)?, replication, tables)
102    }
103    /// Validates primary keys on this connection before sending the binlog request.
104    /// The table list controls validation only; it does not filter binlog events.
105    pub fn from_connection_with_tables<'a>(
106        mut conn: Conn,
107        config: ReplicationConfig,
108        tables: impl IntoIterator<Item = (&'a str, &'a str)>,
109    ) -> Result<Self> {
110        crate::primary_key::validate_primary_keys(&mut conn, tables)?;
111        Self::from_connection(conn, config)
112    }
113    pub fn connect(connection: &ConnectionConfig, replication: ReplicationConfig) -> Result<Self> {
114        Self::from_connection(connect(connection)?, replication)
115    }
116    pub fn from_connection(mut conn: Conn, config: ReplicationConfig) -> Result<Self> {
117        if config.server_id == 0 || config.file.is_empty() || config.position < 4 {
118            return Err(invalid(
119                "replication requires server_id, file and position >= 4",
120            ));
121        }
122        if let Some(image) = config.required_row_image {
123            let actual: Option<String> = conn.query_first("SELECT @@global.binlog_row_image")?;
124            if !actual
125                .as_deref()
126                .is_some_and(|s| s.eq_ignore_ascii_case(image.value()))
127            {
128                return Err(Error::Unsupported(format!(
129                    "requires binlog_row_image={}",
130                    image.value()
131                )));
132            }
133        }
134        conn.query_drop("SET @master_heartbeat_period=30000000000")?;
135        let request = BinlogRequest::new(config.server_id)
136            .with_filename(config.file.as_bytes())
137            .with_pos(config.position)
138            .with_flags(if config.non_blocking {
139                BinlogDumpFlags::BINLOG_DUMP_NON_BLOCK
140            } else {
141                BinlogDumpFlags::empty()
142            });
143        Ok(Self {
144            stream: conn.get_binlog_stream(request)?,
145            current: None,
146            buffer: Vec::new(),
147            framing: FrameDecoder::new(),
148            parser: BinlogParser::new(config.parser),
149        })
150    }
151    /// Selects an image policy before connecting and requesting binlog bytes.
152    pub fn connect_with_mode<M: ImageMode>(
153        connection: &ConnectionConfig,
154        replication: ReplicationConfig,
155    ) -> Result<ReplicationStream<M>> {
156        let replication = bind_image_mode::<M>(replication)?;
157        Self::from_connection_with_mode(connect(connection)?, replication)
158    }
159    pub fn from_connection_with_mode<M: ImageMode>(
160        conn: Conn,
161        mut config: ReplicationConfig,
162    ) -> Result<ReplicationStream<M>> {
163        config = bind_image_mode::<M>(config)?;
164        let stream = Self::from_connection(conn, config)?;
165        Ok(ReplicationStream {
166            stream: stream.stream,
167            current: stream.current,
168            buffer: stream.buffer,
169            framing: stream.framing,
170            parser: stream.parser.with_mode(),
171        })
172    }
173}
174impl<M: ImageMode> ReplicationStream<M> {
175    pub fn for_mode(connection: &ConnectionConfig, config: ReplicationConfig) -> Result<Self> {
176        ReplicationStream::connect_with_mode::<M>(connection, config)
177    }
178}
179impl<M: BinlogMode> ReplicationStream<M> {
180    /// CDC's fast path: checksum verification and a borrowed body, with no raw
181    /// frame assembly, GTID metadata decoding or row-value materialization.
182    pub fn next_view(&mut self) -> Result<Option<crate::EventView<'_>>> {
183        self.current = self.stream.next().transpose()?;
184        let Some(event) = &self.current else {
185            return Ok(None);
186        };
187        validate_driver(event)?;
188        if event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
189            let algorithm = event
190                .footer()
191                .get_checksum_alg()
192                .map_err(|e| Error::Unsupported(e.to_string()))?;
193            self.framing
194                .set_checksum(algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32));
195        }
196        Ok(Some(crate::EventView {
197            header: header_view(event.header()),
198            body: event.data(),
199        }))
200    }
201    pub fn next_frame(&mut self) -> Result<Option<BinlogFrame<'_>>> {
202        read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)
203    }
204    pub fn next_event(&mut self) -> Result<Option<BinlogEvent<'_, M>>> {
205        let Some(frame) = read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)? else {
206            return Ok(None);
207        };
208        self.parser.parse(frame).map(Some)
209    }
210}
211fn bind_image_mode<M: ImageMode>(mut config: ReplicationConfig) -> Result<ReplicationConfig> {
212    let expected = M::ROW_IMAGE.into();
213    if config
214        .required_row_image
215        .is_some_and(|mode| mode != expected)
216    {
217        return Err(invalid(
218            "replication configuration conflicts with the generic image mode",
219        ));
220    }
221    config.required_row_image = Some(expected);
222    Ok(config)
223}
224
225fn read_frame<'a>(
226    stream: &mut BinlogStream,
227    buffer: &'a mut Vec<u8>,
228    framing: &mut FrameDecoder,
229) -> Result<Option<BinlogFrame<'a>>> {
230    let Some(event) = stream.next().transpose()? else {
231        return Ok(None);
232    };
233    retain_driver_bytes(&event, buffer)?;
234    framing.decode(buffer).map(Some)
235}
236
237fn header_view(header: BinlogEventHeader) -> crate::EventHeader {
238    crate::EventHeader {
239        timestamp: header.timestamp(),
240        event_type: header.event_type_raw(),
241        server_id: header.server_id(),
242        event_size: header.event_size(),
243        log_pos: header.log_pos(),
244        flags: header.flags_raw(),
245    }
246}
247fn validate_driver(event: &DriverEvent) -> Result<()> {
248    let length = event.header().event_size() as usize;
249    if !(19..=MAX_EVENT_BYTES).contains(&length) {
250        return Err(invalid("event size outside 19..64MiB"));
251    }
252    let algorithm = event
253        .footer()
254        .get_checksum_alg()
255        .map_err(|e| Error::Unsupported(e.to_string()))?;
256    if algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32)
257        && event.checksum().map(u32::from_le_bytes)
258            != Some(event.calc_checksum(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32))
259    {
260        return Err(Error::Checksum);
261    }
262    let trailer = usize::from(event.checksum().is_some()) * 4
263        + usize::from(event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT);
264    if 19 + event.data().len() + trailer != length {
265        return Err(invalid("driver did not preserve the complete wire event"));
266    }
267    Ok(())
268}
269
270// The driver splits the wire bytes into header, body, algorithm and checksum.
271// Retain each original value; Event::write must not be used because it recalculates
272// the checksum and can normalize the original trailer.
273fn retain_driver_bytes(event: &DriverEvent, buffer: &mut Vec<u8>) -> Result<()> {
274    let header = event.header();
275    let len = header.event_size() as usize;
276    if !(19..=MAX_EVENT_BYTES).contains(&len) {
277        return Err(invalid("event size outside 19..64MiB"));
278    }
279    buffer.clear();
280    buffer.reserve(len);
281    buffer.extend_from_slice(&header.timestamp().to_le_bytes());
282    buffer.push(header.event_type_raw());
283    buffer.extend_from_slice(&header.server_id().to_le_bytes());
284    buffer.extend_from_slice(&header.event_size().to_le_bytes());
285    buffer.extend_from_slice(&header.log_pos().to_le_bytes());
286    buffer.extend_from_slice(&header.flags_raw().to_le_bytes());
287    buffer.extend_from_slice(event.data());
288    if header.event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
289        let algorithm = event
290            .footer()
291            .get_checksum_alg()
292            .map_err(|e| Error::Unsupported(e.to_string()))?
293            .ok_or_else(|| invalid("FDE checksum algorithm missing"))?;
294        buffer.push(match algorithm {
295            BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_OFF => 0,
296            BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32 => 1,
297        });
298    }
299    if let Some(checksum) = event.checksum() {
300        buffer.extend_from_slice(&checksum);
301    }
302    if buffer.len() != len {
303        return Err(invalid("driver did not preserve the complete wire event"));
304    }
305    Ok(())
306}
307
308#[cfg(test)]
309mod tests {
310    use super::*;
311    use mysql_common::binlog::{consts::BinlogVersion, events::FormatDescriptionEvent};
312    #[test]
313    fn driver_trailer_is_retained_instead_of_recalculated() {
314        let mut body = vec![0; 57 + 43];
315        body[..2].copy_from_slice(&4u16.to_le_bytes());
316        body[2..8].copy_from_slice(b"8.4.0\0");
317        body[56] = 19;
318        for algorithm in [0u8, 1] {
319            let mut raw = vec![0; 19];
320            raw[4] = code::FORMAT_DESCRIPTION_EVENT;
321            raw[5..9].copy_from_slice(&123u32.to_le_bytes());
322            raw[17..19].copy_from_slice(&0x8001u16.to_le_bytes());
323            raw.extend_from_slice(&body);
324            raw.push(algorithm);
325            raw.extend_from_slice(&0x12345678u32.to_le_bytes());
326            let length = raw.len() as u32;
327            raw[9..13].copy_from_slice(&length.to_le_bytes());
328            let fde = FormatDescriptionEvent::new(BinlogVersion::Version4);
329            let event = DriverEvent::read(&fde, &raw[..]).unwrap();
330            let mut retained = Vec::new();
331            retain_driver_bytes(&event, &mut retained).unwrap();
332            assert_eq!(retained, raw);
333        }
334    }
335}