Skip to main content

mysql_replication/
stream.rs

1//! Synchronous replication configuration without CDC schema or checkpoints.
2use crate::{
3    BinlogEvent, BinlogFrame, BinlogParser, Error, FrameDecoder, MAX_EVENT_BYTES, Result,
4    error::invalid, event_type as code, events::BinlogMode, parser::ParserConfig,
5};
6use mysql::{
7    BinlogDumpFlags, BinlogRequest, BinlogStream, Conn, Opts, OptsBuilder, SslOpts,
8    prelude::Queryable,
9};
10use mysql_common::binlog::{
11    consts::BinlogChecksumAlg,
12    events::{BinlogEventHeader, Event as DriverEvent},
13};
14use std::{path::PathBuf, time::Duration};
15
16#[derive(Debug, Clone)]
17pub struct ConnectionConfig {
18    pub url_env: String,
19    pub tls_ca: Option<PathBuf>,
20    pub connect_timeout: Duration,
21    pub read_timeout: Duration,
22}
23impl ConnectionConfig {
24    pub fn new(url_env: impl Into<String>) -> Self {
25        Self {
26            url_env: url_env.into(),
27            tls_ca: None,
28            connect_timeout: Duration::from_secs(10),
29            read_timeout: Duration::from_secs(90),
30        }
31    }
32}
33pub fn connect(config: &ConnectionConfig) -> Result<Conn> {
34    let url = std::env::var(&config.url_env)
35        .map_err(|_| invalid(format!("missing environment variable {}", config.url_env)))?;
36    let opts = Opts::from_url(&url).map_err(|_| invalid("invalid MySQL URL"))?;
37    // The status byte is part of the MySQL packet, outside the binlog frame.
38    let packet_limit = opts
39        .get_max_allowed_packet()
40        .unwrap_or(MAX_EVENT_BYTES + 1)
41        .min(MAX_EVENT_BYTES + 1);
42    let mut builder = OptsBuilder::from_opts(opts)
43        .max_allowed_packet(Some(packet_limit))
44        .tcp_connect_timeout(Some(config.connect_timeout))
45        .read_timeout(Some(config.read_timeout));
46    if let Some(ca) = &config.tls_ca {
47        builder = builder.ssl_opts(SslOpts::default().with_root_cert_path(Some(ca.clone())));
48    }
49    Ok(Conn::new(builder)?)
50}
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum RowImage {
53    Minimal,
54    Full,
55    NoBlob,
56}
57impl RowImage {
58    fn value(self) -> &'static str {
59        match self {
60            Self::Minimal => "MINIMAL",
61            Self::Full => "FULL",
62            Self::NoBlob => "NOBLOB",
63        }
64    }
65}
66#[derive(Debug, Clone)]
67pub struct ReplicationConfig {
68    pub server_id: u32,
69    pub file: String,
70    pub position: u64,
71    pub non_blocking: bool,
72    /// Verifies upstream logging; never changes a global server setting.
73    pub required_row_image: Option<RowImage>,
74    pub parser: ParserConfig,
75}
76impl ReplicationConfig {
77    pub fn new(server_id: u32, file: impl Into<String>, position: u64) -> Self {
78        Self {
79            server_id,
80            file: file.into(),
81            position,
82            non_blocking: false,
83            required_row_image: None,
84            parser: ParserConfig::default(),
85        }
86    }
87    /// Checks file-position bounds for the 32-bit COM_BINLOG_DUMP command.
88    pub fn validate(&self) -> Result<()> {
89        if self.server_id == 0
90            || self.file.is_empty()
91            || !(4..=u64::from(u32::MAX)).contains(&self.position)
92        {
93            return Err(invalid(
94                "replication requires server_id, file and position in 4..=u32::MAX",
95            ));
96        }
97        Ok(())
98    }
99}
100
101/// Raw network transport for adapters that own event-body parsing.
102pub struct ReplicationConnection {
103    stream: Option<BinlogStream>,
104    current: Option<DriverEvent>,
105    buffer: Vec<u8>,
106    framing: FrameDecoder,
107}
108impl ReplicationConnection {
109    pub fn connect(connection: &ConnectionConfig, replication: ReplicationConfig) -> Result<Self> {
110        replication.validate()?;
111        Self::from_connection(connect(connection)?, replication)
112    }
113    pub fn from_connection(mut conn: Conn, config: ReplicationConfig) -> Result<Self> {
114        config.validate()?;
115        if let Some(image) = config.required_row_image {
116            let actual: Option<String> = conn.query_first("SELECT @@global.binlog_row_image")?;
117            if !actual
118                .as_deref()
119                .is_some_and(|s| s.eq_ignore_ascii_case(image.value()))
120            {
121                return Err(Error::Unsupported(format!(
122                    "requires binlog_row_image={}",
123                    image.value()
124                )));
125            }
126        }
127        conn.query_drop("SET @master_heartbeat_period=30000000000")?;
128        let request = BinlogRequest::new(config.server_id)
129            .with_filename(config.file.as_bytes())
130            .with_pos(config.position)
131            .with_flags(if config.non_blocking {
132                BinlogDumpFlags::BINLOG_DUMP_NON_BLOCK
133            } else {
134                BinlogDumpFlags::empty()
135            });
136        Ok(Self {
137            stream: Some(conn.get_binlog_stream(request)?),
138            current: None,
139            buffer: Vec::new(),
140            framing: FrameDecoder::new(),
141        })
142    }
143    /// CDC's fast path: checksum verification and a borrowed body, with no raw
144    /// frame assembly, GTID metadata decoding or row-value materialization.
145    pub fn next_view(&mut self) -> Result<Option<crate::EventView<'_>>> {
146        self.current = next_driver_event(&mut self.stream)?;
147        let Some(event) = &self.current else {
148            return Ok(None);
149        };
150        validate_driver(event)?;
151        if event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
152            let algorithm = event
153                .footer()
154                .get_checksum_alg()
155                .map_err(|e| Error::Unsupported(e.to_string()))?;
156            self.framing
157                .set_checksum(algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32));
158        }
159        Ok(Some(crate::EventView {
160            header: header_view(event.header()),
161            body: event.data(),
162        }))
163    }
164    pub fn next_frame(&mut self) -> Result<Option<BinlogFrame<'_>>> {
165        read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)
166    }
167}
168
169/// Typed event subscription; the image mode is explicit and fixed.
170pub struct ReplicationStream<M: BinlogMode> {
171    connection: ReplicationConnection,
172    parser: BinlogParser<M>,
173}
174impl<M: BinlogMode> ReplicationStream<M> {
175    pub fn connect(connection: &ConnectionConfig, config: ReplicationConfig) -> Result<Self> {
176        let config = bind_image_mode::<M>(config)?;
177        Self::from_connection(connect(connection)?, config)
178    }
179    pub fn from_connection(conn: Conn, config: ReplicationConfig) -> Result<Self> {
180        let config = bind_image_mode::<M>(config)?;
181        let parser = BinlogParser::new(config.parser);
182        Ok(Self {
183            connection: ReplicationConnection::from_connection(conn, config)?,
184            parser,
185        })
186    }
187    pub fn next_event(&mut self) -> Result<Option<BinlogEvent<'_, M>>> {
188        let Some(frame) = self.connection.next_frame()? else {
189            return Ok(None);
190        };
191        self.parser.parse(frame).map(Some)
192    }
193}
194
195fn bind_image_mode<M: BinlogMode>(mut config: ReplicationConfig) -> Result<ReplicationConfig> {
196    config.validate()?;
197    let expected = M::ROW_IMAGE.into();
198    if config
199        .required_row_image
200        .is_some_and(|mode| mode != expected)
201    {
202        return Err(invalid(
203            "replication configuration conflicts with the generic image mode",
204        ));
205    }
206    config.required_row_image = Some(expected);
207    Ok(config)
208}
209
210// mysql_common 0.37 unwraps short reads while decoding a complete network packet.
211// Discard the driver on unwind: its parser and socket may already have advanced.
212fn next_driver_event<S>(stream: &mut Option<S>) -> Result<Option<DriverEvent>>
213where
214    S: Iterator<Item = mysql::Result<DriverEvent>>,
215{
216    let Some(reader) = stream.as_mut() else {
217        return Ok(None);
218    };
219    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| reader.next()))
220        .map_err(|_| invalid("driver rejected malformed binlog packet"))
221        .and_then(|item| item.transpose().map_err(Error::from));
222    if !matches!(result, Ok(Some(_))) {
223        *stream = None;
224    }
225    result
226}
227
228fn read_frame<'a>(
229    stream: &mut Option<BinlogStream>,
230    buffer: &'a mut Vec<u8>,
231    framing: &mut FrameDecoder,
232) -> Result<Option<BinlogFrame<'a>>> {
233    let Some(event) = next_driver_event(stream)? else {
234        return Ok(None);
235    };
236    retain_driver_bytes(&event, buffer)?;
237    framing.decode_network(buffer).map(Some)
238}
239
240fn header_view(header: BinlogEventHeader) -> crate::EventHeader {
241    crate::EventHeader {
242        timestamp: header.timestamp(),
243        event_type: header.event_type_raw(),
244        server_id: header.server_id(),
245        event_size: header.event_size(),
246        log_pos: header.log_pos(),
247        flags: header.flags_raw(),
248    }
249}
250fn validate_driver(event: &DriverEvent) -> Result<()> {
251    let length = event.header().event_size() as usize;
252    if !(19..=MAX_EVENT_BYTES).contains(&length) {
253        return Err(invalid("event size outside 19..64MiB"));
254    }
255    let algorithm = event
256        .footer()
257        .get_checksum_alg()
258        .map_err(|e| Error::Unsupported(e.to_string()))?;
259    let crc32 = algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32);
260    let disk_fde = event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT
261        && event.header().log_pos() != 0;
262    if (crc32 || disk_fde)
263        && event.checksum().map(u32::from_le_bytes)
264            != Some(
265                event.calc_checksum(
266                    algorithm.ok_or_else(|| invalid("FDE checksum algorithm missing"))?,
267                ),
268            )
269    {
270        return Err(Error::Checksum);
271    }
272    let trailer = usize::from(event.checksum().is_some()) * 4
273        + usize::from(event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT);
274    if 19 + event.data().len() + trailer != length {
275        return Err(invalid("driver did not preserve the complete wire event"));
276    }
277    Ok(())
278}
279
280// The driver splits the wire bytes into header, body, algorithm and checksum.
281// Retain each original value; Event::write must not be used because it recalculates
282// the checksum and can normalize the original trailer.
283fn retain_driver_bytes(event: &DriverEvent, buffer: &mut Vec<u8>) -> Result<()> {
284    let header = event.header();
285    let len = header.event_size() as usize;
286    if !(19..=MAX_EVENT_BYTES).contains(&len) {
287        return Err(invalid("event size outside 19..64MiB"));
288    }
289    buffer.clear();
290    buffer.reserve(len);
291    buffer.extend_from_slice(&header.timestamp().to_le_bytes());
292    buffer.push(header.event_type_raw());
293    buffer.extend_from_slice(&header.server_id().to_le_bytes());
294    buffer.extend_from_slice(&header.event_size().to_le_bytes());
295    buffer.extend_from_slice(&header.log_pos().to_le_bytes());
296    buffer.extend_from_slice(&header.flags_raw().to_le_bytes());
297    buffer.extend_from_slice(event.data());
298    if header.event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
299        let algorithm = event
300            .footer()
301            .get_checksum_alg()
302            .map_err(|e| Error::Unsupported(e.to_string()))?
303            .ok_or_else(|| invalid("FDE checksum algorithm missing"))?;
304        buffer.push(match algorithm {
305            BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_OFF => 0,
306            BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32 => 1,
307        });
308    }
309    if let Some(checksum) = event.checksum() {
310        buffer.extend_from_slice(&checksum);
311    }
312    if buffer.len() != len {
313        return Err(invalid("driver did not preserve the complete wire event"));
314    }
315    Ok(())
316}
317
318#[cfg(test)]
319mod tests {
320    use super::*;
321    use mysql_common::binlog::{consts::BinlogVersion, events::FormatDescriptionEvent};
322    #[test]
323    fn oversized_positions_fail_before_opening_a_connection() {
324        let config = ReplicationConfig::new(1, "binlog.000001", (1u64 << 32) + 4);
325        assert!(
326            matches!(ReplicationConnection::connect(&ConnectionConfig::new("UNUSED"), config),
327            Err(Error::Invalid(message)) if message.contains("u32::MAX"))
328        );
329        assert!(
330            ReplicationConfig::new(1, "binlog.000001", u64::from(u32::MAX))
331                .validate()
332                .is_ok()
333        );
334    }
335
336    #[test]
337    fn truncated_driver_packet_stops_the_stream() {
338        let mut header = [0u8; 19];
339        header[4] = code::XID_EVENT;
340        header[9..13].copy_from_slice(&27u32.to_le_bytes());
341        let mut stream = Some(std::iter::once_with(|| {
342            let fde = FormatDescriptionEvent::new(BinlogVersion::Version4);
343            Ok(DriverEvent::read(&fde, &header[..])?)
344        }));
345        assert!(matches!(
346            next_driver_event(&mut stream),
347            Err(Error::Invalid(_)) | Err(Error::Mysql(_))
348        ));
349        assert!(stream.is_none());
350        assert!(next_driver_event(&mut stream).unwrap().is_none());
351    }
352    #[test]
353    fn driver_trailer_is_retained_instead_of_recalculated() {
354        let mut body = vec![0; 57 + 43];
355        body[..2].copy_from_slice(&4u16.to_le_bytes());
356        body[2..8].copy_from_slice(b"8.4.0\0");
357        body[56] = 19;
358        for algorithm in [0u8, 1] {
359            let mut raw = vec![0; 19];
360            raw[4] = code::FORMAT_DESCRIPTION_EVENT;
361            raw[5..9].copy_from_slice(&123u32.to_le_bytes());
362            raw[17..19].copy_from_slice(&0x8001u16.to_le_bytes());
363            raw.extend_from_slice(&body);
364            raw.push(algorithm);
365            raw.extend_from_slice(&0x12345678u32.to_le_bytes());
366            let length = raw.len() as u32;
367            raw[9..13].copy_from_slice(&length.to_le_bytes());
368            let fde = FormatDescriptionEvent::new(BinlogVersion::Version4);
369            let event = DriverEvent::read(&fde, &raw[..]).unwrap();
370            let mut retained = Vec::new();
371            retain_driver_bytes(&event, &mut retained).unwrap();
372            assert_eq!(retained, raw);
373        }
374    }
375}