Skip to main content

mysql_replication/
stream.rs

1//! Synchronous replication configuration without CDC schema or checkpoints.
2use crate::{
3    BinlogEvent, BinlogFrame, BinlogParser, Error, FrameDecoder, MAX_EVENT_BYTES, Result,
4    error::invalid, event_type as code, parser::ParserConfig,
5};
6use mysql::{
7    BinlogDumpFlags, BinlogRequest, BinlogStream, Conn, Opts, OptsBuilder, SslOpts,
8    prelude::Queryable,
9};
10use mysql_common::binlog::{
11    consts::BinlogChecksumAlg,
12    events::{BinlogEventHeader, Event as DriverEvent},
13};
14use std::{path::PathBuf, time::Duration};
15
16#[derive(Debug, Clone)]
17pub struct ConnectionConfig {
18    pub url_env: String,
19    pub tls_ca: Option<PathBuf>,
20    pub connect_timeout: Duration,
21    pub read_timeout: Duration,
22}
23impl ConnectionConfig {
24    pub fn new(url_env: impl Into<String>) -> Self {
25        Self {
26            url_env: url_env.into(),
27            tls_ca: None,
28            connect_timeout: Duration::from_secs(10),
29            read_timeout: Duration::from_secs(90),
30        }
31    }
32}
33pub fn connect(config: &ConnectionConfig) -> Result<Conn> {
34    let url = std::env::var(&config.url_env)
35        .map_err(|_| invalid(format!("missing environment variable {}", config.url_env)))?;
36    let opts = Opts::from_url(&url).map_err(|_| invalid("invalid MySQL URL"))?;
37    let mut builder = OptsBuilder::from_opts(opts)
38        .tcp_connect_timeout(Some(config.connect_timeout))
39        .read_timeout(Some(config.read_timeout));
40    if let Some(ca) = &config.tls_ca {
41        builder = builder.ssl_opts(SslOpts::default().with_root_cert_path(Some(ca.clone())));
42    }
43    Ok(Conn::new(builder)?)
44}
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum RowImage {
47    Minimal,
48    Full,
49    NoBlob,
50}
51impl RowImage {
52    fn value(self) -> &'static str {
53        match self {
54            Self::Minimal => "MINIMAL",
55            Self::Full => "FULL",
56            Self::NoBlob => "NOBLOB",
57        }
58    }
59}
60#[derive(Debug, Clone)]
61pub struct ReplicationConfig {
62    pub server_id: u32,
63    pub file: String,
64    pub position: u64,
65    pub non_blocking: bool,
66    /// Verifies upstream logging; never changes a global server setting.
67    pub required_row_image: Option<RowImage>,
68    pub parser: ParserConfig,
69}
70impl ReplicationConfig {
71    pub fn new(server_id: u32, file: impl Into<String>, position: u64) -> Self {
72        Self {
73            server_id,
74            file: file.into(),
75            position,
76            non_blocking: false,
77            required_row_image: None,
78            parser: ParserConfig::default(),
79        }
80    }
81}
82
83pub struct ReplicationStream {
84    stream: BinlogStream,
85    current: Option<DriverEvent>,
86    buffer: Vec<u8>,
87    framing: FrameDecoder,
88    parser: BinlogParser,
89}
90impl ReplicationStream {
91    pub fn connect(connection: &ConnectionConfig, replication: ReplicationConfig) -> Result<Self> {
92        Self::from_connection(connect(connection)?, replication)
93    }
94    pub fn from_connection(mut conn: Conn, config: ReplicationConfig) -> Result<Self> {
95        if config.server_id == 0 || config.file.is_empty() || config.position < 4 {
96            return Err(invalid(
97                "replication requires server_id, file and position >= 4",
98            ));
99        }
100        if let Some(image) = config.required_row_image {
101            let actual: Option<String> = conn.query_first("SELECT @@global.binlog_row_image")?;
102            if !actual
103                .as_deref()
104                .is_some_and(|s| s.eq_ignore_ascii_case(image.value()))
105            {
106                return Err(Error::Unsupported(format!(
107                    "requires binlog_row_image={}",
108                    image.value()
109                )));
110            }
111        }
112        conn.query_drop("SET @master_heartbeat_period=30000000000")?;
113        let request = BinlogRequest::new(config.server_id)
114            .with_filename(config.file.as_bytes())
115            .with_pos(config.position)
116            .with_flags(if config.non_blocking {
117                BinlogDumpFlags::BINLOG_DUMP_NON_BLOCK
118            } else {
119                BinlogDumpFlags::empty()
120            });
121        Ok(Self {
122            stream: conn.get_binlog_stream(request)?,
123            current: None,
124            buffer: Vec::new(),
125            framing: FrameDecoder::new(),
126            parser: BinlogParser::new(config.parser),
127        })
128    }
129    /// CDC's fast path: checksum verification and a borrowed body, with no raw
130    /// frame assembly, GTID metadata decoding or row-value materialization.
131    pub fn next_view(&mut self) -> Result<Option<crate::EventView<'_>>> {
132        self.current = self.stream.next().transpose()?;
133        let Some(event) = &self.current else {
134            return Ok(None);
135        };
136        validate_driver(event)?;
137        if event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
138            let algorithm = event
139                .footer()
140                .get_checksum_alg()
141                .map_err(|e| Error::Unsupported(e.to_string()))?;
142            self.framing
143                .set_checksum(algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32));
144        }
145        Ok(Some(crate::EventView {
146            header: header_view(event.header()),
147            body: event.data(),
148        }))
149    }
150    pub fn next_frame(&mut self) -> Result<Option<BinlogFrame<'_>>> {
151        read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)
152    }
153    pub fn next_event(&mut self) -> Result<Option<BinlogEvent<'_>>> {
154        let Some(frame) = read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)? else {
155            return Ok(None);
156        };
157        self.parser.parse(frame).map(Some)
158    }
159}
160fn read_frame<'a>(
161    stream: &mut BinlogStream,
162    buffer: &'a mut Vec<u8>,
163    framing: &mut FrameDecoder,
164) -> Result<Option<BinlogFrame<'a>>> {
165    let Some(event) = stream.next().transpose()? else {
166        return Ok(None);
167    };
168    retain_driver_bytes(&event, buffer)?;
169    framing.decode(buffer).map(Some)
170}
171
172fn header_view(header: BinlogEventHeader) -> crate::EventHeader {
173    crate::EventHeader {
174        timestamp: header.timestamp(),
175        event_type: header.event_type_raw(),
176        server_id: header.server_id(),
177        event_size: header.event_size(),
178        log_pos: header.log_pos(),
179        flags: header.flags_raw(),
180    }
181}
182fn validate_driver(event: &DriverEvent) -> Result<()> {
183    let length = event.header().event_size() as usize;
184    if !(19..=MAX_EVENT_BYTES).contains(&length) {
185        return Err(invalid("event size outside 19..64MiB"));
186    }
187    let algorithm = event
188        .footer()
189        .get_checksum_alg()
190        .map_err(|e| Error::Unsupported(e.to_string()))?;
191    if algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32)
192        && event.checksum().map(u32::from_le_bytes)
193            != Some(event.calc_checksum(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32))
194    {
195        return Err(Error::Checksum);
196    }
197    let trailer = usize::from(event.checksum().is_some()) * 4
198        + usize::from(event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT);
199    if 19 + event.data().len() + trailer != length {
200        return Err(invalid("driver did not preserve the complete wire event"));
201    }
202    Ok(())
203}
204
205// The driver splits the wire bytes into header, body, algorithm and checksum.
206// Retain each original value; Event::write must not be used because it recalculates
207// the checksum and can normalize the original trailer.
208fn retain_driver_bytes(event: &DriverEvent, buffer: &mut Vec<u8>) -> Result<()> {
209    let header = event.header();
210    let len = header.event_size() as usize;
211    if !(19..=MAX_EVENT_BYTES).contains(&len) {
212        return Err(invalid("event size outside 19..64MiB"));
213    }
214    buffer.clear();
215    buffer.reserve(len);
216    buffer.extend_from_slice(&header.timestamp().to_le_bytes());
217    buffer.push(header.event_type_raw());
218    buffer.extend_from_slice(&header.server_id().to_le_bytes());
219    buffer.extend_from_slice(&header.event_size().to_le_bytes());
220    buffer.extend_from_slice(&header.log_pos().to_le_bytes());
221    buffer.extend_from_slice(&header.flags_raw().to_le_bytes());
222    buffer.extend_from_slice(event.data());
223    if header.event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
224        let algorithm = event
225            .footer()
226            .get_checksum_alg()
227            .map_err(|e| Error::Unsupported(e.to_string()))?
228            .ok_or_else(|| invalid("FDE checksum algorithm missing"))?;
229        buffer.push(match algorithm {
230            BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_OFF => 0,
231            BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32 => 1,
232        });
233    }
234    if let Some(checksum) = event.checksum() {
235        buffer.extend_from_slice(&checksum);
236    }
237    if buffer.len() != len {
238        return Err(invalid("driver did not preserve the complete wire event"));
239    }
240    Ok(())
241}
242
243#[cfg(test)]
244mod tests {
245    use super::*;
246    use mysql_common::binlog::{consts::BinlogVersion, events::FormatDescriptionEvent};
247    #[test]
248    fn driver_trailer_is_retained_instead_of_recalculated() {
249        let mut body = vec![0; 57 + 43];
250        body[..2].copy_from_slice(&4u16.to_le_bytes());
251        body[2..8].copy_from_slice(b"8.4.0\0");
252        body[56] = 19;
253        for algorithm in [0u8, 1] {
254            let mut raw = vec![0; 19];
255            raw[4] = code::FORMAT_DESCRIPTION_EVENT;
256            raw[5..9].copy_from_slice(&123u32.to_le_bytes());
257            raw[17..19].copy_from_slice(&0x8001u16.to_le_bytes());
258            raw.extend_from_slice(&body);
259            raw.push(algorithm);
260            raw.extend_from_slice(&0x12345678u32.to_le_bytes());
261            let length = raw.len() as u32;
262            raw[9..13].copy_from_slice(&length.to_le_bytes());
263            let fde = FormatDescriptionEvent::new(BinlogVersion::Version4);
264            let event = DriverEvent::read(&fde, &raw[..]).unwrap();
265            let mut retained = Vec::new();
266            retain_driver_bytes(&event, &mut retained).unwrap();
267            assert_eq!(retained, raw);
268        }
269    }
270}