1use crate::{
3 BinlogEvent, BinlogFrame, BinlogParser, Error, FrameDecoder, MAX_EVENT_BYTES, Result,
4 error::invalid,
5 event_type as code,
6 events::{BinlogMode, ImageMode, Unspecified},
7 parser::ParserConfig,
8};
9use mysql::{
10 BinlogDumpFlags, BinlogRequest, BinlogStream, Conn, Opts, OptsBuilder, SslOpts,
11 prelude::Queryable,
12};
13use mysql_common::binlog::{
14 consts::BinlogChecksumAlg,
15 events::{BinlogEventHeader, Event as DriverEvent},
16};
17use std::{path::PathBuf, time::Duration};
18
19#[derive(Debug, Clone)]
20pub struct ConnectionConfig {
21 pub url_env: String,
22 pub tls_ca: Option<PathBuf>,
23 pub connect_timeout: Duration,
24 pub read_timeout: Duration,
25}
26impl ConnectionConfig {
27 pub fn new(url_env: impl Into<String>) -> Self {
28 Self {
29 url_env: url_env.into(),
30 tls_ca: None,
31 connect_timeout: Duration::from_secs(10),
32 read_timeout: Duration::from_secs(90),
33 }
34 }
35}
36pub fn connect(config: &ConnectionConfig) -> Result<Conn> {
37 let url = std::env::var(&config.url_env)
38 .map_err(|_| invalid(format!("missing environment variable {}", config.url_env)))?;
39 let opts = Opts::from_url(&url).map_err(|_| invalid("invalid MySQL URL"))?;
40 let mut builder = OptsBuilder::from_opts(opts)
41 .tcp_connect_timeout(Some(config.connect_timeout))
42 .read_timeout(Some(config.read_timeout));
43 if let Some(ca) = &config.tls_ca {
44 builder = builder.ssl_opts(SslOpts::default().with_root_cert_path(Some(ca.clone())));
45 }
46 Ok(Conn::new(builder)?)
47}
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum RowImage {
50 Minimal,
51 Full,
52 NoBlob,
53}
54impl RowImage {
55 fn value(self) -> &'static str {
56 match self {
57 Self::Minimal => "MINIMAL",
58 Self::Full => "FULL",
59 Self::NoBlob => "NOBLOB",
60 }
61 }
62}
63#[derive(Debug, Clone)]
64pub struct ReplicationConfig {
65 pub server_id: u32,
66 pub file: String,
67 pub position: u64,
68 pub non_blocking: bool,
69 pub required_row_image: Option<RowImage>,
71 pub parser: ParserConfig,
72}
73impl ReplicationConfig {
74 pub fn new(server_id: u32, file: impl Into<String>, position: u64) -> Self {
75 Self {
76 server_id,
77 file: file.into(),
78 position,
79 non_blocking: false,
80 required_row_image: None,
81 parser: ParserConfig::default(),
82 }
83 }
84}
85
86pub struct ReplicationStream<M: BinlogMode = Unspecified> {
87 stream: BinlogStream,
88 current: Option<DriverEvent>,
89 buffer: Vec<u8>,
90 framing: FrameDecoder,
91 parser: BinlogParser<M>,
92}
93impl ReplicationStream {
94 pub fn connect_with_tables<'a>(
97 connection: &ConnectionConfig,
98 replication: ReplicationConfig,
99 tables: impl IntoIterator<Item = (&'a str, &'a str)>,
100 ) -> Result<Self> {
101 Self::from_connection_with_tables(connect(connection)?, replication, tables)
102 }
103 pub fn from_connection_with_tables<'a>(
106 mut conn: Conn,
107 config: ReplicationConfig,
108 tables: impl IntoIterator<Item = (&'a str, &'a str)>,
109 ) -> Result<Self> {
110 crate::primary_key::validate_primary_keys(&mut conn, tables)?;
111 Self::from_connection(conn, config)
112 }
113 pub fn connect(connection: &ConnectionConfig, replication: ReplicationConfig) -> Result<Self> {
114 Self::from_connection(connect(connection)?, replication)
115 }
116 pub fn from_connection(mut conn: Conn, config: ReplicationConfig) -> Result<Self> {
117 if config.server_id == 0 || config.file.is_empty() || config.position < 4 {
118 return Err(invalid(
119 "replication requires server_id, file and position >= 4",
120 ));
121 }
122 if let Some(image) = config.required_row_image {
123 let actual: Option<String> = conn.query_first("SELECT @@global.binlog_row_image")?;
124 if !actual
125 .as_deref()
126 .is_some_and(|s| s.eq_ignore_ascii_case(image.value()))
127 {
128 return Err(Error::Unsupported(format!(
129 "requires binlog_row_image={}",
130 image.value()
131 )));
132 }
133 }
134 conn.query_drop("SET @master_heartbeat_period=30000000000")?;
135 let request = BinlogRequest::new(config.server_id)
136 .with_filename(config.file.as_bytes())
137 .with_pos(config.position)
138 .with_flags(if config.non_blocking {
139 BinlogDumpFlags::BINLOG_DUMP_NON_BLOCK
140 } else {
141 BinlogDumpFlags::empty()
142 });
143 Ok(Self {
144 stream: conn.get_binlog_stream(request)?,
145 current: None,
146 buffer: Vec::new(),
147 framing: FrameDecoder::new(),
148 parser: BinlogParser::new(config.parser),
149 })
150 }
151 pub fn connect_with_mode<M: ImageMode>(
153 connection: &ConnectionConfig,
154 replication: ReplicationConfig,
155 ) -> Result<ReplicationStream<M>> {
156 let replication = bind_image_mode::<M>(replication)?;
157 Self::from_connection_with_mode(connect(connection)?, replication)
158 }
159 pub fn from_connection_with_mode<M: ImageMode>(
160 conn: Conn,
161 mut config: ReplicationConfig,
162 ) -> Result<ReplicationStream<M>> {
163 config = bind_image_mode::<M>(config)?;
164 let stream = Self::from_connection(conn, config)?;
165 Ok(ReplicationStream {
166 stream: stream.stream,
167 current: stream.current,
168 buffer: stream.buffer,
169 framing: stream.framing,
170 parser: stream.parser.with_mode(),
171 })
172 }
173}
174impl<M: ImageMode> ReplicationStream<M> {
175 pub fn for_mode(connection: &ConnectionConfig, config: ReplicationConfig) -> Result<Self> {
176 ReplicationStream::connect_with_mode::<M>(connection, config)
177 }
178}
179impl<M: BinlogMode> ReplicationStream<M> {
180 pub fn next_view(&mut self) -> Result<Option<crate::EventView<'_>>> {
183 self.current = self.stream.next().transpose()?;
184 let Some(event) = &self.current else {
185 return Ok(None);
186 };
187 validate_driver(event)?;
188 if event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
189 let algorithm = event
190 .footer()
191 .get_checksum_alg()
192 .map_err(|e| Error::Unsupported(e.to_string()))?;
193 self.framing
194 .set_checksum(algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32));
195 }
196 Ok(Some(crate::EventView {
197 header: header_view(event.header()),
198 body: event.data(),
199 }))
200 }
201 pub fn next_frame(&mut self) -> Result<Option<BinlogFrame<'_>>> {
202 read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)
203 }
204 pub fn next_event(&mut self) -> Result<Option<BinlogEvent<'_, M>>> {
205 let Some(frame) = read_frame(&mut self.stream, &mut self.buffer, &mut self.framing)? else {
206 return Ok(None);
207 };
208 self.parser.parse(frame).map(Some)
209 }
210}
211fn bind_image_mode<M: ImageMode>(mut config: ReplicationConfig) -> Result<ReplicationConfig> {
212 let expected = M::ROW_IMAGE.into();
213 if config
214 .required_row_image
215 .is_some_and(|mode| mode != expected)
216 {
217 return Err(invalid(
218 "replication configuration conflicts with the generic image mode",
219 ));
220 }
221 config.required_row_image = Some(expected);
222 Ok(config)
223}
224
225fn read_frame<'a>(
226 stream: &mut BinlogStream,
227 buffer: &'a mut Vec<u8>,
228 framing: &mut FrameDecoder,
229) -> Result<Option<BinlogFrame<'a>>> {
230 let Some(event) = stream.next().transpose()? else {
231 return Ok(None);
232 };
233 retain_driver_bytes(&event, buffer)?;
234 framing.decode(buffer).map(Some)
235}
236
237fn header_view(header: BinlogEventHeader) -> crate::EventHeader {
238 crate::EventHeader {
239 timestamp: header.timestamp(),
240 event_type: header.event_type_raw(),
241 server_id: header.server_id(),
242 event_size: header.event_size(),
243 log_pos: header.log_pos(),
244 flags: header.flags_raw(),
245 }
246}
247fn validate_driver(event: &DriverEvent) -> Result<()> {
248 let length = event.header().event_size() as usize;
249 if !(19..=MAX_EVENT_BYTES).contains(&length) {
250 return Err(invalid("event size outside 19..64MiB"));
251 }
252 let algorithm = event
253 .footer()
254 .get_checksum_alg()
255 .map_err(|e| Error::Unsupported(e.to_string()))?;
256 if algorithm == Some(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32)
257 && event.checksum().map(u32::from_le_bytes)
258 != Some(event.calc_checksum(BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32))
259 {
260 return Err(Error::Checksum);
261 }
262 let trailer = usize::from(event.checksum().is_some()) * 4
263 + usize::from(event.header().event_type_raw() == code::FORMAT_DESCRIPTION_EVENT);
264 if 19 + event.data().len() + trailer != length {
265 return Err(invalid("driver did not preserve the complete wire event"));
266 }
267 Ok(())
268}
269
270fn retain_driver_bytes(event: &DriverEvent, buffer: &mut Vec<u8>) -> Result<()> {
274 let header = event.header();
275 let len = header.event_size() as usize;
276 if !(19..=MAX_EVENT_BYTES).contains(&len) {
277 return Err(invalid("event size outside 19..64MiB"));
278 }
279 buffer.clear();
280 buffer.reserve(len);
281 buffer.extend_from_slice(&header.timestamp().to_le_bytes());
282 buffer.push(header.event_type_raw());
283 buffer.extend_from_slice(&header.server_id().to_le_bytes());
284 buffer.extend_from_slice(&header.event_size().to_le_bytes());
285 buffer.extend_from_slice(&header.log_pos().to_le_bytes());
286 buffer.extend_from_slice(&header.flags_raw().to_le_bytes());
287 buffer.extend_from_slice(event.data());
288 if header.event_type_raw() == code::FORMAT_DESCRIPTION_EVENT {
289 let algorithm = event
290 .footer()
291 .get_checksum_alg()
292 .map_err(|e| Error::Unsupported(e.to_string()))?
293 .ok_or_else(|| invalid("FDE checksum algorithm missing"))?;
294 buffer.push(match algorithm {
295 BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_OFF => 0,
296 BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32 => 1,
297 });
298 }
299 if let Some(checksum) = event.checksum() {
300 buffer.extend_from_slice(&checksum);
301 }
302 if buffer.len() != len {
303 return Err(invalid("driver did not preserve the complete wire event"));
304 }
305 Ok(())
306}
307
308#[cfg(test)]
309mod tests {
310 use super::*;
311 use mysql_common::binlog::{consts::BinlogVersion, events::FormatDescriptionEvent};
312 #[test]
313 fn driver_trailer_is_retained_instead_of_recalculated() {
314 let mut body = vec![0; 57 + 43];
315 body[..2].copy_from_slice(&4u16.to_le_bytes());
316 body[2..8].copy_from_slice(b"8.4.0\0");
317 body[56] = 19;
318 for algorithm in [0u8, 1] {
319 let mut raw = vec![0; 19];
320 raw[4] = code::FORMAT_DESCRIPTION_EVENT;
321 raw[5..9].copy_from_slice(&123u32.to_le_bytes());
322 raw[17..19].copy_from_slice(&0x8001u16.to_le_bytes());
323 raw.extend_from_slice(&body);
324 raw.push(algorithm);
325 raw.extend_from_slice(&0x12345678u32.to_le_bytes());
326 let length = raw.len() as u32;
327 raw[9..13].copy_from_slice(&length.to_le_bytes());
328 let fde = FormatDescriptionEvent::new(BinlogVersion::Version4);
329 let event = DriverEvent::read(&fde, &raw[..]).unwrap();
330 let mut retained = Vec::new();
331 retain_driver_bytes(&event, &mut retained).unwrap();
332 assert_eq!(retained, raw);
333 }
334 }
335}