mysql_replication/source/mysql/
mod.rs1use super::{EventStream, ServerInfo, SourcePlugin, SourceSession};
3use crate::{
4 ColumnMeta, Error, Result, TableMeta,
5 config::Config,
6 engine::checkpoint::State,
7 error::invalid,
8 protocol::mysql::{
9 file::EventView,
10 version::{VersionAdapter, VersionRegistry},
11 },
12};
13use ::mysql::{
14 BinlogDumpFlags, BinlogRequest, BinlogStream, Conn, Opts, OptsBuilder, SslOpts, params,
15 prelude::Queryable,
16};
17use mysql_common::binlog::{consts::BinlogChecksumAlg, events::Event};
18use std::{collections::BTreeMap, sync::Arc, time::Duration};
19
20pub struct MysqlSource;
21impl SourcePlugin for MysqlSource {
22 fn name(&self) -> &str {
23 "mysql"
24 }
25 fn connect(
26 &self,
27 config: &Config,
28 versions: &VersionRegistry,
29 ) -> Result<Box<dyn SourceSession>> {
30 let mut conn = connect(config)?;
31 let (uuid, version, adapter) = inspect_server(
32 &mut conn,
33 config.source.server_id,
34 versions,
35 config.source.adapter.as_deref(),
36 )?;
37 let info = ServerInfo {
38 uuid,
39 version,
40 adapter: adapter.name().into(),
41 };
42 Ok(Box::new(MysqlSession {
43 conn,
44 info,
45 adapter,
46 }))
47 }
48}
49struct MysqlSession {
50 conn: Conn,
51 info: ServerInfo,
52 adapter: Arc<dyn VersionAdapter>,
53}
54impl SourceSession for MysqlSession {
55 fn info(&self) -> &ServerInfo {
56 &self.info
57 }
58 fn load_tables(&mut self, config: &Config) -> Result<Vec<TableMeta>> {
59 load_tables(&mut self.conn, config)
60 }
61 fn initialize(&mut self, source_id: &str, config: &Config) -> Result<State> {
62 initialize_with_adapter(
63 &mut self.conn,
64 source_id,
65 config,
66 self.info.uuid.clone(),
67 self.adapter.as_ref(),
68 )
69 }
70 fn start(
71 mut self: Box<Self>,
72 config: &Config,
73 state: &State,
74 once: bool,
75 ) -> Result<Box<dyn EventStream>> {
76 self.conn
77 .query_drop("SET @master_heartbeat_period=30000000000")?;
78 let request = BinlogRequest::new(config.source.server_id)
79 .with_filename(state.file.as_bytes())
80 .with_pos(state.position)
81 .with_flags(if once {
82 BinlogDumpFlags::BINLOG_DUMP_NON_BLOCK
83 } else {
84 BinlogDumpFlags::empty()
85 });
86 Ok(Box::new(MysqlEvents {
87 stream: self.conn.get_binlog_stream(request)?,
88 current: None,
89 }))
90 }
91}
92struct MysqlEvents {
93 stream: BinlogStream,
94 current: Option<Event>,
95}
96impl EventStream for MysqlEvents {
97 fn next_event(&mut self) -> Result<Option<EventView<'_>>> {
98 self.current = self.stream.next().transpose()?;
99 let Some(event) = &self.current else {
100 return Ok(None);
101 };
102 if let Some(alg) = event
103 .footer()
104 .get_checksum_alg()
105 .map_err(|e| Error::Unsupported(e.to_string()))?
106 && alg == BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32
107 && event.checksum().map(u32::from_le_bytes) != Some(event.calc_checksum(alg))
108 {
109 return Err(Error::Checksum);
110 }
111 let header = event.header();
112 Ok(Some(EventView {
113 event_type: header.event_type_raw(),
114 flags: header.flags_raw(),
115 log_pos: header.log_pos(),
116 data: event.data(),
117 }))
118 }
119}
120
121pub fn connect(config: &Config) -> Result<Conn> {
122 let url = std::env::var(&config.source.url_env).map_err(|_| {
123 Error::Schema(format!(
124 "missing environment variable {}",
125 config.source.url_env
126 ))
127 })?;
128 let opts = Opts::from_url(&url).map_err(|_| Error::Schema("invalid MySQL URL".into()))?;
130 let mut builder = OptsBuilder::from_opts(opts)
131 .tcp_connect_timeout(Some(Duration::from_secs(10)))
132 .read_timeout(Some(Duration::from_secs(90)));
133 if let Some(ca) = &config.source.tls_ca {
134 builder = builder.ssl_opts(SslOpts::default().with_root_cert_path(Some(ca.clone())));
135 }
136 Ok(Conn::new(builder)?)
137}
138
139pub fn inspect_server(
140 conn: &mut Conn,
141 server_id: u32,
142 registry: &crate::protocol::mysql::version::VersionRegistry,
143 selected: Option<&str>,
144) -> Result<(
145 String,
146 crate::protocol::mysql::version::ServerVersion,
147 std::sync::Arc<dyn crate::protocol::mysql::version::VersionAdapter>,
148)> {
149 let raw: String = conn
150 .query_first("SELECT VERSION()")?
151 .ok_or_else(|| invalid("no VERSION()"))?;
152 let version = crate::protocol::mysql::version::ServerVersion::parse(&raw)?;
153 let adapter = registry.resolve(&version, selected)?;
154 let variables: Vec<(String, String)> = conn.query("SHOW GLOBAL VARIABLES WHERE Variable_name IN ('log_bin','binlog_format','binlog_row_image','binlog_transaction_compression','binlog_row_value_options','server_id')")?;
155 adapter.validate_server(
156 &variables.into_iter().collect::<BTreeMap<_, _>>(),
157 server_id,
158 )?;
159 let uuid = conn
160 .query_first("SELECT @@server_uuid")?
161 .ok_or_else(|| invalid("no server_uuid"))?;
162 Ok((uuid, version, adapter))
163}
164pub fn load_tables(conn: &mut Conn, config: &Config) -> Result<Vec<TableMeta>> {
165 let mut tables = Vec::with_capacity(config.tables.len());
166 for table in &config.tables {
167 let params = params! { "schema" => &table.schema, "table" => &table.table };
168 let engine: Option<String> = conn.exec_first("SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA=:schema AND TABLE_NAME=:table", params.clone())?;
169 if engine.as_deref() != Some("InnoDB") {
170 return Err(Error::Unsupported(format!(
171 "{}.{} must exist and use InnoDB",
172 table.schema, table.table
173 )));
174 }
175 let rows: Vec<mysql::Row> = conn.exec("SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY, CHARACTER_SET_NAME, CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, EXTRA FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=:schema AND TABLE_NAME=:table ORDER BY ORDINAL_POSITION", params)?;
176 let mut columns = Vec::with_capacity(rows.len());
177 for row in rows {
178 type InfoRow = (
179 String,
180 String,
181 String,
182 String,
183 String,
184 Option<String>,
185 Option<u64>,
186 Option<u32>,
187 Option<u32>,
188 Option<u32>,
189 String,
190 );
191 let (
192 name,
193 data_type,
194 column_type,
195 nullable,
196 key,
197 charset,
198 octet_length,
199 numeric_precision,
200 numeric_scale,
201 datetime_precision,
202 extra,
203 ): InfoRow = mysql::from_row_opt(row).map_err(|e| Error::Schema(e.to_string()))?;
204 columns.push(ColumnMeta {
205 name,
206 data_type,
207 column_type,
208 nullable: nullable == "YES",
209 primary_key: key == "PRI",
210 charset,
211 octet_length,
212 numeric_precision,
213 numeric_scale,
214 datetime_precision,
215 extra,
216 });
217 }
218 let meta = TableMeta {
219 schema: table.schema.clone(),
220 table: table.table.clone(),
221 columns,
222 };
223 meta.validate()?;
224 tables.push(meta);
225 }
226 Ok(tables)
227}
228
229fn initialize_with_adapter(
231 conn: &mut Conn,
232 source_id: &str,
233 config: &Config,
234 source_uuid: String,
235 adapter: &dyn crate::protocol::mysql::version::VersionAdapter,
236) -> Result<State> {
237 conn.query_drop("FLUSH TABLES WITH READ LOCK")?;
238 let result: Result<State> = (|| {
239 let tables = load_tables(conn, config)?;
240 let rows: Vec<mysql::Row> = conn.query(adapter.status_query())?;
241 let row = rows
242 .first()
243 .ok_or_else(|| invalid("no binary log status"))?;
244 let file = row
245 .get_opt::<String, _>(0)
246 .ok_or_else(|| invalid("missing log file"))?
247 .map_err(|e| invalid(e.to_string()))?;
248 let position = row
249 .get_opt::<u64, _>(1)
250 .ok_or_else(|| invalid("missing log position"))?
251 .map_err(|e| invalid(e.to_string()))?;
252 let state = State {
253 version: 1,
254 source_uuid,
255 source_id: source_id.into(),
256 source_plugin: config.source.plugin.clone(),
257 file,
258 position,
259 tables,
260 };
261 state.validate()?;
262 Ok(state)
263 })();
264 let unlock = conn.query_drop("UNLOCK TABLES");
265 let state = result?;
266 unlock?;
267 Ok(state)
268}