Skip to main content

mysql_replication/source/mysql/
mod.rs

1//! MySQL transport, capability discovery, schema loading and initial position capture.
2use super::{EventStream, ServerInfo, SourcePlugin, SourceSession};
3use crate::{
4    ColumnMeta, Error, Result, TableMeta,
5    config::Config,
6    engine::checkpoint::State,
7    error::invalid,
8    protocol::mysql::{
9        file::EventView,
10        version::{VersionAdapter, VersionRegistry},
11    },
12};
13use ::mysql::{
14    BinlogDumpFlags, BinlogRequest, BinlogStream, Conn, Opts, OptsBuilder, SslOpts, params,
15    prelude::Queryable,
16};
17use mysql_common::binlog::{consts::BinlogChecksumAlg, events::Event};
18use std::{collections::BTreeMap, sync::Arc, time::Duration};
19
20pub struct MysqlSource;
21impl SourcePlugin for MysqlSource {
22    fn name(&self) -> &str {
23        "mysql"
24    }
25    fn connect(
26        &self,
27        config: &Config,
28        versions: &VersionRegistry,
29    ) -> Result<Box<dyn SourceSession>> {
30        let mut conn = connect(config)?;
31        let (uuid, version, adapter) = inspect_server(
32            &mut conn,
33            config.source.server_id,
34            versions,
35            config.source.adapter.as_deref(),
36        )?;
37        let info = ServerInfo {
38            uuid,
39            version,
40            adapter: adapter.name().into(),
41        };
42        Ok(Box::new(MysqlSession {
43            conn,
44            info,
45            adapter,
46        }))
47    }
48}
49struct MysqlSession {
50    conn: Conn,
51    info: ServerInfo,
52    adapter: Arc<dyn VersionAdapter>,
53}
54impl SourceSession for MysqlSession {
55    fn info(&self) -> &ServerInfo {
56        &self.info
57    }
58    fn load_tables(&mut self, config: &Config) -> Result<Vec<TableMeta>> {
59        load_tables(&mut self.conn, config)
60    }
61    fn initialize(&mut self, source_id: &str, config: &Config) -> Result<State> {
62        initialize_with_adapter(
63            &mut self.conn,
64            source_id,
65            config,
66            self.info.uuid.clone(),
67            self.adapter.as_ref(),
68        )
69    }
70    fn start(
71        mut self: Box<Self>,
72        config: &Config,
73        state: &State,
74        once: bool,
75    ) -> Result<Box<dyn EventStream>> {
76        self.conn
77            .query_drop("SET @master_heartbeat_period=30000000000")?;
78        let request = BinlogRequest::new(config.source.server_id)
79            .with_filename(state.file.as_bytes())
80            .with_pos(state.position)
81            .with_flags(if once {
82                BinlogDumpFlags::BINLOG_DUMP_NON_BLOCK
83            } else {
84                BinlogDumpFlags::empty()
85            });
86        Ok(Box::new(MysqlEvents {
87            stream: self.conn.get_binlog_stream(request)?,
88            current: None,
89        }))
90    }
91}
92struct MysqlEvents {
93    stream: BinlogStream,
94    current: Option<Event>,
95}
96impl EventStream for MysqlEvents {
97    fn next_event(&mut self) -> Result<Option<EventView<'_>>> {
98        self.current = self.stream.next().transpose()?;
99        let Some(event) = &self.current else {
100            return Ok(None);
101        };
102        if let Some(alg) = event
103            .footer()
104            .get_checksum_alg()
105            .map_err(|e| Error::Unsupported(e.to_string()))?
106            && alg == BinlogChecksumAlg::BINLOG_CHECKSUM_ALG_CRC32
107            && event.checksum().map(u32::from_le_bytes) != Some(event.calc_checksum(alg))
108        {
109            return Err(Error::Checksum);
110        }
111        let header = event.header();
112        Ok(Some(EventView {
113            event_type: header.event_type_raw(),
114            flags: header.flags_raw(),
115            log_pos: header.log_pos(),
116            data: event.data(),
117        }))
118    }
119}
120
121pub fn connect(config: &Config) -> Result<Conn> {
122    let url = std::env::var(&config.source.url_env).map_err(|_| {
123        Error::Schema(format!(
124            "missing environment variable {}",
125            config.source.url_env
126        ))
127    })?;
128    // Do not include the URL (credentials) in our errors.
129    let opts = Opts::from_url(&url).map_err(|_| Error::Schema("invalid MySQL URL".into()))?;
130    let mut builder = OptsBuilder::from_opts(opts)
131        .tcp_connect_timeout(Some(Duration::from_secs(10)))
132        .read_timeout(Some(Duration::from_secs(90)));
133    if let Some(ca) = &config.source.tls_ca {
134        builder = builder.ssl_opts(SslOpts::default().with_root_cert_path(Some(ca.clone())));
135    }
136    Ok(Conn::new(builder)?)
137}
138
139pub fn inspect_server(
140    conn: &mut Conn,
141    server_id: u32,
142    registry: &crate::protocol::mysql::version::VersionRegistry,
143    selected: Option<&str>,
144) -> Result<(
145    String,
146    crate::protocol::mysql::version::ServerVersion,
147    std::sync::Arc<dyn crate::protocol::mysql::version::VersionAdapter>,
148)> {
149    let raw: String = conn
150        .query_first("SELECT VERSION()")?
151        .ok_or_else(|| invalid("no VERSION()"))?;
152    let version = crate::protocol::mysql::version::ServerVersion::parse(&raw)?;
153    let adapter = registry.resolve(&version, selected)?;
154    let variables: Vec<(String, String)> = conn.query("SHOW GLOBAL VARIABLES WHERE Variable_name IN ('log_bin','binlog_format','binlog_row_image','binlog_transaction_compression','binlog_row_value_options','server_id')")?;
155    adapter.validate_server(
156        &variables.into_iter().collect::<BTreeMap<_, _>>(),
157        server_id,
158    )?;
159    let uuid = conn
160        .query_first("SELECT @@server_uuid")?
161        .ok_or_else(|| invalid("no server_uuid"))?;
162    Ok((uuid, version, adapter))
163}
164pub fn load_tables(conn: &mut Conn, config: &Config) -> Result<Vec<TableMeta>> {
165    let mut tables = Vec::with_capacity(config.tables.len());
166    for table in &config.tables {
167        let params = params! { "schema" => &table.schema, "table" => &table.table };
168        let engine: Option<String> = conn.exec_first("SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA=:schema AND TABLE_NAME=:table", params.clone())?;
169        if engine.as_deref() != Some("InnoDB") {
170            return Err(Error::Unsupported(format!(
171                "{}.{} must exist and use InnoDB",
172                table.schema, table.table
173            )));
174        }
175        let rows: Vec<mysql::Row> = conn.exec("SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY, CHARACTER_SET_NAME, CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, DATETIME_PRECISION, EXTRA FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=:schema AND TABLE_NAME=:table ORDER BY ORDINAL_POSITION", params)?;
176        let mut columns = Vec::with_capacity(rows.len());
177        for row in rows {
178            type InfoRow = (
179                String,
180                String,
181                String,
182                String,
183                String,
184                Option<String>,
185                Option<u64>,
186                Option<u32>,
187                Option<u32>,
188                Option<u32>,
189                String,
190            );
191            let (
192                name,
193                data_type,
194                column_type,
195                nullable,
196                key,
197                charset,
198                octet_length,
199                numeric_precision,
200                numeric_scale,
201                datetime_precision,
202                extra,
203            ): InfoRow = mysql::from_row_opt(row).map_err(|e| Error::Schema(e.to_string()))?;
204            columns.push(ColumnMeta {
205                name,
206                data_type,
207                column_type,
208                nullable: nullable == "YES",
209                primary_key: key == "PRI",
210                charset,
211                octet_length,
212                numeric_precision,
213                numeric_scale,
214                datetime_precision,
215                extra,
216            });
217        }
218        let meta = TableMeta {
219            schema: table.schema.clone(),
220            table: table.table.clone(),
221            columns,
222        };
223        meta.validate()?;
224        tables.push(meta);
225    }
226    Ok(tables)
227}
228
229/// Capture source identity, schema and commit boundary under a brief global read lock.
230fn initialize_with_adapter(
231    conn: &mut Conn,
232    source_id: &str,
233    config: &Config,
234    source_uuid: String,
235    adapter: &dyn crate::protocol::mysql::version::VersionAdapter,
236) -> Result<State> {
237    conn.query_drop("FLUSH TABLES WITH READ LOCK")?;
238    let result: Result<State> = (|| {
239        let tables = load_tables(conn, config)?;
240        let rows: Vec<mysql::Row> = conn.query(adapter.status_query())?;
241        let row = rows
242            .first()
243            .ok_or_else(|| invalid("no binary log status"))?;
244        let file = row
245            .get_opt::<String, _>(0)
246            .ok_or_else(|| invalid("missing log file"))?
247            .map_err(|e| invalid(e.to_string()))?;
248        let position = row
249            .get_opt::<u64, _>(1)
250            .ok_or_else(|| invalid("missing log position"))?
251            .map_err(|e| invalid(e.to_string()))?;
252        let state = State {
253            version: 1,
254            source_uuid,
255            source_id: source_id.into(),
256            source_plugin: config.source.plugin.clone(),
257            file,
258            position,
259            tables,
260        };
261        state.validate()?;
262        Ok(state)
263    })();
264    let unlock = conn.query_drop("UNLOCK TABLES");
265    let state = result?;
266    unlock?;
267    Ok(state)
268}