mysql_replication/engine/
runtime.rs1use crate::{
2 Error, Result,
3 config::{AppConfig, SourceEntry},
4 engine::checkpoint::{State, StateLock},
5 engine::processor::Processor,
6 protocol::mysql::version::VersionRegistry,
7 sink::TransactionSink,
8 source::{SourceRegistry, SourceSession},
9};
10use std::{
11 collections::HashSet,
12 sync::{
13 Arc,
14 atomic::{AtomicBool, Ordering},
15 },
16};
17
18#[derive(Default)]
19pub struct Runtime {
20 pub sources: SourceRegistry,
21 pub versions: VersionRegistry,
22}
23struct Prepared {
24 entry: SourceEntry,
25 session: Box<dyn SourceSession>,
26 state: State,
27 _lock: StateLock,
28}
29impl Runtime {
30 fn connect_all(&self, config: &AppConfig) -> Result<Vec<Box<dyn SourceSession>>> {
31 config.validate()?;
32 let mut identities = HashSet::new();
33 let mut sessions = Vec::with_capacity(config.sources.len());
34 for entry in &config.sources {
35 let session = self
36 .sources
37 .resolve(&entry.config.source.plugin)?
38 .connect(&entry.config, &self.versions)
39 .map_err(|e| Error::Sources(format!("{}: {e}", entry.id)))?;
40 let info = session.info();
41 if info.uuid.is_empty()
42 || !identities.insert((info.uuid.clone(), entry.config.source.server_id))
43 {
44 return Err(Error::Schema(
45 "same upstream UUID requires distinct replication server_id values".into(),
46 ));
47 }
48 eprintln!(
49 "source {}: {} / {} / UUID {}",
50 entry.id, info.version.raw, info.adapter, info.uuid
51 );
52 sessions.push(session);
53 }
54 Ok(sessions)
55 }
56 pub fn check(&self, config: &AppConfig) -> Result<()> {
57 for (entry, mut session) in config.sources.iter().zip(self.connect_all(config)?) {
58 let tables = session.load_tables(&entry.config)?;
59 eprintln!(
60 "source {}: validated {} whitelist tables",
61 entry.id,
62 tables.len()
63 );
64 }
65 Ok(())
66 }
67 fn prepare(&self, config: AppConfig, initialize_only: bool) -> Result<Vec<Prepared>> {
68 let sessions = self.connect_all(&config)?;
69 let locks: Vec<_> = config
71 .sources
72 .iter()
73 .map(|s| StateLock::acquire(&s.config.state_file))
74 .collect::<Result<_>>()?;
75 let mut saved = Vec::new();
76 for (entry, session) in config.sources.iter().zip(&sessions) {
77 let state = if entry.config.state_file.exists() {
78 if initialize_only {
79 return Err(Error::Schema(format!(
80 "{}: state file exists; initialize with a new state_file",
81 entry.id
82 )));
83 }
84 let state = State::read(&entry.config.state_file)?;
85 state.verify_whitelist(&entry.config)?;
86 if state.source_uuid != session.info().uuid
87 || state.source_id != entry.id
88 || state.source_plugin != entry.config.source.plugin
89 {
90 return Err(Error::Schema(format!(
91 "{}: checkpoint source UUID/id/plugin mismatch",
92 entry.id
93 )));
94 }
95 Some(state)
96 } else {
97 None
98 };
99 saved.push(state);
100 }
101 let mut ready = Vec::new();
102 for (((entry, mut session), lock), saved) in config
103 .sources
104 .into_iter()
105 .zip(sessions)
106 .zip(locks)
107 .zip(saved)
108 {
109 let state = match saved {
110 Some(state) => state,
111 None => session.initialize(&entry.id, &entry.config)?,
112 };
113 state.validate()?;
114 state.verify_whitelist(&entry.config)?;
115 if state.source_uuid != session.info().uuid
116 || state.source_id != entry.id
117 || state.source_plugin != entry.config.source.plugin
118 {
119 return Err(Error::Schema(format!(
120 "{}: initialized checkpoint UUID/id/plugin differs from configured source",
121 entry.id
122 )));
123 }
124 state.save(&entry.config.state_file)?;
125 ready.push(Prepared {
126 entry,
127 session,
128 state,
129 _lock: lock,
130 });
131 }
132 Ok(ready)
133 }
134 pub fn initialize(&self, config: AppConfig) -> Result<()> {
135 self.prepare(config, true)?;
136 Ok(())
137 }
138 pub fn run(&self, config: AppConfig, once: bool, sink: Arc<dyn TransactionSink>) -> Result<()> {
141 let prepared = self.prepare(config, false)?;
142 let cancelled = Arc::new(AtomicBool::new(false));
143 std::thread::scope(|scope| {
144 let mut workers = Vec::new();
145 for prepared in prepared {
146 let sink = Arc::clone(&sink);
147 let cancelled = Arc::clone(&cancelled);
148 let registry = self.versions.clone();
149 workers.push(scope.spawn(move || {
150 let _panic_guard = CancelOnPanic(&cancelled);
151 let Prepared {
152 entry,
153 session,
154 state,
155 _lock,
156 } = prepared;
157 let result = (|| {
158 let mut processor = Processor::with_registry(
159 state,
160 Some(entry.config.state_file.clone()),
161 entry.config.max_transaction_bytes,
162 registry,
163 entry.config.source.adapter.clone(),
164 )?;
165 let mut stream = session.start(&entry.config, processor.state(), once)?;
166 while !cancelled.load(Ordering::Acquire) {
167 let Some(event) = stream.next_event()? else {
168 return processor.finish();
169 };
170 if cancelled.load(Ordering::Acquire) {
171 return Ok(());
172 }
173 processor.process_to(event, sink.as_ref())?;
174 }
175 Ok(())
176 })();
177 if let Err(error) = &result {
178 cancelled.store(true, Ordering::Release);
179 eprintln!("source {} stopped: {error}", entry.id);
180 }
181 result.map_err(|e| format!("{}: {e}", entry.id))
182 }));
183 }
184 let mut failures = Vec::new();
185 for worker in workers {
186 match worker.join() {
187 Ok(Ok(())) => {}
188 Ok(Err(error)) => failures.push(error),
189 Err(_) => {
190 cancelled.store(true, Ordering::Release);
191 failures.push("source worker panicked".into());
192 }
193 }
194 }
195 if failures.is_empty() {
196 Ok(())
197 } else {
198 Err(Error::Sources(failures.join("; ")))
199 }
200 })
201 }
202}
203
204struct CancelOnPanic<'a>(&'a AtomicBool);
205impl Drop for CancelOnPanic<'_> {
206 fn drop(&mut self) {
207 if std::thread::panicking() {
208 self.0.store(true, Ordering::Release);
209 }
210 }
211}