halephant_core/proxy/transaction.rs
1//! Transaction-mode proxy: the outer loop cycles between an idle
2//! state (no server connection) and an active state (holding a
3//! checked-out [`crate::connections::server::ServerConn`] for the
4//! duration of a single transaction). On `ReadyForQuery(Idle)` the
5//! connection is returned to the pool; on `Terminate` or disconnect
6//! the loop exits.
7
8use std::sync::Arc;
9
10use futures_util::{SinkExt, StreamExt};
11use tokio::net::TcpStream;
12use tokio_util::codec::Framed;
13use tracing::{Instrument, debug, field, info_span, trace, warn};
14
15use crate::clients::{ClientGuard, ClientState};
16use crate::config::cluster::pool::ListenMode;
17use crate::config::otel::QueryText;
18use crate::connections::server::{Reply, ServerConn};
19use crate::listener::ClientNotifications;
20use crate::o11y;
21use crate::pool::PoolManager;
22use crate::proto;
23use crate::proxy::intercept::{
24 handle_close_intercept, handle_deallocate_intercept, handle_listen_intercept,
25};
26use crate::proxy::prepared::{ClientPrepared, PreparedGuard, rewrite_outbound};
27use crate::sql;
28
29/// Track session-state-affecting commands on the server connection so
30/// halephant's per-connection pool state stays in sync with what
31/// PostgreSQL actually holds. Two trackers are kept in step here:
32///
33/// - `server.dirty_vars` — GUC variables the client has SET and that
34/// `reset_connection` must RESET on checkin.
35/// - `server.statements` — canonical names of prepared statements
36/// registered on this backend via `ensure_prepared`.
37///
38/// `stmt` is the caller's one-per-message classification, and callers
39/// only pass classifications of simple-protocol Query messages — a SET
40/// inside a Parse is not tracked because it hasn't been executed yet.
41fn track_set_reset(stmt: &sql::Statement, server: &mut ServerConn) {
42 // SET LOCAL falls into the wildcard arm below: its effect is
43 // transaction-scoped and unwound automatically on COMMIT/ROLLBACK,
44 // so there's nothing to track in dirty_vars.
45 match stmt {
46 sql::Statement::Set {
47 scope: sql::SetScope::Session,
48 parameter,
49 ..
50 } => {
51 server.dirty_vars.insert(parameter.clone());
52 }
53 sql::Statement::Reset {
54 target: sql::ResetTarget::All,
55 } => {
56 server.dirty_vars.clear();
57 }
58 sql::Statement::Reset {
59 target: sql::ResetTarget::Parameter(name),
60 } => {
61 server.dirty_vars.remove(name);
62 }
63 sql::Statement::Discard {
64 target: sql::DiscardTarget::All,
65 } => {
66 server.statements.discard_all();
67 server.dirty_vars.clear();
68 }
69 _ => {}
70 }
71}
72
73/// Why we returned from the active forwarding state.
74enum Release {
75 /// Server sent ReadyForQuery(Idle) — transaction complete, connection can
76 /// be returned to the pool. The client remains connected and may start
77 /// another transaction.
78 Idle,
79 /// Client sent Terminate or disconnected.
80 ClientDone,
81}
82
83/// Outer loop: alternate between idle (no server connection) and active
84/// (forwarding a transaction).
85#[allow(clippy::too_many_arguments)]
86pub async fn forward(
87 client: &mut Framed<TcpStream, proto::codec::FrontendCodec>,
88 pools: &Arc<PoolManager>,
89 client_guard: &ClientGuard,
90 notifications: &mut Option<ClientNotifications>,
91 listen_mode: ListenMode,
92 database: &str,
93 user: &str,
94 read_only: bool,
95 query_text: QueryText,
96) -> anyhow::Result<()> {
97 let mut client_prepared = PreparedGuard {
98 inner: ClientPrepared::new(),
99 pools: Arc::clone(pools),
100 };
101 forward_loop(
102 client,
103 pools,
104 client_guard,
105 notifications,
106 listen_mode,
107 database,
108 user,
109 read_only,
110 &mut client_prepared.inner,
111 query_text,
112 )
113 .await
114}
115
116#[allow(clippy::too_many_arguments)]
117async fn forward_loop(
118 client: &mut Framed<TcpStream, proto::codec::FrontendCodec>,
119 pools: &Arc<PoolManager>,
120 client_guard: &ClientGuard,
121 notifications: &mut Option<ClientNotifications>,
122 listen_mode: ListenMode,
123 database: &str,
124 user: &str,
125 read_only: bool,
126 client_prepared: &mut ClientPrepared,
127 query_text: QueryText,
128) -> anyhow::Result<()> {
129 loop {
130 // Idle — wait for the client to send something (or a notification).
131 let (first_msg, first_stmt) = loop {
132 tokio::select! {
133 msg = client.next() => match msg.transpose()? {
134 Some(proto::frontend::FrontendMessage::Terminate) => {
135 trace!("client sent Terminate (idle)");
136 return Ok(());
137 }
138 Some(msg) => {
139 // Classify the SQL body once — every intercept and
140 // routing decision on this message reads this
141 // single classification.
142 let stmt = classify(&msg);
143 // Interceptors act only on simple-protocol Query
144 // statements: a Parse-carried statement is being
145 // prepared, not executed, and a synthetic
146 // simple-query response would desynchronize the
147 // extended-protocol reply stream.
148 let simple_query =
149 matches!(msg, proto::frontend::FrontendMessage::Query(_));
150
151 // We're in the outer idle loop with no server
152 // connection checked out — by definition the
153 // session has no active transaction, so any
154 // synthetic intercept response advertises
155 // `Idle` to the client.
156 let idle = proto::types::TransactionStatus::Idle;
157
158 // In multiplex mode, intercept LISTEN/UNLISTEN in idle state
159 // (no server connection needed).
160 if simple_query
161 && listen_mode == ListenMode::Multiplex
162 && handle_listen_intercept(&stmt, notifications, client, idle).await?
163 {
164 continue; // stay idle
165 }
166
167 // Intercept Close for named statements — return synthetic
168 // CloseComplete without needing a server connection.
169 if handle_close_intercept(&msg, client_prepared, client, pools).await? {
170 continue; // stay idle
171 }
172
173 // Intercept DEALLOCATE queries for the same
174 // reason — halephant renames client Parse names
175 // to canonical hashes, so a raw DEALLOCATE
176 // would not find the statement on the server.
177 if simple_query
178 && handle_deallocate_intercept(&stmt, client_prepared, client, pools, idle).await?
179 {
180 continue; // stay idle
181 }
182
183 break (msg, stmt);
184 }
185 None => return Ok(()),
186 },
187 Some(notif) = recv_notification(notifications) => {
188 client.send(notif).await?;
189 }
190 }
191 };
192
193 // Reject read-write overrides on read-only sessions.
194 if read_only && first_stmt.is_read_write_override() {
195 reject_read_write(client, proto::types::TransactionStatus::Idle).await?;
196 continue;
197 }
198
199 // Determine routing for this transaction. If the first message is
200 // BEGIN READ ONLY (or START TRANSACTION READ ONLY), route to a replica
201 // even if the session isn't globally read-only.
202 let txn_read_only = read_only || is_read_only_begin(&first_stmt);
203
204 let txn_span = info_span!(
205 "proxy.transaction",
206 db.namespace = database,
207 user,
208 server.address = field::Empty,
209 otel.status_code = field::Empty,
210 otel.status_description = field::Empty,
211 );
212
213 let release = async {
214 let result: anyhow::Result<Release> = async {
215 let mut guard = pools
216 .checkout(client_guard, database, user, txn_read_only)
217 .await?;
218 client_guard.set_state(ClientState::InTransaction);
219 tracing::Span::current().record("server.address", guard.node());
220
221 // Reset this connection's injected-response FIFOs only. A
222 // client that disconnected mid extended-protocol batch can
223 // leave a ParseComplete/CloseComplete disposition whose reply
224 // the backend skipped and will never send; clearing here
225 // keeps it from misaligning this transaction's filter.
226 //
227 // This is NOT where stale prepared-cache entries are handled:
228 // a rejected `Parse` rolls back its optimistic insert at the
229 // `ErrorResponse` arm below (before checkin), so it never
230 // reaches a later checkout. `reset_pending` is FIFO alignment
231 // only, not a catch-all for error-induced staleness.
232 guard.conn().statements.reset_pending();
233
234 // In pin mode, check if the first message is LISTEN — pin immediately.
235 let mut pinned = false;
236 if matches!(first_stmt, sql::Statement::Listen { .. }) {
237 pinned = listen_mode == ListenMode::Pin;
238 }
239 if pinned {
240 warn!("LISTEN detected, pinning connection to session mode");
241 }
242
243 let mut tracker = o11y::spans::StatementTracker::new(query_text);
244
245 // Track SET/RESET and rewrite/send the first message.
246 // A first message that is itself a `Sync` ends its
247 // (empty) batch immediately, so the inner loop must wait
248 // for that `Sync`'s `ReadyForQuery` before reading any
249 // pipelined follow-on messages (which belong to the next
250 // transaction and need a fresh checkout).
251 tracker.on_client_msg(&first_msg, |name| {
252 resolve_stmt_query(name, client_prepared, pools)
253 });
254 // Only a simple-protocol Query executes immediately — a
255 // SET carried in a Parse hasn't run yet — so only Query
256 // classifications feed the SET tracker.
257 if matches!(first_msg, proto::frontend::FrontendMessage::Query(_)) {
258 track_set_reset(&first_stmt, guard.conn());
259 }
260 // Derive the batch boundary from what actually reaches the
261 // backend, not the raw message: if `rewrite_outbound`
262 // suppresses the `Sync` (returns `None`), no `ReadyForQuery`
263 // is owed and the gate must stay open. Mirrors the inner-loop
264 // check below and eliminates the coupling to the raw message.
265 let mut first_ends_batch = false;
266 if let Some(first_msg) =
267 rewrite_outbound(first_msg, client_prepared, guard.conn(), pools, client)
268 .await?
269 {
270 first_ends_batch = matches!(first_msg, proto::frontend::FrontendMessage::Sync);
271 guard.conn().framed.send(first_msg).await?;
272 }
273 debug!("transaction started");
274
275 // Forward until the transaction completes (ReadyForQuery Idle).
276 let result = forward_until_idle(
277 client,
278 guard.conn(),
279 notifications,
280 listen_mode,
281 &mut pinned,
282 client_prepared,
283 pools,
284 txn_read_only,
285 &mut tracker,
286 first_ends_batch,
287 )
288 .await;
289
290 match result {
291 Ok(Release::Idle) => {
292 debug!("transaction complete, connection returned to pool");
293 guard.checkin();
294 Ok(Release::Idle)
295 }
296 Ok(Release::ClientDone) => {
297 guard.checkin();
298 Ok(Release::ClientDone)
299 }
300 Err(e) => Err(e),
301 }
302 }
303 .await;
304
305 // Back to Idle regardless of result — InTransaction is only
306 // set while we're actively holding a backend. On error the
307 // checkin happened inside the inner result; on success the
308 // explicit checkin above already ran.
309 client_guard.set_state(ClientState::Idle);
310
311 let span = tracing::Span::current();
312 match &result {
313 Ok(_) => {
314 span.record("otel.status_code", "OK");
315 }
316 Err(e) => {
317 span.record("otel.status_code", "ERROR");
318 span.record("otel.status_description", e.to_string().as_str());
319 }
320 }
321 result
322 }
323 .instrument(txn_span)
324 .await;
325
326 match release {
327 Ok(Release::Idle) => {}
328 Ok(Release::ClientDone) => return Ok(()),
329 Err(e) => return Err(e),
330 }
331 }
332}
333
334/// Inner loop: forward bidirectionally until ReadyForQuery(Idle) or the client
335/// goes away.
336#[allow(clippy::too_many_arguments)]
337async fn forward_until_idle(
338 client: &mut Framed<TcpStream, proto::codec::FrontendCodec>,
339 server: &mut ServerConn,
340 notifications: &mut Option<ClientNotifications>,
341 listen_mode: ListenMode,
342 pinned: &mut bool,
343 client_prepared: &mut ClientPrepared,
344 pools: &Arc<PoolManager>,
345 txn_read_only: bool,
346 tracker: &mut o11y::spans::StatementTracker,
347 awaiting_ready: bool,
348) -> anyhow::Result<Release> {
349 // While `awaiting_ready` is set, a `Sync` has been forwarded and the
350 // backend owes a `ReadyForQuery`; we must NOT read further client
351 // messages, because a client that pipelines batches has the *next*
352 // transaction's messages already buffered, and forwarding them onto
353 // this (about-to-be-checked-in) backend would strand their responses
354 // when this batch's `ReadyForQuery(Idle)` returns us to the pool.
355 let mut awaiting_ready = awaiting_ready;
356 // The transaction status carried by the most recent `ReadyForQuery`
357 // from this backend, or `None` before the first one arrives this
358 // checkout. Both transaction-scoped decisions below derive from it:
359 //
360 // - The `Sync` gate stays engaged unless `is_some_and(|s| s != Idle)`:
361 // a real `ReadyForQuery(InTransaction | Failed)` confirms an explicit
362 // `BEGIN`…`COMMIT` holds this backend until the transaction ends, so
363 // cross-`Sync` read-ahead is safe and the gate is skipped to keep
364 // intra-transaction pipelining. `None` must not relax the gate:
365 // before any `ReadyForQuery` the session may be autocommit, and
366 // reading past the first batch's `Sync` would strand the next
367 // batch's responses.
368 // - Synthetic responses for intercepted client messages advertise
369 // `unwrap_or(InTransaction)`: we entered this function after sending
370 // the first message of a new transaction — even if no server
371 // response has arrived yet (the client may have pipelined a
372 // BEGIN + LISTEN), the expected post-first-message state is
373 // in-transaction. The server's eventual `ReadyForQuery` replaces
374 // the default with the real status, including `Failed`.
375 let mut last_rfq: Option<proto::types::TransactionStatus> = None;
376
377 loop {
378 tokio::select! {
379 msg = client.next(), if !awaiting_ready => match msg.transpose()? {
380 Some(proto::frontend::FrontendMessage::Terminate) => {
381 trace!("client sent Terminate (in transaction)");
382 return Ok(Release::ClientDone);
383 }
384 Some(msg) => {
385 // Classify the SQL body once — every intercept, routing,
386 // pinning, and tracking decision on this message reads
387 // this single classification.
388 let stmt = classify(&msg);
389 // Interceptors and the SET tracker act only on
390 // simple-protocol Query statements: a Parse-carried
391 // statement is being prepared, not executed, so it must
392 // reach the backend verbatim.
393 let simple_query = matches!(msg, proto::frontend::FrontendMessage::Query(_));
394
395 // Multiplex: intercept LISTEN/UNLISTEN even mid-transaction.
396 if simple_query
397 && listen_mode == ListenMode::Multiplex
398 && handle_listen_intercept(
399 &stmt,
400 notifications,
401 client,
402 last_rfq.unwrap_or(proto::types::TransactionStatus::InTransaction),
403 )
404 .await?
405 {
406 continue;
407 }
408
409 // Reject read-write overrides on read-only transactions.
410 if txn_read_only && stmt.is_read_write_override() {
411 reject_read_write(client, proto::types::TransactionStatus::InTransaction).await?;
412 continue;
413 }
414
415 // Intercept Close for named statements.
416 if handle_close_intercept(&msg, client_prepared, client, pools).await? {
417 continue;
418 }
419
420 // Intercept DEALLOCATE queries.
421 if simple_query
422 && handle_deallocate_intercept(
423 &stmt,
424 client_prepared,
425 client,
426 pools,
427 last_rfq.unwrap_or(proto::types::TransactionStatus::InTransaction),
428 )
429 .await?
430 {
431 continue;
432 }
433
434 // Pin mode: detect LISTEN/UNLISTEN for pinning.
435 if listen_mode == ListenMode::Pin {
436 match &stmt {
437 sql::Statement::Listen { .. } if !*pinned => {
438 warn!("LISTEN detected, pinning connection to session mode");
439 *pinned = true;
440 }
441 sql::Statement::Unlisten {
442 target: sql::UnlistenTarget::Star,
443 } if *pinned => {
444 debug!("UNLISTEN * detected, unpinning connection");
445 *pinned = false;
446 }
447 _ => {}
448 }
449 }
450
451 // Record statement span before rewriting.
452 tracker.on_client_msg(&msg, |name| resolve_stmt_query(name, client_prepared, pools));
453
454 // Track SET/RESET (simple protocol only — a Parse-carried
455 // SET hasn't executed) and rewrite prepared statement names.
456 if simple_query {
457 track_set_reset(&stmt, server);
458 }
459 if let Some(msg) = rewrite_outbound(msg, client_prepared, server, pools, client).await? {
460 // A `Sync` ends the current extended-protocol batch.
461 // In autocommit the next `ReadyForQuery(Idle)` checks
462 // this backend in, so stop reading the client until it
463 // arrives — otherwise a pipelining client's next batch
464 // is read ahead onto this backend and stranded. Inside
465 // an explicit transaction the backend is held to
466 // COMMIT, so read-ahead is safe and the gate is skipped.
467 let in_explicit_txn = last_rfq
468 .is_some_and(|s| s != proto::types::TransactionStatus::Idle);
469 if matches!(msg, proto::frontend::FrontendMessage::Sync)
470 && !in_explicit_txn
471 {
472 awaiting_ready = true;
473 }
474 trace!(?msg, "client -> server");
475 server.framed.send(msg).await?;
476 }
477 }
478 None => return Ok(Release::ClientDone),
479 },
480 msg = server.framed.next() => match msg.transpose()? {
481 Some(proto::backend::BackendMessage::ReadyForQuery(status)) => {
482 // The batch's boundary arrived — resume reading the client.
483 awaiting_ready = false;
484 last_rfq = Some(status);
485 server.last_tx_status = status;
486 tracker.drain();
487 client.send(proto::backend::BackendMessage::ReadyForQuery(status)).await?;
488 if status == proto::types::TransactionStatus::Idle && !*pinned {
489 return Ok(Release::Idle);
490 }
491 }
492 Some(proto::backend::BackendMessage::CommandComplete(ref tag)) => {
493 tracker.on_command_complete(tag);
494 client.send(proto::backend::BackendMessage::CommandComplete(tag.clone())).await?;
495 }
496 Some(proto::backend::BackendMessage::EmptyQueryResponse) => {
497 tracker.on_command_complete("");
498 client.send(proto::backend::BackendMessage::EmptyQueryResponse).await?;
499 }
500 Some(proto::backend::BackendMessage::ErrorResponse(ref err)) => {
501 tracker.on_error(err.message().unwrap_or("unknown"));
502 // The error aborts the extended-protocol batch: every
503 // message after it is skipped, so any Parse halephant
504 // recorded optimistically will never be confirmed. Roll
505 // those inserts back so the pooled connection doesn't
506 // claim statements the backend never prepared.
507 server.statements.roll_back_after_error();
508 client.send(proto::backend::BackendMessage::ErrorResponse(err.clone())).await?;
509 }
510 Some(proto::backend::BackendMessage::ParseComplete) => {
511 // A ParseComplete answers a Parse halephant either
512 // forwarded for the client (deliver it) or injected to
513 // re-prepare on the client's behalf (swallow it). The
514 // disposition was recorded in send order when the Parse
515 // went out.
516 match server.statements.next_parse_reply() {
517 Reply::Suppress => trace!("suppressing injected ParseComplete"),
518 Reply::Forward => {
519 client.send(proto::backend::BackendMessage::ParseComplete).await?;
520 }
521 }
522 }
523 Some(proto::backend::BackendMessage::CloseComplete) => {
524 // Likewise for CloseComplete: forward the client's portal
525 // closes, swallow halephant's LRU-eviction closes.
526 match server.statements.next_close_reply() {
527 Reply::Suppress => trace!("suppressing injected CloseComplete"),
528 Reply::Forward => {
529 client.send(proto::backend::BackendMessage::CloseComplete).await?;
530 }
531 }
532 }
533 Some(msg) => {
534 trace!(?msg, "server -> client");
535 client.send(msg).await?;
536 }
537 None => {
538 tracker.drain();
539 // Send a synthesized FATAL ErrorResponse so the
540 // client knows the transaction was lost, rather
541 // than just seeing EOF.
542 crate::messages::error::send_fatal(
543 client,
544 "08006",
545 "upstream server closed unexpectedly",
546 )
547 .await;
548 anyhow::bail!("upstream server closed unexpectedly");
549 }
550 },
551 Some(notif) = recv_notification(notifications) => {
552 client.send(notif).await?;
553 },
554 }
555 }
556}
557
558// ---------------------------------------------------------------------------
559// Helpers
560// ---------------------------------------------------------------------------
561
562/// Receive a notification from the client's subscription, or pend forever if
563/// multiplexing is not active for this client.
564async fn recv_notification(
565 notifs: &mut Option<ClientNotifications>,
566) -> Option<proto::backend::BackendMessage> {
567 match notifs {
568 Some(n) => n.recv().await,
569 None => std::future::pending().await,
570 }
571}
572
573/// Look up the query text for a named prepared statement via the client's
574/// name mapping and the global statement store. Used as the fallback for
575/// cross-transaction statement reuse.
576fn resolve_stmt_query(
577 client_name: &str,
578 client_prepared: &ClientPrepared,
579 pools: &PoolManager,
580) -> Option<String> {
581 let canon = client_prepared.resolve(client_name)?;
582 let store = pools.stmt_store.lock();
583 store.get(canon).map(|parse| parse.query.clone())
584}
585
586/// Parse the SQL body of a client message, if any.
587fn classify(msg: &proto::frontend::FrontendMessage) -> sql::Statement {
588 match msg {
589 proto::frontend::FrontendMessage::Query(q) => sql::parse(q),
590 proto::frontend::FrontendMessage::Parse(p) => sql::parse(&p.query),
591 _ => sql::Statement::Other,
592 }
593}
594
595/// Check if a statement is a `BEGIN READ ONLY` (or `START TRANSACTION READ ONLY`).
596fn is_read_only_begin(stmt: &sql::Statement) -> bool {
597 matches!(
598 stmt,
599 sql::Statement::Begin {
600 options: sql::TransactionOptions {
601 read_only: Some(true),
602 },
603 }
604 )
605}
606
607/// Send an error to the client rejecting a read-write override on a
608/// replica-routed connection. Thin wrapper around
609/// [`crate::messages::error::send_error`] that hardcodes the
610/// SQLSTATE and message for this specific rejection.
611async fn reject_read_write(
612 client: &mut Framed<TcpStream, proto::codec::FrontendCodec>,
613 status: proto::types::TransactionStatus,
614) -> anyhow::Result<()> {
615 crate::messages::error::send_error(
616 client,
617 "25006",
618 "cannot switch to read-write mode: connection is routed to a read replica",
619 status,
620 )
621 .await
622}