use crate::config::PubsubSourceConfig;
use crate::convert::{MessageParts, record_with_policy, timestamp_millis};
use crate::state::{PubsubBookmark, state_key};
use faucet_core::lease::LeaseExtender;
use faucet_core::{FaucetError, Stream, StreamPage};
use gcloud_pubsub::client::Client;
use gcloud_pubsub::subscriber::ReceivedMessage;
use gcloud_pubsub::subscription::Subscription;
use serde_json::Value;
use std::pin::Pin;
use std::sync::Mutex;
use std::time::{Duration, Instant};
pub struct PubsubSource {
config: PubsubSourceConfig,
client: Client,
start_bookmark: Mutex<Option<PubsubBookmark>>,
}
impl PubsubSource {
pub async fn new(config: PubsubSourceConfig) -> Result<Self, FaucetError> {
config.validate()?;
let client = faucet_common_pubsub::build_client(&config.connection).await?;
Ok(Self {
config,
client,
start_bookmark: Mutex::new(None),
})
}
fn subscription(&self) -> Subscription {
self.client.subscription(&self.config.subscription)
}
}
async fn pull_messages(
subscription: &Subscription,
max: usize,
) -> Result<Vec<ReceivedMessage>, FaucetError> {
subscription
.pull(max as i32, None)
.await
.map_err(|e| FaucetError::Source(format!("pubsub: pull failed: {e}")))
}
async fn nack_messages(messages: Vec<ReceivedMessage>) {
use futures::StreamExt;
futures::stream::iter(messages)
.map(|m| async move {
if let Err(e) = m.nack().await {
tracing::warn!(error = %e, "pubsub: nack failed; the message redelivers after its ack deadline");
}
})
.buffer_unordered(RENEW_CONCURRENCY)
.collect::<Vec<()>>()
.await;
}
pub(crate) fn pull_size(per_pull: usize, max_messages: Option<usize>, total: usize) -> usize {
match max_messages {
Some(max) => per_pull.min(max.saturating_sub(total)).max(1),
None => per_pull,
}
}
const LATE_PULL_GRACE: Duration = Duration::from_secs(2);
const ACK_BATCH: usize = 500;
const RENEW_CONCURRENCY: usize = 16;
async fn ack_messages(subscription: &Subscription, messages: &[ReceivedMessage]) {
for chunk in messages.chunks(ACK_BATCH) {
let ids = chunk.iter().map(|m| m.ack_id().to_string()).collect();
if let Err(e) = subscription.ack(ids).await {
tracing::warn!(error = %e, "pubsub: ack failed; messages will be redelivered");
}
}
}
async fn renew_deadlines(messages: Vec<ReceivedMessage>, secs: i32) {
use futures::StreamExt;
futures::stream::iter(messages)
.map(|m| async move {
if let Err(e) = m.modify_ack_deadline(secs).await {
tracing::warn!(error = %e, "pubsub: ack deadline renewal failed");
}
})
.buffer_unordered(RENEW_CONCURRENCY)
.collect::<Vec<()>>()
.await;
}
pub(crate) fn renew_every(extension_secs: u32) -> Duration {
Duration::from_secs(u64::from((extension_secs / 3).clamp(1, 5)))
}
impl PubsubSource {
fn publish_millis(m: &ReceivedMessage) -> Option<i64> {
m.message
.publish_time
.as_ref()
.map(|t| timestamp_millis(t.seconds, t.nanos))
}
}
#[faucet_core::async_trait]
impl faucet_core::Source for PubsubSource {
async fn fetch_with_context(
&self,
context: &std::collections::HashMap<String, Value>,
) -> Result<Vec<Value>, FaucetError> {
use futures::StreamExt;
let mut pages = self.stream_pages(context, self.config.batch_size);
let mut all = Vec::new();
while let Some(page) = pages.next().await {
all.extend(page?.records);
}
Ok(all)
}
fn stream_pages<'a>(
&'a self,
_context: &'a std::collections::HashMap<String, Value>,
_batch_size: usize,
) -> Pin<Box<dyn Stream<Item = Result<StreamPage, FaucetError>> + Send + 'a>> {
let chunk = if self.config.batch_size == 0 {
usize::MAX
} else {
self.config.batch_size
};
let idle = self.config.idle_termination_secs.map(Duration::from_secs);
let max_messages = self.config.max_messages;
let per_pull = self.config.max_messages_per_pull;
let format = self.config.value_format;
let attributes_key = self.config.attributes_key.clone();
let on_decode_error = self.config.on_decode_error;
let extension = self.config.ack_deadline_extension_secs;
Box::pin(async_stream::try_stream! {
let subscription = self.subscription();
let lease = (extension > 0).then(|| {
LeaseExtender::spawn(renew_every(extension), move |held: Vec<ReceivedMessage>| {
renew_deadlines(held, extension as i32)
})
});
let hold = |page: &[ReceivedMessage], yielded: &[ReceivedMessage]| {
if let Some(l) = &lease {
l.hold(page.iter().chain(yielded).cloned().collect());
}
};
let mut cumulative = self
.start_bookmark
.lock()
.expect("bookmark mutex poisoned")
.clone()
.unwrap_or_default();
let mut pending: Vec<ReceivedMessage> = Vec::new();
let mut buffer: Vec<Value> = Vec::new();
let mut page_msgs: Vec<ReceivedMessage> = Vec::new();
let mut total = 0usize;
let mut last_activity = Instant::now();
'consume: loop {
if !pending.is_empty() {
ack_messages(&subscription, &pending).await;
pending.clear();
hold(&page_msgs, &pending);
}
let want = pull_size(per_pull, max_messages, total);
let pull_sub = self.subscription();
let mut pull = tokio::spawn(async move { pull_messages(&pull_sub, want).await });
let joined = match idle {
Some(window) => match tokio::time::timeout(window, &mut pull).await {
Ok(joined) => joined,
Err(_) => {
tracing::info!(
subscription = %self.config.subscription,
idle_secs = window.as_secs(),
"pubsub: idle termination reached"
);
match tokio::time::timeout(LATE_PULL_GRACE, &mut pull).await {
Ok(Ok(Ok(late))) if !late.is_empty() => nack_messages(late).await,
Ok(_) => {}
Err(_) => pull.abort(),
}
break 'consume;
}
},
None => pull.await,
};
let messages = joined
.map_err(|e| FaucetError::Source(format!("pubsub: pull task failed: {e}")))??;
if messages.is_empty() {
if let Some(window) = idle
&& last_activity.elapsed() >= window
{
tracing::info!(
subscription = %self.config.subscription,
idle_secs = window.as_secs(),
"pubsub: idle termination reached"
);
break 'consume;
}
tokio::time::sleep(Duration::from_millis(200)).await;
continue;
}
last_activity = Instant::now();
if lease.is_some() {
let mut pulled = messages.clone();
pulled.extend(page_msgs.iter().cloned());
hold(&pulled, &pending);
}
let mut messages = messages.into_iter();
while let Some(m) = messages.next() {
let record = record_with_policy(
&MessageParts {
data: &m.message.data,
attributes: &m.message.attributes,
message_id: &m.message.message_id,
ordering_key: &m.message.ordering_key,
publish_time_millis: Self::publish_millis(&m),
},
format,
&attributes_key,
on_decode_error,
)?;
cumulative.advance(&m.message.message_id);
if let Some(record) = record {
buffer.push(record);
}
page_msgs.push(m);
total += 1;
if buffer.len() >= chunk {
let records = std::mem::take(&mut buffer);
pending.append(&mut page_msgs);
yield StreamPage {
records,
bookmark: Some(cumulative.to_value()),
};
}
if let Some(max) = max_messages
&& total >= max
{
tracing::info!(
subscription = %self.config.subscription,
max,
"pubsub: max_messages reached"
);
let rest: Vec<ReceivedMessage> = messages.collect();
if !rest.is_empty() {
nack_messages(rest).await;
}
break 'consume;
}
}
}
if !buffer.is_empty() {
let records = std::mem::take(&mut buffer);
pending.append(&mut page_msgs);
yield StreamPage {
records,
bookmark: Some(cumulative.to_value()),
};
}
pending.append(&mut page_msgs);
if !pending.is_empty() {
ack_messages(&subscription, &pending).await;
}
drop(lease);
tracing::info!(
subscription = %self.config.subscription,
records = total,
"pubsub source stream complete"
);
})
}
fn config_schema(&self) -> Value {
serde_json::to_value(faucet_core::schema_for!(PubsubSourceConfig))
.expect("schema serialization")
}
fn state_key(&self) -> Option<String> {
Some(state_key(&self.config.subscription))
}
async fn apply_start_bookmark(&self, bookmark: Value) -> Result<(), FaucetError> {
*self.start_bookmark.lock().expect("bookmark mutex poisoned") =
Some(PubsubBookmark::from_value(&bookmark));
Ok(())
}
fn consumes_destructively(&self) -> bool {
true
}
fn connector_name(&self) -> &'static str {
"pubsub"
}
fn dataset_uri(&self) -> String {
format!(
"pubsub://{}/subscriptions/{}",
self.config
.connection
.project_id
.as_deref()
.unwrap_or("default"),
self.config.subscription
)
}
async fn check(
&self,
ctx: &faucet_core::CheckContext,
) -> Result<faucet_core::CheckReport, FaucetError> {
use faucet_core::{CheckReport, Probe};
let start = std::time::Instant::now();
let subscription = self.subscription();
let fut = subscription.exists(None);
let probe = match tokio::time::timeout(ctx.timeout, fut).await {
Err(_) => Probe::fail("subscription_exists", start.elapsed(), "timed out"),
Ok(Ok(true)) => Probe::pass("subscription_exists", start.elapsed()),
Ok(Ok(false)) => Probe::fail(
"subscription_exists",
start.elapsed(),
format!("subscription '{}' does not exist", self.config.subscription),
),
Ok(Err(e)) => Probe::fail("subscription_exists", start.elapsed(), e.to_string()),
};
Ok(CheckReport::single(probe))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pulls_never_exceed_what_the_run_still_needs() {
assert_eq!(pull_size(1000, None, 50), 1000);
assert_eq!(pull_size(1000, Some(30), 25), 5);
assert_eq!(pull_size(10, Some(30), 0), 10);
assert_eq!(pull_size(10, Some(30), 30), 1);
}
#[test]
fn state_key_and_dataset_uri_helpers() {
assert_eq!(state_key("orders-sub"), "pubsub:orders-sub");
}
#[test]
fn config_schema_exposes_fields() {
let schema = serde_json::to_value(faucet_core::schema_for!(PubsubSourceConfig)).unwrap();
assert!(schema["properties"]["subscription"].is_object());
assert!(schema["properties"]["value_format"].is_object());
}
#[tokio::test]
async fn new_rejects_config_without_termination() {
let err = match PubsubSource::new(PubsubSourceConfig::new("orders-sub")).await {
Ok(_) => panic!("expected a termination-config error"),
Err(e) => e,
};
assert!(err.to_string().contains("idle_termination_secs"), "{err}");
}
#[test]
fn renewal_runs_every_five_seconds_or_a_third_of_a_short_window() {
assert_eq!(renew_every(60), Duration::from_secs(5));
assert_eq!(renew_every(12), Duration::from_secs(4));
assert_eq!(renew_every(10), Duration::from_secs(3));
assert_eq!(renew_every(600), Duration::from_secs(5));
}
}