Skip to main content

grow_better_stack_logs/
query.rs

1//! SQL query API (ClickHouse HTTP interface on the query host), API.md §5.
2
3use std::collections::{BTreeMap, HashSet};
4use std::sync::Arc;
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use async_trait::async_trait;
8use serde_json::{Map, Value};
9
10use crate::client::Inner;
11use crate::config::is_valid_identifier;
12use crate::core::{CancellationToken, Error, Result, Secret};
13use crate::http::map_query_error;
14use crate::http::{base64_encode, base64url_decode, clickhouse_code, error_for_clickhouse};
15use crate::logging::{
16    poll_tail, Attributes, LogQueryClient, LogRecord, LogSearchResult, RecordCallback,
17    SearchOptions, Severity, SortOrder, TailHandle, TailOptions,
18};
19
20/// Largest `limit` of one search page.
21pub const MAX_SEARCH_LIMIT: usize = 10_000;
22/// Default `limit` of one search page.
23pub const DEFAULT_SEARCH_LIMIT: usize = 100;
24const DAY: Duration = Duration::from_secs(24 * 60 * 60);
25
26/// Value of a bound query parameter (`{name:Type}` placeholder + `param_<name>` URL
27/// parameter). [`QueryParam::Null`] is sent as `\N` (use a `Nullable(…)` type).
28#[derive(Debug, Clone, PartialEq)]
29pub enum QueryParam {
30    /// SQL `NULL`.
31    Null,
32    /// `true` / `false`.
33    Bool(bool),
34    /// Signed integer.
35    Int(i64),
36    /// Unsigned integer.
37    UInt(u64),
38    /// Float (must be finite).
39    Float(f64),
40    /// Text (also identifiers for `{name:Identifier}`).
41    String(String),
42    /// Time, sent as UTC `YYYY-MM-DD HH:MM:SS.ffffff` (for `DateTime64(6)` / `String`).
43    Time(SystemTime),
44}
45
46macro_rules! param_from {
47    ($($t:ty => $v:ident as $c:ty),* $(,)?) => {$(
48        impl From<$t> for QueryParam {
49            fn from(v: $t) -> Self {
50                QueryParam::$v(v as $c)
51            }
52        }
53    )*};
54}
55param_from!(i8 => Int as i64, i16 => Int as i64, i32 => Int as i64, i64 => Int as i64,
56    u8 => UInt as u64, u16 => UInt as u64, u32 => UInt as u64, u64 => UInt as u64,
57    usize => UInt as u64, f32 => Float as f64, f64 => Float as f64);
58
59impl From<bool> for QueryParam {
60    fn from(v: bool) -> Self {
61        QueryParam::Bool(v)
62    }
63}
64impl From<&str> for QueryParam {
65    fn from(v: &str) -> Self {
66        QueryParam::String(v.to_owned())
67    }
68}
69impl From<String> for QueryParam {
70    fn from(v: String) -> Self {
71        QueryParam::String(v)
72    }
73}
74impl From<SystemTime> for QueryParam {
75    fn from(v: SystemTime) -> Self {
76        QueryParam::Time(v)
77    }
78}
79impl<T: Into<QueryParam>> From<Option<T>> for QueryParam {
80    fn from(v: Option<T>) -> Self {
81        v.map_or(QueryParam::Null, Into::into)
82    }
83}
84
85impl QueryParam {
86    /// The text sent as `param_<name>` (ClickHouse's escaped text form; strings go through
87    /// [`escape_param_value`]).
88    pub fn to_param_text(&self) -> Result<String> {
89        Ok(match self {
90            QueryParam::Null => "\\N".to_owned(),
91            QueryParam::Bool(b) => b.to_string(),
92            QueryParam::Int(i) => i.to_string(),
93            QueryParam::UInt(u) => u.to_string(),
94            QueryParam::Float(f) if f.is_finite() => f.to_string(),
95            QueryParam::Float(_) => {
96                return Err(Error::validation("query: a float parameter must be finite"))
97            }
98            QueryParam::String(s) => escape_param_value(s),
99            QueryParam::Time(t) => format_datetime64(*t),
100        })
101    }
102}
103
104/// Bound parameters of [`BetterStackQuery::query`]: name → value.
105pub type QueryParams = BTreeMap<String, QueryParam>;
106
107/// One result row (JSON object; ClickHouse quotes 64-bit integers as strings).
108pub type QueryRow = Map<String, Value>;
109
110/// Builds [`QueryParams`]: `query_params! { "needle" => "x", "n" => 3 }`.
111#[macro_export]
112macro_rules! query_params {
113    () => { $crate::QueryParams::new() };
114    ($($k:expr => $v:expr),+ $(,)?) => {{
115        let mut m = $crate::QueryParams::new();
116        $( m.insert(::std::string::String::from($k), $crate::QueryParam::from($v)); )+
117        m
118    }};
119}
120
121/// Options of [`BetterStackQuery::query`] and [`BetterStackQuery::query_raw`].
122#[derive(Debug, Clone, Default)]
123pub struct QueryOptions {
124    /// ClickHouse settings sent as URL parameters (e.g. `max_execution_time` → `30`).
125    pub settings: BTreeMap<String, String>,
126    /// Cancels the call (`CANCELLED`).
127    pub cancel: Option<CancellationToken>,
128}
129
130impl QueryOptions {
131    /// Adds one ClickHouse setting.
132    pub fn setting(mut self, name: impl Into<String>, value: impl ToString) -> Self {
133        self.settings.insert(name.into(), value.to_string());
134        self
135    }
136    /// Sets the cancellation token.
137    pub fn cancel(mut self, token: CancellationToken) -> Self {
138        self.cancel = Some(token);
139        self
140    }
141}
142
143/// Options of [`BetterStackQuery::search_logs`].
144#[derive(Debug, Clone, PartialEq, Eq)]
145pub struct SearchLogsOptions {
146    /// Collection (`t1234_myapp`); default: the configured `source_table`.
147    pub source: Option<String>,
148    /// Case-sensitive literal text searched in the raw event JSON; it is JSON-escaped first
149    /// (`"`, `\` and control characters), so it matches text inside string values. Empty or
150    /// unset matches everything.
151    pub contains: Option<String>,
152    /// Window start (inclusive); default 24 h before `end` (or before now).
153    pub start: Option<SystemTime>,
154    /// Window end (inclusive); default: no upper bound (robust to clock skew and ingest delay).
155    pub end: Option<SystemTime>,
156    /// Records per page, 1..=10000 (default 100).
157    pub limit: usize,
158    /// Newest first (default) or oldest first.
159    pub order: SortOrder,
160    /// Opaque cursor from [`LogSearchResult::next_cursor`].
161    pub cursor: Option<String>,
162}
163
164impl Default for SearchLogsOptions {
165    fn default() -> Self {
166        SearchLogsOptions {
167            source: None,
168            contains: None,
169            start: None,
170            end: None,
171            limit: DEFAULT_SEARCH_LIMIT,
172            order: SortOrder::Desc,
173            cursor: None,
174        }
175    }
176}
177
178impl SearchLogsOptions {
179    /// Sets the collection.
180    pub fn source(mut self, source: impl Into<String>) -> Self {
181        self.source = Some(source.into());
182        self
183    }
184    /// Sets the text to search.
185    pub fn contains(mut self, text: impl Into<String>) -> Self {
186        self.contains = Some(text.into());
187        self
188    }
189    /// Sets the window (both ends inclusive).
190    pub fn range(mut self, start: Option<SystemTime>, end: Option<SystemTime>) -> Self {
191        self.start = start;
192        self.end = end;
193        self
194    }
195    /// Sets the page size.
196    pub fn limit(mut self, limit: usize) -> Self {
197        self.limit = limit;
198        self
199    }
200    /// Sets the order.
201    pub fn order(mut self, order: SortOrder) -> Self {
202        self.order = order;
203        self
204    }
205    /// Sets the cursor.
206    pub fn cursor(mut self, cursor: impl Into<String>) -> Self {
207        self.cursor = Some(cursor.into());
208        self
209    }
210}
211
212/// Escapes a `param_<name>` value for ClickHouse, which parses query parameter values in the
213/// escaped (TabSeparated) text format: backslash → `\\`, tab → `\t`, newline → `\n`,
214/// carriage return → `\r` (DECISIONS 12). Without it the server would unescape a bound
215/// backslash.
216pub fn escape_param_value(value: &str) -> String {
217    let mut out = String::with_capacity(value.len());
218    for c in value.chars() {
219        match c {
220            '\\' => out.push_str("\\\\"),
221            '\t' => out.push_str("\\t"),
222            '\n' => out.push_str("\\n"),
223            '\r' => out.push_str("\\r"),
224            c => out.push(c),
225        }
226    }
227    out
228}
229
230/// Escapes a value as a ClickHouse single-quoted string literal (`\`, `'` and control
231/// characters escaped with a backslash).
232pub fn sql_string(value: &str) -> String {
233    let mut out = String::with_capacity(value.len() + 2);
234    out.push('\'');
235    for c in value.chars() {
236        match c {
237            '\\' => out.push_str("\\\\"),
238            '\'' => out.push_str("\\'"),
239            '\n' => out.push_str("\\n"),
240            '\r' => out.push_str("\\r"),
241            '\t' => out.push_str("\\t"),
242            '\0' => out.push_str("\\0"),
243            c => out.push(c),
244        }
245    }
246    out.push('\'');
247    out
248}
249
250/// JSON string escaping without the surrounding quotes, as in the raw event JSON.
251pub fn json_escape(text: &str) -> String {
252    let s = Value::String(text.to_owned()).to_string();
253    s[1..s.len() - 1].to_owned()
254}
255
256/// `YYYY-MM-DD HH:MM:SS.ffffff` (UTC), the `DateTime64(6)` text form. Times before the epoch
257/// are clamped to it.
258pub fn format_datetime64(t: SystemTime) -> String {
259    let t = if t < UNIX_EPOCH { UNIX_EPOCH } else { t };
260    let s = humantime::format_rfc3339_micros(t).to_string(); // 2026-11-18T21:00:00.123456Z
261    format!("{} {}", &s[..10], &s[11..26])
262}
263
264/// Parses a ClickHouse `DateTime64` text (`2026-11-18 21:00:00.123456`, UTC, 0–9 fraction
265/// digits), an RFC 3339 string (`Z` or numeric offset) or UNIX seconds / milliseconds.
266pub fn parse_datetime(value: &Value) -> Option<SystemTime> {
267    match value {
268        Value::Number(n) => {
269            if let Some(i) = n.as_u64() {
270                return Some(if i > 1_000_000_000_000 {
271                    UNIX_EPOCH + Duration::from_millis(i)
272                } else {
273                    UNIX_EPOCH + Duration::from_secs(i)
274                });
275            }
276            let f = n.as_f64()?;
277            if !f.is_finite() || f < 0.0 {
278                return None;
279            }
280            let secs = if f > 1e12 { f / 1000.0 } else { f };
281            Some(UNIX_EPOCH + Duration::from_secs_f64(secs))
282        }
283        Value::String(s) => parse_datetime_text(s),
284        _ => None,
285    }
286}
287
288fn parse_datetime_text(text: &str) -> Option<SystemTime> {
289    let s = text.trim();
290    if !s.is_ascii() || s.len() < 19 || !matches!(s.as_bytes()[10], b' ' | b'T') {
291        return None;
292    }
293    let (base, offset): (&str, i64) = if let Some(b) = s.strip_suffix('Z') {
294        (b, 0)
295    } else if s.len() >= 25 && matches!(s.as_bytes()[s.len() - 6], b'+' | b'-') {
296        let tz = &s[s.len() - 6..];
297        if tz.as_bytes()[3] != b':' {
298            return None;
299        }
300        let h: i64 = tz[1..3].parse().ok()?;
301        let m: i64 = tz[4..].parse().ok()?;
302        let secs = h * 3600 + m * 60;
303        (
304            &s[..s.len() - 6],
305            if tz.starts_with('+') { secs } else { -secs },
306        )
307    } else {
308        (s, 0)
309    };
310    let t = humantime::parse_rfc3339_weak(base).ok()?;
311    if offset >= 0 {
312        t.checked_sub(Duration::from_secs(offset as u64))
313    } else {
314        t.checked_add(Duration::from_secs(offset.unsigned_abs()))
315    }
316}
317
318const TOP_LEVEL_KEYS: &[&str] = &[
319    "dt",
320    "message",
321    "msg",
322    "level",
323    "severity",
324    "severity_number",
325    "span",
326    "trace_id",
327    "span_id",
328    "trace_flags",
329    "resource",
330    "attributes",
331    "observed_dt",
332];
333
334fn hex_id(v: Option<&Value>, len: usize) -> Option<String> {
335    let s = v?.as_str()?.trim().to_ascii_lowercase();
336    (s.len() == len && s.bytes().all(|b| b.is_ascii_hexdigit()) && s.bytes().any(|b| b != b'0'))
337        .then_some(s)
338}
339
340fn severity_from_value(v: &Value) -> Option<Severity> {
341    match v {
342        Value::Number(n) => n.as_u64().map(|n| Severity::from_number(n.min(255) as u8)),
343        Value::String(s) => s.parse().ok(),
344        _ => None,
345    }
346}
347
348fn object_attributes(v: Option<&Value>) -> Attributes {
349    match v {
350        Some(Value::Object(m)) => m.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
351        _ => Attributes::new(),
352    }
353}
354
355/// Normalizes a `{dt, raw}` row into a [`LogRecord`] (API.md §5.3): `body` = `message` (else
356/// `msg`, else the raw text), severity from `severity_number` / `level` / `severity`, trace ids
357/// from `span.trace_id` / `span.span_id` (or top-level `trace_id` / `span_id`), `resource` from
358/// `resource`, and every other field plus `attributes` into `attributes`.
359pub fn log_record_from_row(row: &QueryRow) -> LogRecord {
360    let raw_text = match row.get("raw") {
361        Some(Value::String(s)) => s.clone(),
362        Some(v) => v.to_string(),
363        None => "null".to_owned(),
364    };
365    let parsed: Option<Value> = serde_json::from_str(&raw_text).ok();
366    let empty = Map::new();
367    let event = match &parsed {
368        Some(Value::Object(m)) => m,
369        _ => &empty,
370    };
371    let timestamp = row
372        .get("dt")
373        .and_then(parse_datetime)
374        .or_else(|| event.get("dt").and_then(parse_datetime))
375        .unwrap_or(UNIX_EPOCH);
376    let body = match event.get("message").or_else(|| event.get("msg")) {
377        Some(Value::String(s)) => s.clone(),
378        Some(v) => v.to_string(),
379        None => raw_text.clone(),
380    };
381    let level_text = ["level", "severity"].iter().find_map(|k| {
382        event
383            .get(*k)
384            .and_then(Value::as_str)
385            .filter(|s| !s.is_empty())
386    });
387    let from_number = event.get("severity_number").and_then(severity_from_value);
388    let from_text = level_text.and_then(|t| t.parse::<Severity>().ok());
389    let severity = from_number.or(from_text).unwrap_or(Severity::Info);
390    let severity_text = match (from_text, level_text) {
391        (Some(s), _) => s.text().to_owned(),
392        (None, Some(t)) => t.to_owned(),
393        (None, None) => severity.text().to_owned(),
394    };
395    let mut attributes: Attributes = event
396        .iter()
397        .filter(|(k, _)| !TOP_LEVEL_KEYS.contains(&k.as_str()))
398        .map(|(k, v)| (k.clone(), v.clone()))
399        .collect();
400    attributes.extend(object_attributes(event.get("attributes")));
401    let span = event.get("span").and_then(Value::as_object);
402    let trace_id = hex_id(
403        span.and_then(|s| s.get("trace_id"))
404            .or_else(|| event.get("trace_id")),
405        32,
406    );
407    let (span_id, trace_flags) = if trace_id.is_some() {
408        (
409            hex_id(
410                span.and_then(|s| s.get("span_id"))
411                    .or_else(|| event.get("span_id")),
412                16,
413            ),
414            event
415                .get("trace_flags")
416                .and_then(Value::as_u64)
417                .map(|f| f.min(255) as u8),
418        )
419    } else {
420        (None, None)
421    };
422    LogRecord {
423        timestamp,
424        observed_timestamp: event.get("observed_dt").and_then(parse_datetime),
425        severity,
426        severity_text,
427        body,
428        attributes,
429        resource: object_attributes(event.get("resource")),
430        trace_id,
431        span_id,
432        trace_flags,
433    }
434}
435
436/// Raises the mapped error when a 200 body carries a ClickHouse exception (error after 200).
437fn exception_in(text: &str) -> Error {
438    if let Some(code) = clickhouse_code(text) {
439        let line = text
440            .lines()
441            .find(|l| clickhouse_code(l).is_some())
442            .unwrap_or(text)
443            .trim();
444        let line: String = line.chars().take(300).collect();
445        let message = format!("query: {line}");
446        return error_for_clickhouse(code, &message).unwrap_or_else(|| {
447            Error::unavailable(message)
448                .with_status(200)
449                .with_retryable(false)
450        });
451    }
452    Error::unavailable("query: invalid JSON in response").with_retryable(false)
453}
454
455fn rows_from_array(v: Vec<Value>) -> Result<Vec<QueryRow>> {
456    v.into_iter()
457        .map(|item| match item {
458            Value::Object(m) => Ok(m),
459            _ => Err(
460                Error::unavailable("query: a result row is not a JSON object")
461                    .with_retryable(false),
462            ),
463        })
464        .collect()
465}
466
467/// Parses a JSON-family response body (`JSONEachRow`, `JSON`, `JSONEachRow` with
468/// `output_format_json_array_of_rows`) into rows. A ClickHouse exception appended to the
469/// body is returned as the mapped error.
470pub fn parse_rows(text: &str) -> Result<Vec<QueryRow>> {
471    let trimmed = text.trim();
472    if trimmed.is_empty() {
473        return Ok(Vec::new());
474    }
475    if trimmed.starts_with('[') {
476        return match serde_json::from_str::<Vec<Value>>(trimmed) {
477            Ok(v) => rows_from_array(v),
478            Err(_) => Err(exception_in(trimmed)),
479        };
480    }
481    let mut rows = Vec::new();
482    for line in trimmed.lines().map(str::trim).filter(|l| !l.is_empty()) {
483        match serde_json::from_str::<Value>(line) {
484            Ok(Value::Object(m)) => rows.push(m),
485            _ => {
486                if rows.is_empty() {
487                    if let Ok(Value::Object(mut doc)) = serde_json::from_str::<Value>(trimmed) {
488                        if let Some(Value::Array(data)) = doc.remove("data") {
489                            return rows_from_array(data);
490                        }
491                    }
492                }
493                return Err(exception_in(line));
494            }
495        }
496    }
497    if rows.len() == 1 && rows[0].contains_key("meta") {
498        if let Some(Value::Array(data)) = rows[0].get("data").cloned() {
499            return rows_from_array(data);
500        }
501    }
502    Ok(rows)
503}
504
505const JSON_FORMATS: &[&str] = &[
506    "jsoneachrow",
507    "json",
508    "jsonstringseachrow",
509    "ndjson",
510    "jsonlines",
511];
512
513/// The `FORMAT <name>` at the end of a statement, if any.
514fn format_clause(sql: &str) -> Option<&str> {
515    let words: Vec<&str> = sql.split_whitespace().collect();
516    match words.as_slice() {
517        [.., kw, name] if kw.eq_ignore_ascii_case("FORMAT") => Some(name.trim_end_matches(';')),
518        _ => None,
519    }
520}
521
522fn encode_cursor(offset: usize) -> String {
523    base64_encode(format!("{{\"o\":{offset}}}").as_bytes(), true)
524}
525
526fn decode_cursor(cursor: &str) -> Result<usize> {
527    base64url_decode(cursor)
528        .and_then(|b| serde_json::from_slice::<Value>(&b).ok())
529        .and_then(|v| v.get("o").and_then(Value::as_u64))
530        .map(|o| o as usize)
531        .ok_or_else(|| Error::validation("logs.search: invalid cursor"))
532}
533
534/// SQL query client for Better Stack Logs (ClickHouse HTTP interface with the connection's own
535/// Basic credentials). Implements [`LogQueryClient`]: `search` runs
536/// [`search_logs`](Self::search_logs) on the configured `source_table`, `tail` polls `search`,
537/// `list_streams` returns the configured collection and `list_labels` is `UNSUPPORTED`
538/// (Better Stack has no labels). Cheap to clone.
539#[derive(Clone)]
540pub struct BetterStackQuery {
541    inner: Arc<Inner>,
542}
543
544impl std::fmt::Debug for BetterStackQuery {
545    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
546        f.debug_struct("BetterStackQuery")
547            .field("source_table", &self.inner.config.source_table())
548            .finish()
549    }
550}
551
552impl BetterStackQuery {
553    pub(crate) fn new(inner: Arc<Inner>) -> Self {
554        BetterStackQuery { inner }
555    }
556
557    /// Runs SQL and returns the response text unchanged. `params` are bound server side
558    /// (`{name:Type}` placeholders + `param_<name>` URL parameters).
559    pub async fn query_raw(
560        &self,
561        sql: &str,
562        params: &QueryParams,
563        opts: &QueryOptions,
564    ) -> Result<String> {
565        const OP: &str = "query";
566        if sql.trim().is_empty() {
567            return Err(Error::validation("query: sql must not be empty"));
568        }
569        let transport = self.inner.query_transport(OP)?;
570        let basic = self
571            .inner
572            .config
573            .basic_credentials()
574            .ok_or_else(|| Error::config("query: query credentials are required"))?;
575        let mut req = transport
576            .post("/")
577            .operation(OP)
578            .secret_header(
579                "authorization",
580                &Secret::new(format!("Basic {}", basic.expose_secret())),
581            )
582            .idempotent(true);
583        for (k, v) in &opts.settings {
584            req = req.query(k.clone(), v.clone());
585        }
586        for (name, value) in params {
587            if !is_valid_identifier(name) {
588                return Err(Error::validation(format!(
589                    "query: invalid parameter name {name:?}"
590                )));
591            }
592            req = req.query(format!("param_{name}"), value.to_param_text()?);
593        }
594        if let Some(token) = &opts.cancel {
595            req = req.cancel_token(token.clone());
596        }
597        let resp = req
598            .body(sql.as_bytes().to_vec(), "text/plain; charset=utf-8")
599            .send()
600            .await
601            .map_err(map_query_error)?;
602        resp.text()
603    }
604
605    /// Runs SQL and returns the rows. Without a `FORMAT` clause, `FORMAT JSONEachRow` is
606    /// appended; `FORMAT JSON` / `JSONEachRow` (also with `output_format_json_array_of_rows`)
607    /// are parsed, other formats are a `VALIDATION` error (use [`query_raw`](Self::query_raw)).
608    /// A ClickHouse exception appended to a 200 response is returned as the mapped error.
609    ///
610    /// ```no_run
611    /// # async fn demo(q: grow_better_stack_logs::BetterStackQuery) -> grow_better_stack_logs::Result<()> {
612    /// use grow_better_stack_logs::{query_params, QueryOptions};
613    /// let rows = q
614    ///     .query(
615    ///         "SELECT count() AS c FROM remote(t1_app_logs) WHERE position(raw, {needle:String}) > 0",
616    ///         &query_params! { "needle" => "checkout" },
617    ///         &QueryOptions::default(),
618    ///     )
619    ///     .await?;
620    /// # Ok(()) }
621    /// ```
622    pub async fn query(
623        &self,
624        sql: &str,
625        params: &QueryParams,
626        opts: &QueryOptions,
627    ) -> Result<Vec<QueryRow>> {
628        let trimmed = sql.trim().trim_end_matches(';').trim_end();
629        if trimmed.is_empty() {
630            return Err(Error::validation("query: sql must not be empty"));
631        }
632        let statement = match format_clause(trimmed) {
633            Some(name) if JSON_FORMATS.contains(&name.to_ascii_lowercase().as_str()) => {
634                trimmed.to_owned()
635            }
636            Some(name) => {
637                return Err(Error::validation(format!(
638                    "query: FORMAT {name} cannot be parsed into rows; use query_raw or FORMAT JSONEachRow"
639                )))
640            }
641            None => format!("{trimmed} FORMAT JSONEachRow"),
642        };
643        let text = self.query_raw(&statement, params, opts).await?;
644        parse_rows(&text)
645    }
646
647    /// Renders the [`search_logs`](Self::search_logs) SQL: hot `remote(<source>_logs)` UNION ALL
648    /// cold `s3Cluster(primary, <source>_s3)`, values as escaped literals,
649    /// `toDateTime64(…, 6, 'UTC')` window, `position(raw, …)` substring match, `LIMIT n+1`.
650    pub fn render_search_sql(&self, opts: &SearchLogsOptions) -> Result<String> {
651        let source = opts
652            .source
653            .as_deref()
654            .or(self.inner.config.source_table())
655            .ok_or_else(|| {
656                Error::config(
657                    "invalid configuration: search_logs needs a source or a configured source_table (GROW_BETTER_STACK_LOGS_SOURCE_TABLE)",
658                )
659            })?;
660        if !is_valid_identifier(source) {
661            return Err(Error::validation(
662                "logs.search: source must match ^[A-Za-z_][A-Za-z0-9_]*$",
663            ));
664        }
665        if !(1..=MAX_SEARCH_LIMIT).contains(&opts.limit) {
666            return Err(Error::validation(format!(
667                "logs.search: limit must be between 1 and {MAX_SEARCH_LIMIT}"
668            )));
669        }
670        let start = match (opts.start, opts.end) {
671            (Some(s), _) => s,
672            (None, Some(e)) => e.checked_sub(DAY).unwrap_or(UNIX_EPOCH),
673            (None, None) => SystemTime::now() - DAY,
674        };
675        if opts.end.is_some_and(|e| start > e) {
676            return Err(Error::validation(
677                "logs.search: start must not be after end",
678            ));
679        }
680        let offset = match &opts.cursor {
681            Some(c) => decode_cursor(c)?,
682            None => 0,
683        };
684        let ts = |t: SystemTime| {
685            format!(
686                "toDateTime64({}, 6, 'UTC')",
687                sql_string(&format_datetime64(t))
688            )
689        };
690        let mut wheres = vec![match opts.end {
691            None => format!("dt >= {}", ts(start)),
692            Some(e) => format!("dt BETWEEN {} AND {}", ts(start), ts(e)),
693        }];
694        if let Some(c) = opts.contains.as_deref().filter(|c| !c.is_empty()) {
695            wheres.push(format!(
696                "position(raw, {}) > 0",
697                sql_string(&json_escape(c))
698            ));
699        }
700        let order = match opts.order {
701            SortOrder::Desc => "DESC",
702            SortOrder::Asc => "ASC",
703        };
704        let offset_sql = if offset > 0 {
705            format!(" OFFSET {offset}")
706        } else {
707            String::new()
708        };
709        Ok(format!(
710            "SELECT dt, raw FROM (SELECT dt, raw FROM remote({source}_logs) UNION ALL \
711             SELECT dt, raw FROM s3Cluster(primary, {source}_s3) WHERE _row_type = 1) \
712             WHERE {} ORDER BY dt {order} LIMIT {}{offset_sql} FORMAT JSONEachRow",
713            wheres.join(" AND "),
714            opts.limit + 1
715        ))
716    }
717
718    /// Searches hot and cold logs of a collection: inclusive time window, literal substring
719    /// match, newest first by default. Rows are normalized ([`log_record_from_row`]) and
720    /// deduplicated on `(dt, raw)`; `next_cursor` is set when more rows exist.
721    pub async fn search_logs(&self, opts: SearchLogsOptions) -> Result<LogSearchResult> {
722        self.search_logs_with(opts, &QueryOptions::default()).await
723    }
724
725    /// [`search_logs`](Self::search_logs) with query options (settings, cancellation).
726    pub async fn search_logs_with(
727        &self,
728        opts: SearchLogsOptions,
729        query_opts: &QueryOptions,
730    ) -> Result<LogSearchResult> {
731        let sql = self.render_search_sql(&opts)?;
732        let offset = match &opts.cursor {
733            Some(c) => decode_cursor(c)?,
734            None => 0,
735        };
736        let rows = self.query(&sql, &QueryParams::new(), query_opts).await?;
737        let more = rows.len() > opts.limit;
738        let mut seen = HashSet::new();
739        let records = rows
740            .iter()
741            .take(opts.limit)
742            .filter(|row| {
743                let key = (
744                    row.get("dt").map(Value::to_string),
745                    row.get("raw").map(Value::to_string),
746                );
747                seen.insert(key)
748            })
749            .map(log_record_from_row)
750            .collect();
751        Ok(LogSearchResult {
752            records,
753            next_cursor: more.then(|| encode_cursor(offset + opts.limit)),
754            total: None,
755        })
756    }
757}
758
759#[async_trait]
760impl LogQueryClient for BetterStackQuery {
761    /// `query` is the `contains` text, searched in the configured `source_table`.
762    async fn search(&self, query: &str, opts: SearchOptions) -> Result<LogSearchResult> {
763        self.search_logs(SearchLogsOptions {
764            source: None,
765            contains: Some(query.to_owned()),
766            start: opts.start,
767            end: opts.end,
768            limit: opts.limit,
769            order: opts.order,
770            cursor: opts.cursor,
771        })
772        .await
773    }
774
775    /// Polls [`search`](LogQueryClient::search) oldest-first (Better Stack has no public
776    /// streaming endpoint); stopped by the handle or by the client's `close`.
777    async fn tail(
778        &self,
779        query: &str,
780        on_record: RecordCallback,
781        opts: TailOptions,
782    ) -> Result<TailHandle> {
783        self.inner.query_transport("logs.tail")?;
784        if self.inner.config.source_table().is_none() {
785            return Err(Error::config(
786                "logs.tail: a configured source_table is required",
787            ));
788        }
789        let handle = poll_tail(Arc::new(self.clone()), query, on_record, opts);
790        let h = handle.clone();
791        self.inner.on_close.register(move || {
792            Box::pin(async move {
793                h.stop();
794            })
795        });
796        Ok(handle)
797    }
798
799    /// The configured collections (`[source_table]`, or `[]`); Better Stack has no data-plane
800    /// listing.
801    async fn list_streams(&self) -> Result<Vec<String>> {
802        Ok(self
803            .inner
804            .config
805            .source_table()
806            .map(|t| vec![t.to_owned()])
807            .unwrap_or_default())
808    }
809
810    /// Always `UNSUPPORTED`: Better Stack has no label concept.
811    async fn list_labels(&self) -> Result<Vec<String>> {
812        Err(Error::unsupported(
813            "logs.listLabels: Better Stack has no labels",
814        ))
815    }
816}