use super::*;
use crate::observe::*;
pub(super) fn migrate(connection: &mut Connection) -> Result<(), String> {
let transaction = connection
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(|error| error.to_string())?;
transaction
.execute_batch(
"
CREATE TABLE IF NOT EXISTS observation_meta (
singleton INTEGER PRIMARY KEY CHECK(singleton = 1),
instance TEXT NOT NULL, floor INTEGER NOT NULL DEFAULT 0
);
INSERT OR IGNORE INTO observation_meta(singleton, instance)
VALUES(1, lower(hex(randomblob(16))));
CREATE TABLE IF NOT EXISTS observation_versions (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL, scope_kind TEXT NOT NULL, scope_key TEXT,
track TEXT NOT NULL, deleted INTEGER NOT NULL, value TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS observation_id_version
ON observation_versions(id, sequence DESC);
CREATE INDEX IF NOT EXISTS observation_scope_version
ON observation_versions(scope_kind, scope_key, sequence);
CREATE INDEX IF NOT EXISTS observation_track_version
ON observation_versions(track, sequence);
",
)
.map_err(|error| error.to_string())?;
let excluded = CONSUMER_COMMANDS
.iter()
.map(|name| format!("'{name}'"))
.collect::<Vec<_>>()
.join(",");
for (event, source, deleted) in [
("INSERT", "new", 0),
("UPDATE", "new", 0),
("DELETE", "old", 1),
] {
let sql = format!(
"
CREATE TRIGGER IF NOT EXISTS observation_invocation_{event}
AFTER {event} ON invocations
WHEN {source}.command_path NOT IN ({excluded})
BEGIN
INSERT INTO observation_versions(id, scope_kind, scope_key, track, deleted, value)
VALUES('invocation:' || {source}.id, {source}.scope_kind, {source}.scope_key,
{source}.command_path, {deleted},
json_object(
'source', 'invocation', 'source_id', {source}.id,
'label', {source}.command_path, 'status', {source}.status,
'start_time_ns', CAST({source}.started_at AS TEXT) || '000000',
'end_time_ns', CASE WHEN {source}.completed_at IS NULL THEN NULL
ELSE CAST({source}.completed_at AS TEXT) || '000000' END,
'execution_id', {source}.execution_id, 'parent_id', {source}.parent_id));
END;"
);
transaction
.execute_batch(&sql)
.map_err(|error| error.to_string())?;
}
transaction.commit().map_err(|error| error.to_string())
}
fn bounds(query: &ObserveQuery) -> Result<(usize, usize), String> {
if !(1..=1000).contains(&query.limit) || !(1024..=1_048_576).contains(&query.byte_limit) {
return Err("OBSERVE_INVALID_LIMIT: rows must be 1..1000 and bytes 1024..1048576".into());
}
Ok((query.limit as usize, query.byte_limit as usize))
}
fn scope_parts(scope: &ObserveScope) -> Result<(Option<String>, Option<&str>), String> {
match scope {
ObserveScope::Global => Ok((None, None)),
ObserveScope::Project { scope_kind, key } if !key.is_empty() => {
let kind = serde_json::to_value(scope_kind).map_err(|error| error.to_string())?;
Ok((kind.as_str().map(str::to_owned), Some(key)))
}
_ => Err("OBSERVE_INVALID_SCOPE: a project requires an exact scope key".into()),
}
}
fn state(connection: &Connection) -> Result<(String, u64, u64), String> {
connection
.query_row(
"SELECT instance, floor,
COALESCE((SELECT seq FROM sqlite_sequence WHERE name='observation_versions'), 0)
FROM observation_meta WHERE singleton=1",
[],
|row| {
Ok((
row.get(0)?,
row.get::<_, i64>(1)? as u64,
row.get::<_, i64>(2)? as u64,
))
},
)
.map_err(|error| error.to_string())
}
fn cursor(
connection: &Connection,
query: &ObserveQuery,
sequence: u64,
) -> Result<ObserveCursor, String> {
Ok(ObserveCursor {
instance: state(connection)?.0,
projection_version: PROJECTION_VERSION,
query_digest: digest(query)?,
sequence: sequence.to_string(),
})
}
fn validate(
connection: &Connection,
query: &ObserveQuery,
cursor: &ObserveCursor,
) -> Result<u64, String> {
let (instance, floor, high) = state(connection)?;
let sequence = cursor
.sequence
.parse::<u64>()
.map_err(|_| "OBSERVE_INVALID_CURSOR: sequence must be an unsigned decimal integer")?;
if cursor.instance != instance
|| cursor.projection_version != PROJECTION_VERSION
|| cursor.query_digest != digest(query)?
|| sequence > high
{
return Err(
"OBSERVE_INVALID_CURSOR: instance, projection, scope, or filter changed".into(),
);
}
if sequence < floor {
return Err("OBSERVE_CURSOR_EXPIRED: take a new snapshot".into());
}
Ok(sequence)
}
fn decode(row: &rusqlite::Row<'_>) -> rusqlite::Result<Observation> {
let text: String = row.get(6)?;
let value: Value = serde_json::from_str(&text).map_err(|error| {
rusqlite::Error::FromSqlConversionFailure(6, rusqlite::types::Type::Text, Box::new(error))
})?;
let optional = |key: &str| value[key].as_str().map(str::to_owned);
Ok(Observation {
id: row.get(1)?,
revision: row.get::<_, i64>(0)?.to_string(),
scope_kind: row.get(2)?,
scope_key: row.get(3)?,
track: row.get(4)?,
deleted: row.get(5)?,
source: optional("source").unwrap_or_default(),
source_id: optional("source_id").unwrap_or_default(),
label: optional("label").unwrap_or_default(),
status: optional("status").unwrap_or_default(),
start_time_ns: optional("start_time_ns").unwrap_or_default(),
end_time_ns: optional("end_time_ns"),
execution_id: optional("execution_id"),
parent_id: optional("parent_id"),
})
}
fn page(
connection: &Connection,
query: &ObserveQuery,
after: u64,
high: u64,
snapshot: bool,
) -> Result<(Vec<Observation>, bool), String> {
let (limit, bytes) = bounds(query)?;
let (scope_kind, scope_key) = scope_parts(&query.scope)?;
let latest = if snapshot {
"AND v.deleted=0 AND NOT EXISTS (
SELECT 1 FROM observation_versions n WHERE n.id=v.id
AND n.sequence>v.sequence AND n.sequence<=?2)"
} else {
""
};
let sql = format!(
"SELECT v.sequence,v.id,v.scope_kind,v.scope_key,v.track,v.deleted,v.value
FROM observation_versions v WHERE v.sequence>?1 AND v.sequence<=?2
AND (?3 IS NULL OR (v.scope_kind=?3 AND v.scope_key=?4))
AND (?5 IS NULL OR v.track=?5) AND (?6 IS NULL OR v.id=?6)
{latest} ORDER BY v.sequence LIMIT ?7"
);
let mut statement = connection
.prepare(&sql)
.map_err(|error| error.to_string())?;
let mapped = statement
.query_map(
params![
after as i64,
high as i64,
scope_kind,
scope_key,
query.track,
query.id,
(limit + 1) as i64
],
decode,
)
.map_err(|error| error.to_string())?;
let mut rows = Vec::new();
let mut used = 0;
let mut more = false;
for row in mapped {
let row = row.map_err(|error| error.to_string())?;
let size = serde_json::to_vec(&row)
.map_err(|error| error.to_string())?
.len();
if rows.len() == limit || used + size > bytes {
if rows.is_empty() {
return Err("OBSERVE_ROW_TOO_LARGE: raise byte_limit for this row".into());
}
more = true;
break;
}
used += size;
rows.push(row);
}
Ok((rows, more))
}
impl Store {
pub fn observe_capabilities(&self) -> ObserveCapabilities {
ObserveCapabilities {
projection_version: PROJECTION_VERSION,
sources: vec!["invocation".into()],
max_rows: 1000,
max_bytes: 1_048_576,
source_clock: "unix_epoch_ns; millisecond source precision".into(),
historical_backfill: false,
}
}
pub fn observe_snapshot(
&self,
query: &ObserveQuery,
next_page: Option<&str>,
) -> Result<ObservePage, String> {
let mut connection = self.connection.lock().map_err(|error| error.to_string())?;
let transaction = connection
.transaction()
.map_err(|error| error.to_string())?;
let (after, high, cursor) = match next_page {
Some(token) => {
let position: SnapshotPosition = serde_json::from_str(token)
.map_err(|_| "OBSERVE_INVALID_CURSOR: invalid snapshot page")?;
let high = validate(&transaction, query, &position.cursor)?;
if position.after > high {
return Err("OBSERVE_INVALID_CURSOR: page exceeds snapshot".into());
}
(position.after, high, position.cursor)
}
None => {
let high = state(&transaction)?.2;
(0, high, cursor(&transaction, query, high)?)
}
};
let (rows, has_more) = page(&transaction, query, after, high, true)?;
let next_page = if has_more {
Some(
serde_json::to_string(&SnapshotPosition {
cursor: cursor.clone(),
after: rows
.last()
.expect("nonempty continued page")
.revision
.parse()
.map_err(|_| "invalid revision")?,
})
.map_err(|error| error.to_string())?,
)
} else {
None
};
Ok(ObservePage {
rows,
cursor,
next_page,
has_more,
})
}
pub fn observe_changes(
&self,
query: &ObserveQuery,
after: &ObserveCursor,
) -> Result<ObservePage, String> {
let mut connection = self.connection.lock().map_err(|error| error.to_string())?;
let transaction = connection
.transaction()
.map_err(|error| error.to_string())?;
let sequence = validate(&transaction, query, after)?;
let high = state(&transaction)?.2;
let (rows, has_more) = page(&transaction, query, sequence, high, false)?;
let acknowledged = if has_more {
rows.last()
.expect("nonempty continued page")
.revision
.parse()
.map_err(|_| "invalid revision")?
} else {
high
};
Ok(ObservePage {
rows,
cursor: cursor(&transaction, query, acknowledged)?,
next_page: None,
has_more,
})
}
pub fn observe_projects(&self) -> Result<Vec<ObserveProject>, String> {
let connection = self.connection.lock().map_err(|error| error.to_string())?;
let mut statement = connection.prepare(
"SELECT scope_kind,scope_key,COUNT(*) FROM observation_versions v
WHERE scope_key IS NOT NULL AND deleted=0
AND NOT EXISTS(SELECT 1 FROM observation_versions n WHERE n.id=v.id AND n.sequence>v.sequence)
GROUP BY scope_kind,scope_key ORDER BY scope_kind,scope_key LIMIT 1000")
.map_err(|error| error.to_string())?;
statement
.query_map([], |row| {
Ok(ObserveProject {
scope_kind: row.get(0)?,
key: row.get(1)?,
rows: row.get::<_, i64>(2)? as u64,
})
})
.map_err(|error| error.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())
}
pub fn observe_tracks(&self, query: &ObserveQuery) -> Result<Vec<ObserveTrack>, String> {
bounds(query)?;
let (scope_kind, scope_key) = scope_parts(&query.scope)?;
let connection = self.connection.lock().map_err(|error| error.to_string())?;
let mut statement = connection.prepare(
"SELECT track,COUNT(*) FROM observation_versions v WHERE deleted=0
AND (?1 IS NULL OR (scope_kind=?1 AND scope_key=?2))
AND (?3 IS NULL OR track=?3) AND (?4 IS NULL OR id=?4)
AND NOT EXISTS(SELECT 1 FROM observation_versions n WHERE n.id=v.id AND n.sequence>v.sequence)
GROUP BY track ORDER BY track LIMIT ?5").map_err(|error| error.to_string())?;
statement
.query_map(
params![scope_kind, scope_key, query.track, query.id, query.limit],
|row| {
Ok(ObserveTrack {
id: row.get(0)?,
rows: row.get::<_, i64>(1)? as u64,
})
},
)
.map_err(|error| error.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())
}
pub fn prune_observations(&self, through: u64) -> Result<usize, String> {
let mut connection = self.connection.lock().map_err(|error| error.to_string())?;
let transaction = connection
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(|error| error.to_string())?;
let (_, floor, high) = state(&transaction)?;
if through < floor || through > high {
return Err("OBSERVE_INVALID_RETENTION: boundary outside retained sequence".into());
}
let removed = transaction
.execute(
"DELETE FROM observation_versions WHERE sequence<=?1
AND sequence NOT IN (SELECT MAX(sequence) FROM observation_versions GROUP BY id)",
[through as i64],
)
.map_err(|error| error.to_string())?;
transaction
.execute(
"UPDATE observation_meta SET floor=?1 WHERE singleton=1",
[through as i64],
)
.map_err(|error| error.to_string())?;
transaction.commit().map_err(|error| error.to_string())?;
Ok(removed)
}
}