apoc 0.9.1

Agentic process control for LLMs
Documentation
//! Transactional, sanitized invocation projections and durable resume cursors.
use super::*;
use crate::observe::*;

pub(super) fn migrate(connection: &mut Connection) -> Result<(), String> {
    let transaction = connection
        .transaction_with_behavior(TransactionBehavior::Immediate)
        .map_err(|error| error.to_string())?;
    transaction
        .execute_batch(
            "
        CREATE TABLE IF NOT EXISTS observation_meta (
            singleton INTEGER PRIMARY KEY CHECK(singleton = 1),
            instance TEXT NOT NULL, floor INTEGER NOT NULL DEFAULT 0
        );
        INSERT OR IGNORE INTO observation_meta(singleton, instance)
            VALUES(1, lower(hex(randomblob(16))));
        CREATE TABLE IF NOT EXISTS observation_versions (
            sequence INTEGER PRIMARY KEY AUTOINCREMENT,
            id TEXT NOT NULL, scope_kind TEXT NOT NULL, scope_key TEXT,
            track TEXT NOT NULL, deleted INTEGER NOT NULL, value TEXT NOT NULL
        );
        CREATE INDEX IF NOT EXISTS observation_id_version
            ON observation_versions(id, sequence DESC);
        CREATE INDEX IF NOT EXISTS observation_scope_version
            ON observation_versions(scope_kind, scope_key, sequence);
        CREATE INDEX IF NOT EXISTS observation_track_version
            ON observation_versions(track, sequence);
    ",
        )
        .map_err(|error| error.to_string())?;
    let excluded = CONSUMER_COMMANDS
        .iter()
        .map(|name| format!("'{name}'"))
        .collect::<Vec<_>>()
        .join(",");
    for (event, source, deleted) in [
        ("INSERT", "new", 0),
        ("UPDATE", "new", 0),
        ("DELETE", "old", 1),
    ] {
        // Only selected metadata enters the projection. Arguments, output, and error
        // bodies remain in their existing addressed stores.
        let sql = format!(
            "
            CREATE TRIGGER IF NOT EXISTS observation_invocation_{event}
            AFTER {event} ON invocations
            WHEN {source}.command_path NOT IN ({excluded})
            BEGIN
                INSERT INTO observation_versions(id, scope_kind, scope_key, track, deleted, value)
                VALUES('invocation:' || {source}.id, {source}.scope_kind, {source}.scope_key,
                    {source}.command_path, {deleted},
                    json_object(
                        'source', 'invocation', 'source_id', {source}.id,
                        'label', {source}.command_path, 'status', {source}.status,
                        'start_time_ns', CAST({source}.started_at AS TEXT) || '000000',
                        'end_time_ns', CASE WHEN {source}.completed_at IS NULL THEN NULL
                            ELSE CAST({source}.completed_at AS TEXT) || '000000' END,
                        'execution_id', {source}.execution_id, 'parent_id', {source}.parent_id));
            END;"
        );
        transaction
            .execute_batch(&sql)
            .map_err(|error| error.to_string())?;
    }
    transaction.commit().map_err(|error| error.to_string())
}

fn bounds(query: &ObserveQuery) -> Result<(usize, usize), String> {
    if !(1..=1000).contains(&query.limit) || !(1024..=1_048_576).contains(&query.byte_limit) {
        return Err("OBSERVE_INVALID_LIMIT: rows must be 1..1000 and bytes 1024..1048576".into());
    }
    Ok((query.limit as usize, query.byte_limit as usize))
}

fn scope_parts(scope: &ObserveScope) -> Result<(Option<String>, Option<&str>), String> {
    match scope {
        ObserveScope::Global => Ok((None, None)),
        ObserveScope::Project { scope_kind, key } if !key.is_empty() => {
            let kind = serde_json::to_value(scope_kind).map_err(|error| error.to_string())?;
            Ok((kind.as_str().map(str::to_owned), Some(key)))
        }
        _ => Err("OBSERVE_INVALID_SCOPE: a project requires an exact scope key".into()),
    }
}

fn state(connection: &Connection) -> Result<(String, u64, u64), String> {
    connection
        .query_row(
            "SELECT instance, floor,
            COALESCE((SELECT seq FROM sqlite_sequence WHERE name='observation_versions'), 0)
         FROM observation_meta WHERE singleton=1",
            [],
            |row| {
                Ok((
                    row.get(0)?,
                    row.get::<_, i64>(1)? as u64,
                    row.get::<_, i64>(2)? as u64,
                ))
            },
        )
        .map_err(|error| error.to_string())
}

fn cursor(
    connection: &Connection,
    query: &ObserveQuery,
    sequence: u64,
) -> Result<ObserveCursor, String> {
    Ok(ObserveCursor {
        instance: state(connection)?.0,
        projection_version: PROJECTION_VERSION,
        query_digest: digest(query)?,
        sequence: sequence.to_string(),
    })
}

fn validate(
    connection: &Connection,
    query: &ObserveQuery,
    cursor: &ObserveCursor,
) -> Result<u64, String> {
    let (instance, floor, high) = state(connection)?;
    let sequence = cursor
        .sequence
        .parse::<u64>()
        .map_err(|_| "OBSERVE_INVALID_CURSOR: sequence must be an unsigned decimal integer")?;
    if cursor.instance != instance
        || cursor.projection_version != PROJECTION_VERSION
        || cursor.query_digest != digest(query)?
        || sequence > high
    {
        return Err(
            "OBSERVE_INVALID_CURSOR: instance, projection, scope, or filter changed".into(),
        );
    }
    if sequence < floor {
        return Err("OBSERVE_CURSOR_EXPIRED: take a new snapshot".into());
    }
    Ok(sequence)
}

fn decode(row: &rusqlite::Row<'_>) -> rusqlite::Result<Observation> {
    let text: String = row.get(6)?;
    let value: Value = serde_json::from_str(&text).map_err(|error| {
        rusqlite::Error::FromSqlConversionFailure(6, rusqlite::types::Type::Text, Box::new(error))
    })?;
    let optional = |key: &str| value[key].as_str().map(str::to_owned);
    Ok(Observation {
        id: row.get(1)?,
        revision: row.get::<_, i64>(0)?.to_string(),
        scope_kind: row.get(2)?,
        scope_key: row.get(3)?,
        track: row.get(4)?,
        deleted: row.get(5)?,
        source: optional("source").unwrap_or_default(),
        source_id: optional("source_id").unwrap_or_default(),
        label: optional("label").unwrap_or_default(),
        status: optional("status").unwrap_or_default(),
        start_time_ns: optional("start_time_ns").unwrap_or_default(),
        end_time_ns: optional("end_time_ns"),
        execution_id: optional("execution_id"),
        parent_id: optional("parent_id"),
    })
}

fn page(
    connection: &Connection,
    query: &ObserveQuery,
    after: u64,
    high: u64,
    snapshot: bool,
) -> Result<(Vec<Observation>, bool), String> {
    let (limit, bytes) = bounds(query)?;
    let (scope_kind, scope_key) = scope_parts(&query.scope)?;
    let latest = if snapshot {
        "AND v.deleted=0 AND NOT EXISTS (
            SELECT 1 FROM observation_versions n WHERE n.id=v.id
            AND n.sequence>v.sequence AND n.sequence<=?2)"
    } else {
        ""
    };
    let sql = format!(
        "SELECT v.sequence,v.id,v.scope_kind,v.scope_key,v.track,v.deleted,v.value
        FROM observation_versions v WHERE v.sequence>?1 AND v.sequence<=?2
        AND (?3 IS NULL OR (v.scope_kind=?3 AND v.scope_key=?4))
        AND (?5 IS NULL OR v.track=?5) AND (?6 IS NULL OR v.id=?6)
        {latest} ORDER BY v.sequence LIMIT ?7"
    );
    let mut statement = connection
        .prepare(&sql)
        .map_err(|error| error.to_string())?;
    let mapped = statement
        .query_map(
            params![
                after as i64,
                high as i64,
                scope_kind,
                scope_key,
                query.track,
                query.id,
                (limit + 1) as i64
            ],
            decode,
        )
        .map_err(|error| error.to_string())?;
    let mut rows = Vec::new();
    let mut used = 0;
    let mut more = false;
    for row in mapped {
        let row = row.map_err(|error| error.to_string())?;
        let size = serde_json::to_vec(&row)
            .map_err(|error| error.to_string())?
            .len();
        if rows.len() == limit || used + size > bytes {
            if rows.is_empty() {
                return Err("OBSERVE_ROW_TOO_LARGE: raise byte_limit for this row".into());
            }
            more = true;
            break;
        }
        used += size;
        rows.push(row);
    }
    Ok((rows, more))
}

impl Store {
    pub fn observe_capabilities(&self) -> ObserveCapabilities {
        ObserveCapabilities {
            projection_version: PROJECTION_VERSION,
            sources: vec!["invocation".into()],
            max_rows: 1000,
            max_bytes: 1_048_576,
            source_clock: "unix_epoch_ns; millisecond source precision".into(),
            historical_backfill: false,
        }
    }

    /// The transaction pins both the source revisions and high-water mark.
    pub fn observe_snapshot(
        &self,
        query: &ObserveQuery,
        next_page: Option<&str>,
    ) -> Result<ObservePage, String> {
        let mut connection = self.connection.lock().map_err(|error| error.to_string())?;
        let transaction = connection
            .transaction()
            .map_err(|error| error.to_string())?;
        let (after, high, cursor) = match next_page {
            Some(token) => {
                let position: SnapshotPosition = serde_json::from_str(token)
                    .map_err(|_| "OBSERVE_INVALID_CURSOR: invalid snapshot page")?;
                let high = validate(&transaction, query, &position.cursor)?;
                if position.after > high {
                    return Err("OBSERVE_INVALID_CURSOR: page exceeds snapshot".into());
                }
                (position.after, high, position.cursor)
            }
            None => {
                let high = state(&transaction)?.2;
                (0, high, cursor(&transaction, query, high)?)
            }
        };
        let (rows, has_more) = page(&transaction, query, after, high, true)?;
        let next_page = if has_more {
            Some(
                serde_json::to_string(&SnapshotPosition {
                    cursor: cursor.clone(),
                    after: rows
                        .last()
                        .expect("nonempty continued page")
                        .revision
                        .parse()
                        .map_err(|_| "invalid revision")?,
                })
                .map_err(|error| error.to_string())?,
            )
        } else {
            None
        };
        Ok(ObservePage {
            rows,
            cursor,
            next_page,
            has_more,
        })
    }

    pub fn observe_changes(
        &self,
        query: &ObserveQuery,
        after: &ObserveCursor,
    ) -> Result<ObservePage, String> {
        let mut connection = self.connection.lock().map_err(|error| error.to_string())?;
        let transaction = connection
            .transaction()
            .map_err(|error| error.to_string())?;
        let sequence = validate(&transaction, query, after)?;
        let high = state(&transaction)?.2;
        let (rows, has_more) = page(&transaction, query, sequence, high, false)?;
        let acknowledged = if has_more {
            rows.last()
                .expect("nonempty continued page")
                .revision
                .parse()
                .map_err(|_| "invalid revision")?
        } else {
            high
        };
        Ok(ObservePage {
            rows,
            cursor: cursor(&transaction, query, acknowledged)?,
            next_page: None,
            has_more,
        })
    }

    pub fn observe_projects(&self) -> Result<Vec<ObserveProject>, String> {
        let connection = self.connection.lock().map_err(|error| error.to_string())?;
        let mut statement = connection.prepare(
            "SELECT scope_kind,scope_key,COUNT(*) FROM observation_versions v
             WHERE scope_key IS NOT NULL AND deleted=0
             AND NOT EXISTS(SELECT 1 FROM observation_versions n WHERE n.id=v.id AND n.sequence>v.sequence)
             GROUP BY scope_kind,scope_key ORDER BY scope_kind,scope_key LIMIT 1000")
            .map_err(|error| error.to_string())?;
        statement
            .query_map([], |row| {
                Ok(ObserveProject {
                    scope_kind: row.get(0)?,
                    key: row.get(1)?,
                    rows: row.get::<_, i64>(2)? as u64,
                })
            })
            .map_err(|error| error.to_string())?
            .collect::<Result<Vec<_>, _>>()
            .map_err(|error| error.to_string())
    }

    pub fn observe_tracks(&self, query: &ObserveQuery) -> Result<Vec<ObserveTrack>, String> {
        bounds(query)?;
        let (scope_kind, scope_key) = scope_parts(&query.scope)?;
        let connection = self.connection.lock().map_err(|error| error.to_string())?;
        let mut statement = connection.prepare(
            "SELECT track,COUNT(*) FROM observation_versions v WHERE deleted=0
             AND (?1 IS NULL OR (scope_kind=?1 AND scope_key=?2))
             AND (?3 IS NULL OR track=?3) AND (?4 IS NULL OR id=?4)
             AND NOT EXISTS(SELECT 1 FROM observation_versions n WHERE n.id=v.id AND n.sequence>v.sequence)
             GROUP BY track ORDER BY track LIMIT ?5").map_err(|error| error.to_string())?;
        statement
            .query_map(
                params![scope_kind, scope_key, query.track, query.id, query.limit],
                |row| {
                    Ok(ObserveTrack {
                        id: row.get(0)?,
                        rows: row.get::<_, i64>(1)? as u64,
                    })
                },
            )
            .map_err(|error| error.to_string())?
            .collect::<Result<Vec<_>, _>>()
            .map_err(|error| error.to_string())
    }

    /// Prunes superseded versions atomically while retaining each current row.
    /// Consumers older than the boundary must explicitly resnapshot.
    pub fn prune_observations(&self, through: u64) -> Result<usize, String> {
        let mut connection = self.connection.lock().map_err(|error| error.to_string())?;
        let transaction = connection
            .transaction_with_behavior(TransactionBehavior::Immediate)
            .map_err(|error| error.to_string())?;
        let (_, floor, high) = state(&transaction)?;
        if through < floor || through > high {
            return Err("OBSERVE_INVALID_RETENTION: boundary outside retained sequence".into());
        }
        let removed = transaction
            .execute(
                "DELETE FROM observation_versions WHERE sequence<=?1
             AND sequence NOT IN (SELECT MAX(sequence) FROM observation_versions GROUP BY id)",
                [through as i64],
            )
            .map_err(|error| error.to_string())?;
        transaction
            .execute(
                "UPDATE observation_meta SET floor=?1 WHERE singleton=1",
                [through as i64],
            )
            .map_err(|error| error.to_string())?;
        transaction.commit().map_err(|error| error.to_string())?;
        Ok(removed)
    }
}