Skip to main content

orch/store/
mod.rs

1//! Unified task store — SQLite as single source of truth.
2//!
3//! `TaskStore` replaces the legacy `internal_tasks` table and GitHub labels as the
4//! authoritative task state. External backends (GitHub, Linear) become sync adapters
5//! that mirror state changes outward.
6//!
7//! Uses sqlx for async SQLite access with file-based migrations.
8//!
9//! All task state, metrics, KV, and rate limits flow through this module.
10//!
11//! # Safety Warning: Never Use `SELECT *`
12//!
13//! Using `SELECT *` with `sqlx-sqlite`'s prepared statement cache can cause **OOB
14//! (out-of-bounds) panics** when the schema changes (e.g., new migration adds a column).
15//!
16//! The issue occurs because:
17//! 1. SQLite caches prepared statements with column metadata at prepare time
18//! 2. When a migration adds a new column, `sqlite3_column_count()` returns the updated count
19//! 3. The cached statement's column metadata vector was built with the old count
20//! 4. Accessing column index N when only N columns exist (0..N-1) causes a panic
21//!
22//! **Solution:** Always use explicit column lists via constants like `TASK_COLS`,
23//! `CONTROL_MESSAGE_COLS`, `JOB_STATE_COLS`, etc. See `tasks.rs` and `control.rs`
24//! for examples. CI will fail if `SELECT *` is detected in store files.
25
26pub(crate) mod control;
27pub(crate) mod dependencies;
28mod helpers;
29pub(crate) mod jobs;
30pub(crate) mod kv;
31mod metrics;
32pub(crate) mod replica;
33pub(crate) mod tasks;
34#[cfg(test)]
35mod tests;
36mod usage;
37
38use anyhow::Context;
39use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqlitePoolOptions};
40use std::fmt;
41use std::fmt::Debug;
42use std::fmt::Formatter;
43use std::path::Path;
44use std::path::PathBuf;
45use std::sync::{Arc, OnceLock};
46use std::time::Duration;
47
48pub use control::{ChatCostSummary, ControlMessage, MemoryEntry, parse_since_duration};
49pub use dependencies::{
50    BlockerSnapshot, DepSource, NewDependency, ObservedPullRequest, TaskDependency,
51    WaitingOnDependency, blocker_failure_kind, blocker_is_satisfied, blocker_snapshot_key,
52    dep_label, worktree_refresh_key,
53};
54pub use helpers::{
55    StoreResult, get_recent_memory, get_recent_memory_result, get_reported_cost,
56    get_reported_cost_result, get_task_field, get_task_field_direct, get_token_usage,
57    get_token_usage_result, opt_store_get_task, opt_store_get_task_by_id, resolve_store_id,
58    store_increment, store_increment_by_id, store_log_activity, store_reset_counters,
59    store_reset_failure_counters, store_set, store_set_by_id, store_set_result,
60    store_set_result_by_id, store_touch_updated_at,
61};
62pub use jobs::JobState;
63pub use metrics::{
64    AgentStat, CostByGroup, CostPeriod, CostSummary, ErrorStat, HighReviewCycleTask,
65    InsertTaskMetric, MetricsSummary, SlowTaskInfo,
66};
67use sqlx::error::DatabaseError;
68pub use tasks::{
69    CliTaskRef, CliTaskSpec, CompleteRun, NewTask, RunTokenUsage, StartRun, StatsEvents,
70    StatusChange, StoreRoute, Task, TaskActivity, TaskRun, TaskStatus, UpsertExternal,
71    parse_cli_task_spec,
72};
73use tokio::time;
74pub use usage::TokenUsage;
75
76use crate::cluster;
77use crate::cluster::Command;
78use crate::cluster::Node;
79use crate::cluster::Reply;
80use crate::cluster::Request;
81use crate::cluster::local;
82use crate::cluster::local::Endpoint;
83use crate::home;
84
85/// Default database path: `~/.orch/orch.db`
86pub async fn default_db_path() -> anyhow::Result<PathBuf> {
87    home::db_path().await
88}
89
90/// The unified task store backed by SQLite (via sqlx).
91#[derive(Clone)]
92pub struct TaskStore {
93    pool: SqlitePool,
94    /// Set once on the daemon. Without it, replicated writes apply directly, as in tests and the CLI
95    raft: Arc<OnceLock<Node>>,
96    /// Set on the CLI when a daemon runs, so its writes go through the daemon's log
97    daemon: Arc<OnceLock<Endpoint>>,
98}
99
100impl Debug for TaskStore {
101    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
102        f.debug_struct("TaskStore").finish_non_exhaustive()
103    }
104}
105
106impl TaskStore {
107    /// Open (or create) the store at the given database path.
108    ///
109    /// Runs file-based migrations from the `migrations/` directory.
110    pub async fn open(db_path: &Path) -> anyhow::Result<Self> {
111        let options = SqliteConnectOptions::new()
112            .filename(db_path)
113            .journal_mode(SqliteJournalMode::Wal)
114            .create_if_missing(true)
115            .busy_timeout(Duration::from_secs(5))
116            .pragma("foreign_keys", "ON");
117
118        let pool = SqlitePoolOptions::new()
119            .max_connections(5)
120            .connect_with(options)
121            .await
122            .with_context(|| format!("opening task store: {}", db_path.display()))?;
123
124        let store = Self {
125            pool,
126            raft: Arc::default(),
127            daemon: Arc::default(),
128        };
129        store.migrate().await?;
130        Ok(store)
131    }
132
133    /// Open a store with a single connection.
134    ///
135    /// Uses `max_connections(1)` to avoid WAL visibility issues where a write
136    /// on one pooled connection isn't visible to a read on another. Used by
137    /// integration tests that need read-your-own-writes consistency.
138    pub async fn open_single(db_path: &Path) -> anyhow::Result<Self> {
139        let options = SqliteConnectOptions::new()
140            .filename(db_path)
141            .journal_mode(SqliteJournalMode::Wal)
142            .create_if_missing(true)
143            .busy_timeout(Duration::from_secs(5))
144            .pragma("foreign_keys", "ON");
145
146        let pool = SqlitePoolOptions::new()
147            .max_connections(1)
148            .connect_with(options)
149            .await
150            .with_context(|| format!("opening task store (single): {}", db_path.display()))?;
151
152        let store = Self {
153            pool,
154            raft: Arc::default(),
155            daemon: Arc::default(),
156        };
157        store.migrate().await?;
158        Ok(store)
159    }
160
161    /// Open an in-memory store (for testing).
162    #[cfg(test)]
163    pub async fn open_memory() -> anyhow::Result<Self> {
164        let options = SqliteConnectOptions::new()
165            .filename(":memory:")
166            .pragma("foreign_keys", "ON");
167        let pool = SqlitePoolOptions::new()
168            .max_connections(1)
169            .connect_with(options)
170            .await?;
171
172        let store = Self {
173            pool,
174            raft: Arc::default(),
175            daemon: Arc::default(),
176        };
177        store.migrate().await?;
178        Ok(store)
179    }
180
181    /// Run migrations.
182    async fn migrate(&self) -> anyhow::Result<()> {
183        sqlx::migrate!("./migrations")
184            .run(&self.pool)
185            .await
186            .context("running task store migrations")?;
187        Ok(())
188    }
189
190    pub fn attach_node(&self, node: Node) {
191        if self.raft.set(node).is_err() {
192            tracing::warn!("raft already attached to the task store");
193        }
194    }
195
196    /// True on the node that decides: the Raft leader, or a store with no Raft, as in the CLI and tests.
197    /// A single node always leads, so one machine runs every phase as before
198    pub fn is_leader(&self) -> bool {
199        self.raft.get().is_none_or(super::cluster::Node::is_leader)
200    }
201
202    /// Writes go through Raft or a daemon that runs it, so they reach the other machines
203    pub(crate) fn replicates(&self) -> bool {
204        self.raft.get().is_some() || self.daemon.get().is_some()
205    }
206
207    pub fn node(&self) -> Option<&Node> {
208        self.raft.get()
209    }
210
211    pub fn attach_daemon(&self, endpoint: Endpoint) {
212        let _ = self.daemon.set(endpoint);
213    }
214
215    /// Propose a replicated write through Raft, or apply it directly when no Raft is attached
216    pub(crate) async fn write(&self, cmd: Command) -> anyhow::Result<Reply> {
217        let Some(node) = self.raft.get() else {
218            if let Some(endpoint) = self.daemon.get() {
219                match local::send(endpoint, &cmd).await {
220                    Ok(reply) => return reply_result(reply),
221                    // Nothing was sent, so applying here cannot apply the command twice
222                    Err(e) if e.is_connect() => {
223                        tracing::debug!(err = %e, "daemon not listening, writing directly");
224                    }
225                    Err(e) => return Err(e.into()),
226                }
227            }
228
229            return self.apply_direct(cmd).await;
230        };
231        let req = Request {
232            now: cluster::now_stamp(),
233            cmd,
234        };
235        reply_result(node.write(req).await?)
236    }
237
238    pub(crate) async fn write_int(&self, cmd: Command) -> anyhow::Result<i64> {
239        match self.write(cmd).await? {
240            Reply::Int(n) => Ok(n),
241            other => anyhow::bail!("expected an integer reply, got {other:?}"),
242        }
243    }
244
245    pub(crate) async fn write_bool(&self, cmd: Command) -> anyhow::Result<bool> {
246        match self.write(cmd).await? {
247            Reply::Bool(b) => Ok(b),
248            other => anyhow::bail!("expected a boolean reply, got {other:?}"),
249        }
250    }
251
252    /// The same SQL apply runs, in one transaction on this node only
253    pub(crate) async fn apply_direct(&self, cmd: Command) -> anyhow::Result<Reply> {
254        let mut attempt = 0;
255        loop {
256            match self.apply_once(cmd.clone()).await {
257                // A dropped transaction rolls back, so running the command again is safe
258                Err(e) if attempt < WRITE_LOCK_RETRIES && is_write_lock_error(&e) => {
259                    attempt += 1;
260                    tracing::warn!(attempt, err = %e, "database is locked, retrying the write");
261                    time::sleep(Duration::from_millis(100 << (attempt - 1))).await;
262                }
263                result => return result,
264            }
265        }
266    }
267
268    async fn apply_once(&self, cmd: Command) -> anyhow::Result<Reply> {
269        let mut tx = self.begin_write().await?;
270        let reply = cmd.apply(&mut tx, &cluster::now_stamp(), false).await?;
271        tx.commit().await?;
272        Ok(reply)
273    }
274
275    /// Start a transaction that holds the write lock from its first statement.
276    ///
277    /// A deferred `BEGIN` that reads and then writes fails at once with `database is locked` when
278    /// another connection commits in between, and `busy_timeout` cannot wait that out. `BEGIN
279    /// IMMEDIATE` waits for the lock up front, where `busy_timeout` applies
280    pub(crate) async fn begin_write(
281        &self,
282    ) -> sqlx::Result<sqlx::Transaction<'static, sqlx::Sqlite>> {
283        self.pool.begin_with("BEGIN IMMEDIATE").await
284    }
285
286    /// Get a reference to the underlying pool (for advanced queries).
287    pub fn pool(&self) -> &SqlitePool {
288        &self.pool
289    }
290}
291
292/// Retries for a write that still finds the database locked after `busy_timeout`
293const WRITE_LOCK_RETRIES: u32 = 4;
294
295/// `SQLITE_BUSY` (5) or `SQLITE_BUSY_SNAPSHOT` (517) anywhere in the error chain
296fn is_write_lock_error(err: &anyhow::Error) -> bool {
297    err.chain().any(|cause| {
298        cause
299            .downcast_ref::<sqlx::Error>()
300            .and_then(|e| e.as_database_error())
301            .and_then(DatabaseError::code)
302            .is_some_and(|code| code == "5" || code == "517")
303    })
304}
305
306fn reply_result(reply: Reply) -> anyhow::Result<Reply> {
307    match reply {
308        Reply::Err(e) => Err(anyhow::anyhow!(e)),
309        other => Ok(other),
310    }
311}