use inillucent_driver::{Status, Support};
use inillucent_value::Value;
use super::outcome::{columns_from, table, Column, Failed, Outcome};
use super::{Arguments, Context};
use crate::json::{self, Json};
pub fn value_to_json(value: &Value<'static>) -> Json {
match value {
Value::Null => Json::Null,
Value::Integer(number) => Json::Int(*number),
Value::Real(number) => Json::Real(*number),
Value::Text(text) => json::text(String::from_utf8_lossy(text.raw()).into_owned()),
Value::Blob(bytes) => {
let mut hex = String::with_capacity(bytes.raw().len().saturating_mul(2));
for byte in bytes.raw() {
hex.push_str(&format!("{byte:02x}"));
}
json::object(vec![("blob", json::text(hex))])
}
}
}
fn literal_of(value: &Json) -> Result<String, Failed> {
match value {
Json::Null => Ok("NULL".to_string()),
Json::Bool(true) => Ok("1".to_string()),
Json::Bool(false) => Ok("0".to_string()),
Json::Int(number) => Ok(number.to_string()),
Json::Real(number) if number.is_finite() => Ok(format!("{number:?}")),
Json::Real(_) => Err(Failed::misuse(
"a parameter cannot be NaN or infinity: SQL has no spelling for either.",
)),
Json::Text(text) if is_blob_literal(text) => Ok(text.clone()),
Json::Text(text) => Ok(format!("'{}'", text.replace('\'', "''"))),
Json::Array(values) => vector_literal(values),
Json::Object(fields) => blob_literal(fields),
}
}
fn vector_literal(values: &[Json]) -> Result<String, Failed> {
if values.is_empty() {
return Err(Failed::misuse(
"a parameter that is an array is a vector, so it needs at least one number.",
));
}
let mut hex = String::from("x'");
for value in values {
let number = match value {
Json::Int(whole) => *whole as f64,
Json::Real(real) if real.is_finite() => *real,
_ => {
return Err(Failed::misuse(
"a parameter that is an array is a vector, so every element has to be a \
finite number.",
))
}
};
for byte in (number as f32).to_le_bytes() {
hex.push_str(&format!("{byte:02x}"));
}
}
hex.push('\'');
Ok(hex)
}
fn blob_literal(fields: &[(String, Json)]) -> Result<String, Failed> {
let held = fields
.iter()
.find(|(name, _)| name == "blob")
.map(|(_, value)| value);
let Some(Json::Text(hex)) = held else {
return Err(Failed::misuse(
"a parameter has to be a string, a number, a boolean, null, an array of numbers \
for a vector, or {\"blob\": \"<hex>\"} for bytes.",
));
};
if hex.is_empty() || hex.len() % 2 != 0 || !hex.chars().all(|digit| digit.is_ascii_hexdigit()) {
return Err(Failed::misuse(
"the value of \"blob\" has to be an even number of hexadecimal digits.",
));
}
Ok(format!("x'{hex}'"))
}
fn is_blob_literal(text: &str) -> bool {
let Some(inner) = text
.strip_prefix("x'")
.and_then(|rest| rest.strip_suffix('\''))
else {
return false;
};
!inner.is_empty()
&& inner.len() % 2 == 0
&& inner.chars().all(|digit| digit.is_ascii_hexdigit())
}
fn quoted(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
fn quoted_text(text: &str) -> String {
format!("'{}'", text.replace('\'', "''"))
}
fn produce(
context: &mut Context,
command: &str,
sql: &str,
params: &[Json],
limit: usize,
) -> Result<Outcome, Failed> {
context.refuse_if_it_writes(sql)?;
refuse_a_script(context, command, sql)?;
let mut bound = Vec::with_capacity(params.len());
for value in params {
let literal = literal_of(value)?;
let held = context
.shell()
.collect(&format!("SELECT {literal}"))
.map_err(|failure| Failed::from_shell(&failure))?
.1
.first()
.and_then(|row| row.first())
.cloned()
.unwrap_or(Value::Null);
bound.push(inillucent_tree::datum::OwnedDatum::from(&held));
}
let started = std::time::Instant::now();
let collected = context.shell().collect_bound(sql, &bound);
let elapsed = started.elapsed().as_secs_f64() * 1000.0;
let (names, rows) = collected.map_err(|failure| Failed::from_shell(&failure))?;
Ok(rows_to_outcome(
context, command, names, rows, limit, elapsed,
))
}
fn rows_to_outcome(
context: &mut Context,
command: &str,
names: Vec<String>,
rows: Vec<Vec<Value<'static>>>,
limit: usize,
elapsed: f64,
) -> Outcome {
let total = rows.len();
let kept = if limit == 0 { total } else { limit.min(total) };
let cells: Vec<Vec<Json>> = rows
.iter()
.take(kept)
.map(|row| row.iter().map(value_to_json).collect())
.collect();
let columns = columns_from(&names, &cells);
let connection = context.shell().connection();
let changes = connection.total_changes().unwrap_or_default();
let rowid = connection.last_insert_rowid().unwrap_or_default();
let _ = connection;
let mut text = table(&columns, &cells, &context.null);
if kept < total {
text.push_str(&format!("\n({kept} of {total} rows)"));
}
Outcome {
command: command.to_string(),
columns,
rows: cells,
total,
more: kept < total,
changes,
last_insert_rowid: rowid,
elapsed_ms: elapsed,
text,
extra: Vec::new(),
}
}
fn limit_of(context: &Context, arguments: &Arguments) -> Result<usize, Failed> {
let asked = match arguments.integer("limit") {
Some(asked) if asked < 0 => {
return Err(Failed::misuse(format!(
"limit={asked} is not a number of rows. Write 0 for every row, or a positive \
count."
)))
}
Some(asked) => asked as usize,
None => context.limit,
};
context.cap_rows(asked)
}
fn refuse_a_script(context: &mut Context, command: &str, sql: &str) -> Result<(), Failed> {
let Some(rest) = context.shell().trailing_statement(sql) else {
return Ok(());
};
Err(Failed::said(
Status::InvalidState,
format!(
"{command} runs one statement and this is several; the next one begins {rest:?}. \
Use `batch`, which runs them all in one transaction."
),
))
}
fn bound_values(context: &Context, arguments: &Arguments) -> Result<Vec<Json>, Failed> {
let inline = arguments.values("params");
let Some(named) = arguments.text("params-file") else {
return Ok(inline);
};
if !inline.is_empty() {
return Err(Failed::misuse(
"give the values in 'params' or in 'params-file', not both.",
));
}
let text = match named {
"-" if context.confined() => {
return Err(Failed::said(
Status::InvalidState,
"this surface is confined to a directory with --root, and '-' reads the \
parameters from standard input, which such a surface does not have to itself. \
Write them with 'params', or name a file inside the root.",
))
}
"-" => {
let mut held = String::new();
std::io::Read::read_to_string(&mut std::io::stdin(), &mut held)
.map_err(|error| Failed::said(Status::Io, format!("standard input: {error}")))?;
held
}
path => {
let admitted = context.confine(path)?;
std::fs::read_to_string(&admitted)
.map_err(|error| Failed::said(Status::Io, format!("{path}: {error}")))?
}
};
if text.len() > MAX_PARAMS_FILE_BYTES {
return Err(Failed::said(
Status::TooBig,
format!(
"'params-file' is {} bytes, past the {MAX_PARAMS_FILE_BYTES} byte limit. \
Parameters are a list of values, not a data file.",
text.len()
),
));
}
let parsed = json::parse(text.trim())
.map_err(|why| Failed::misuse(format!("'params-file' is not JSON: {why}")))?;
match parsed {
Json::Array(values) => Ok(values),
_ => Err(Failed::misuse("'params-file' has to hold a JSON array.")),
}
}
const MAX_PARAMS_FILE_BYTES: usize = 1024 * 1024;
pub fn query(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let sql = arguments.required_text("sql")?.to_string();
let params = bound_values(context, arguments)?;
let limit = limit_of(context, arguments)?;
produce(context, "query", &sql, ¶ms, limit)
}
pub fn exec(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let sql = arguments.required_text("sql")?.to_string();
let params = bound_values(context, arguments)?;
let before = context
.shell()
.connection()
.total_changes()
.map_err(|error| Failed::from_engine(&error))?;
let mut produced = produce(context, "exec", &sql, ¶ms, 0)?;
let after = context
.shell()
.connection()
.total_changes()
.map_err(|error| Failed::from_engine(&error))?;
produced.changes = after - before;
produced.text = match produced.rows.is_empty() {
true => format!(
"ok. {} row{} changed.",
produced.changes,
if produced.changes == 1 { "" } else { "s" }
),
false => produced.text.clone(),
};
Ok(produced)
}
pub fn batch(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let sql = arguments.required_text("sql")?.to_string();
context.refuse_if_it_writes(&sql)?;
let joined = !context
.shell()
.connection()
.autocommit()
.map_err(|error| Failed::from_engine(&error))?;
let before = context
.shell()
.connection()
.total_changes()
.map_err(|error| Failed::from_engine(&error))?;
if !joined {
context
.shell()
.execute("BEGIN")
.map_err(|message| Failed::said(Status::Syntax, message))?;
}
let ran = context.shell().connection().execute_batch(&sql);
if let Err(error) = ran {
let failed = Failed::from_engine(&error);
if !joined {
if let Err(second) = context.shell().execute("ROLLBACK") {
return Err(Failed {
message: format!("{} (and the rollback failed: {second})", failed.message),
..failed
});
}
}
return Err(failed);
}
if !joined {
context
.shell()
.execute("COMMIT")
.map_err(|message| Failed::said(Status::Syntax, message))?;
}
let after = context
.shell()
.connection()
.total_changes()
.map_err(|error| Failed::from_engine(&error))?;
let changes = after - before;
let mut produced = Outcome::said(
"batch",
format!(
"ok. {changes} row{} changed.",
if changes == 1 { "" } else { "s" }
),
);
produced.changes = changes;
produced.extra.push((
"transaction".to_string(),
Json::Text(
if joined {
"joined the open transaction; not committed"
} else {
"committed"
}
.to_string(),
),
));
Ok(produced)
}
fn take_failure(context: &mut Context, printed: &str) -> Option<Failed> {
let shell = context.shell();
let failed = std::mem::replace(&mut shell.failed, false);
let error = shell.first_error.take();
if !failed {
return None;
}
let message = printed.trim_end().to_string();
Some(match error {
Some(error) => Failed {
message,
..Failed::from_engine(&error)
},
None => Failed::said(Status::Syntax, message),
})
}
pub fn run_input(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let input = arguments.required_text("input")?.to_string();
if context.readonly() {
for line in input.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('.') {
continue;
}
context.refuse_if_it_writes(trimmed)?;
}
}
let printed = context.collect_output(&input);
if let Some(failed) = take_failure(context, &printed) {
return Err(failed);
}
Ok(Outcome::said("run", printed.trim_end()))
}
pub fn create(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let path = arguments.required_text("path")?.to_string();
let confined = context.confine(&path)?;
if confined.exists() {
return Err(Failed::said(
Status::InvalidState,
format!("\"{path}\" already exists. Open it instead of creating it."),
));
}
let named = confined.to_string_lossy().into_owned();
context.use_database(&named)?;
context
.shell()
.execute("PRAGMA user_version = 0")
.map_err(|message| Failed::said(Status::Io, message))?;
Ok(Outcome::said("create", format!("created {named}")).with("path", json::text(&named)))
}
fn listing(context: &mut Context, command: &str, sql: &str) -> Result<Outcome, Failed> {
produce(context, command, sql, &[], 0)
}
pub fn tables(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let mut sql = String::from(
"SELECT name, type FROM sqlite_master WHERE type IN ('table','view') \
AND name NOT LIKE 'sqlite_%'",
);
if let Some(pattern) = arguments.text("pattern") {
sql.push_str(&format!(" AND name LIKE {}", quoted_text(pattern)));
}
sql.push_str(" ORDER BY name");
listing(context, "tables", &sql)
}
pub fn indexes(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let pattern = arguments.text("pattern").map(str::to_string);
let mut sql =
String::from("SELECT name, tbl_name AS \"table\" FROM sqlite_master WHERE type = 'index'");
if let Some(pattern) = &pattern {
sql.push_str(&format!(" AND name LIKE {}", quoted_text(pattern)));
}
context.refuse_if_it_writes(&sql)?;
let started = std::time::Instant::now();
let (names, mut rows) = context
.shell()
.collect(&sql)
.map_err(|failure| Failed::from_shell(&failure))?;
rows.extend(module_indexes(context, pattern.as_deref())?);
rows.sort_by(|left, right| {
let key = |row: &Vec<Value<'static>>| {
(
row.get(1).map(text_of_value).unwrap_or_default(),
row.first().map(text_of_value).unwrap_or_default(),
)
};
key(left).cmp(&key(right))
});
let elapsed = started.elapsed().as_secs_f64() * 1000.0;
Ok(rows_to_outcome(context, "indexes", names, rows, 0, elapsed))
}
fn module_indexes(
context: &mut Context,
pattern: Option<&str>,
) -> Result<Vec<Vec<Value<'static>>>, Failed> {
let tables = context
.shell()
.column("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'");
let mut found = Vec::new();
for table in tables {
let listed = context
.shell()
.collect(&format!("PRAGMA index_list({})", quoted_text(&table)))
.map_err(|failure| Failed::from_shell(&failure))?
.1;
for row in listed {
if row.get(3).map(text_of_value).as_deref() != Some("v") {
continue;
}
let Some(name) = row.get(1).map(text_of_value) else {
continue;
};
if let Some(pattern) = pattern {
if !like(&name, pattern) {
continue;
}
}
let (Ok(named), Ok(owner)) = (
Value::owned_text(name.as_bytes()),
Value::owned_text(table.as_bytes()),
) else {
continue;
};
found.push(vec![named, owner]);
}
}
Ok(found)
}
fn text_of_value(value: &Value<'static>) -> String {
match value {
Value::Text(text) => String::from_utf8_lossy(text.raw()).into_owned(),
_ => String::new(),
}
}
fn like(name: &str, pattern: &str) -> bool {
let folded = name.to_lowercase();
let wanted = pattern.to_lowercase();
let parts: Vec<&str> = wanted.split('%').collect();
let mut at = 0usize;
for (which, part) in parts.iter().enumerate() {
if part.is_empty() {
continue;
}
let Some(found) = folded.get(at..).and_then(|rest| rest.find(part)) else {
return false;
};
if which == 0 && !wanted.starts_with('%') && found != 0 {
return false;
}
at = at.saturating_add(found).saturating_add(part.len());
}
if !wanted.ends_with('%') {
if let Some(last) = parts.last() {
if !last.is_empty() && at != folded.len() {
return false;
}
}
}
true
}
pub fn databases(context: &mut Context, _arguments: &Arguments) -> Result<Outcome, Failed> {
listing(context, "databases", "PRAGMA database_list")
}
pub fn schema(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let mut line = String::from(".schema");
if arguments.flag("indent") {
line.push_str(" --indent");
}
if let Some(pattern) = arguments.text("pattern") {
line.push(' ');
line.push_str(pattern);
}
let printed = context.collect_output(&line);
context.shell().failed = false;
context.shell().first_error = None;
Ok(Outcome::said("schema", printed.trim_end()))
}
pub fn describe(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let name = arguments.required_text("table")?.to_string();
let info = format!(
"SELECT cid, name, type, \"notnull\", dflt_value, pk, CASE hidden WHEN 1 THEN 'hidden' WHEN 2 THEN 'generated virtual' WHEN 3 THEN 'generated stored' ELSE '' END AS kind FROM pragma_table_xinfo({})",
quoted_text(&name)
);
let mut produced = produce(context, "describe", &info, &[], 0)?;
if produced.rows.is_empty() {
return Err(Failed::said(
Status::NotFound,
format!("no such table: {name}"),
));
}
let ddl = context
.shell()
.scalar(&format!(
"SELECT sql FROM sqlite_master WHERE name = {}",
quoted_text(&name)
))
.unwrap_or_default();
let index_rows = context.shell().column(&format!(
"SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = {} ORDER BY name",
quoted_text(&name)
));
let count = context
.shell()
.scalar(&format!("SELECT count(*) FROM {}", quoted(&name)))
.unwrap_or_default();
let indexes: Vec<Json> = index_rows.iter().map(json::text).collect();
let drawn = table(&produced.columns, &produced.rows, &context.null);
produced.text = format!(
"{name}: {} column{}, {count} row{}\n\n{drawn}\n\nindexes: {}\n\n{ddl}",
produced.rows.len(),
if produced.rows.len() == 1 { "" } else { "s" },
if count == "1" { "" } else { "s" },
match index_rows.is_empty() {
true => "none".to_string(),
false => index_rows.join(", "),
}
);
Ok(produced
.with("table", json::text(&name))
.with("ddl", json::text(ddl))
.with("indexes", Json::Array(indexes))
.with(
"row_count_in_table",
Json::Int(count.parse::<i64>().unwrap_or(-1)),
))
}
pub fn explain(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let sql = arguments.required_text("sql")?.to_string();
let plan = context
.shell()
.connection()
.explain(&sql)
.map_err(|error| Failed::from_engine(&error))?;
let rows: Vec<Vec<Json>> = plan.iter().map(|line| vec![json::text(line)]).collect();
let columns = vec![Column {
name: "plan".to_string(),
kind: "text".to_string(),
}];
Ok(Outcome {
command: "explain".to_string(),
text: plan.join("\n"),
total: rows.len(),
rows,
columns,
more: false,
changes: 0,
last_insert_rowid: 0,
elapsed_ms: 0.0,
extra: Vec::new(),
})
}
fn dot(context: &mut Context, command: &str, line: &str) -> Result<Outcome, Failed> {
let guarded = std::mem::replace(&mut context.shell().safe, false);
let printed = context.collect_output(line);
context.shell().safe = guarded;
if let Some(failed) = take_failure(context, &printed) {
return Err(failed);
}
Ok(Outcome::said(command, printed.trim_end()))
}
pub fn dump(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let mut line = String::from(".dump");
if arguments.flag("data_only") {
line.push_str(" --data-only");
}
if let Some(objects) = arguments.text("objects") {
line.push(' ');
line.push_str(objects);
}
dot(context, "dump", &line)
}
pub fn import(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let file = arguments.required_text("file")?.to_string();
let table_name = arguments.required_text("table")?.to_string();
let confined = context.confine(&file)?;
let mut line = String::from(".import");
match arguments.text("format").unwrap_or("csv") {
"csv" => line.push_str(" --csv"),
"ascii" => line.push_str(" --ascii"),
"tabs" => line.push_str(" --colsep \"\t\""),
other => {
return Err(Failed::misuse(format!(
"'{other}' is not a format this reads. Use csv, tabs or ascii."
)))
}
}
if let Some(skip) = arguments.integer("skip") {
line.push_str(&format!(" --skip {skip}"));
}
line.push_str(&format!(
" \"{}\" \"{table_name}\"",
confined.to_string_lossy()
));
let before_changes = context
.shell()
.connection()
.total_changes()
.map_err(|error| Failed::from_engine(&error))?;
let before_rows = row_count(context, &table_name);
let mut produced = dot(context, "import", &line)?;
let after_changes = context
.shell()
.connection()
.total_changes()
.map_err(|error| Failed::from_engine(&error))?;
produced.changes = after_changes - before_changes;
if produced.changes == 0 {
produced.changes = row_count(context, &table_name).saturating_sub(before_rows);
}
if produced.text.is_empty() {
produced.text = format!("imported {} rows into {table_name}", produced.changes);
}
Ok(produced)
}
fn row_count(context: &mut Context, table: &str) -> i64 {
let sql = format!("SELECT count(*) FROM \"{}\"", table.replace('"', "\"\""));
context
.shell()
.scalar(&sql)
.and_then(|text| text.parse::<i64>().ok())
.unwrap_or(0)
}
pub fn export(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let sql = match (arguments.text("sql"), arguments.text("table")) {
(Some(_), Some(_)) => {
return Err(Failed::misuse(
"export accepts either 'sql' or 'table', not both.",
))
}
(Some(sql), None) => sql.to_string(),
(None, Some(name)) => format!("SELECT * FROM {}", quoted(name)),
(None, None) => return Err(Failed::misuse("export needs either 'sql' or 'table'.")),
};
let format = arguments.text("format").unwrap_or("csv").to_string();
let mode = match format.as_str() {
"csv" | "json" | "tabs" | "markdown" | "insert" | "quote" | "line" | "html" => format,
other => {
return Err(Failed::misuse(format!(
"'{other}' is not an export format. Use csv, json, tabs, markdown, insert, \
quote, line or html."
)))
}
};
context.refuse_if_it_writes(&sql)?;
let destination = match arguments.text("out") {
None => None,
Some(out) => Some(context.confine(out)?),
};
if let Some(path) = destination.as_ref() {
let named = path.to_string_lossy().into_owned();
context
.shell()
.redirect(Some(&named), true)
.map_err(|message| {
Failed::said(Status::Io, format!("cannot open \"{named}\": {message}"))
})?;
}
let script = format!(".mode {mode}\n.headers on\n{sql};");
let printed = context.collect_output(&script);
let rows = context.shell().rows_since_redirect;
if destination.is_some() {
let _ = context.shell().redirect(None, false);
}
if let Some(failed) = take_failure(context, &printed) {
return Err(failed);
}
let Some(path) = destination else {
return Ok(Outcome::said("export", printed.trim_end()));
};
wrote_a_file(&path, rows)
}
fn wrote_a_file(path: &std::path::Path, rows: usize) -> Result<Outcome, Failed> {
let named = path.to_string_lossy().into_owned();
let bytes = std::fs::metadata(path)
.map(|found| found.len())
.unwrap_or(0);
let mut produced = Outcome::said(
"export",
format!(
"wrote {rows} row{} ({bytes} bytes) to {named}",
if rows == 1 { "" } else { "s" }
),
);
produced.total = rows;
Ok(produced
.with("wrote", json::text(&named))
.with("bytes", Json::Int(bytes as i64)))
}
pub fn backup(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let file = arguments.required_text("file")?.to_string();
let confined = context.confine(&file)?;
let named = confined.to_string_lossy().into_owned();
context
.shell()
.backup_to(&named)
.map_err(|message| Failed::said(Status::Io, message))?;
Ok(Outcome::said("backup", format!("wrote {named}")).with("wrote", json::text(&named)))
}
pub fn restore(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let file = arguments.required_text("file")?.to_string();
let confined = context.confine(&file)?;
if !confined.is_file() {
return Err(Failed::said(
Status::NotFound,
format!(
"{}: there is no such backup file to restore from",
confined.to_string_lossy()
),
));
}
dot(
context,
"restore",
&format!(".restore \"{}\"", confined.to_string_lossy()),
)
}
pub fn checkpoint(context: &mut Context, _arguments: &Arguments) -> Result<Outcome, Failed> {
listing(context, "checkpoint", "PRAGMA wal_checkpoint")
}
pub fn integrity_check(context: &mut Context, _arguments: &Arguments) -> Result<Outcome, Failed> {
let produced = listing(context, "integrity-check", "PRAGMA integrity_check")?;
if let Some(damage) = first_damage(&produced) {
return Err(Failed::said(Status::Corrupt, damage));
}
Ok(produced)
}
fn first_damage(produced: &Outcome) -> Option<String> {
let said: Vec<String> = produced
.rows
.iter()
.flatten()
.map(|value| match value {
Json::Text(text) => text.clone(),
other => format!("{other:?}"),
})
.collect();
if said.is_empty() {
return Some(
"PRAGMA integrity_check returned no rows at all, so this database's soundness is \
unknown rather than confirmed"
.to_string(),
);
}
if said.iter().all(|line| line.trim() == "ok") {
return None;
}
Some(said.join("; "))
}
pub fn analyze(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let sql = match arguments.text("table") {
Some(name) => format!("ANALYZE {}", quoted(name)),
None => "ANALYZE".to_string(),
};
context
.shell()
.execute(&sql)
.map_err(|message| Failed::said(Status::Syntax, message))?;
Ok(Outcome::said("analyze", "ok. sqlite_stat1 is up to date."))
}
pub fn stats(context: &mut Context, _arguments: &Arguments) -> Result<Outcome, Failed> {
let cache = context.shell().cache_stats();
let pool = context.shell().pool_bytes();
let pages = context
.shell()
.scalar("PRAGMA page_count")
.unwrap_or_default();
let size = context
.shell()
.scalar("PRAGMA page_size")
.unwrap_or_default();
let free = context
.shell()
.scalar("PRAGMA freelist_count")
.unwrap_or_default();
let text = format!(
"pool bytes: {pool}\npage size: {size}\npage count: {pages}\n\
free pages: {free}\ncache hits: {}\ncache misses: {}",
cache.hits, cache.misses
);
Ok(Outcome::said("stats", text)
.with("pool_bytes", Json::Int(pool as i64))
.with("page_size", Json::Int(size.parse::<i64>().unwrap_or(0)))
.with("page_count", Json::Int(pages.parse::<i64>().unwrap_or(0)))
.with("free_pages", Json::Int(free.parse::<i64>().unwrap_or(0)))
.with("cache_hits", Json::Int(cache.hits as i64))
.with("cache_misses", Json::Int(cache.misses as i64)))
}
fn neighbours_asked_for(arguments: &Arguments) -> Result<i64, Failed> {
let k = arguments.integer("k").unwrap_or(10);
if k < 1 {
return Err(Failed::misuse(
"'k' has to be one or more: it is how many rows to return.",
));
}
Ok(k)
}
pub fn search(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let query_text = arguments.required_text("query")?.to_string();
let name = arguments.required_text("table")?.to_string();
let k = neighbours_asked_for(arguments)?;
let sql = format!(
"SELECT rowid, * FROM {0} WHERE {0} MATCH {1} ORDER BY rank LIMIT {k}",
quoted(&name),
quoted_text(&query_text)
);
let mut produced = produce(context, "search", &sql, &[], 0)?;
produced.command = "search".to_string();
Ok(produced.with("query", json::text(&query_text)))
}
pub fn vector_search(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let name = arguments.required_text("table")?.to_string();
let column = arguments.required_text("column")?.to_string();
let numbers = arguments.values("vector");
if numbers.is_empty() {
return Err(Failed::misuse(
"'vector' has to be an array of numbers, one per dimension.",
));
}
let mut blob = String::from("x'");
for value in &numbers {
let Some(number) = value.integer().map(|whole| whole as f64).or(match value {
Json::Real(real) => Some(*real),
_ => None,
}) else {
return Err(Failed::misuse(
"every element of 'vector' has to be a number.",
));
};
for byte in (number as f32).to_bits().to_le_bytes() {
blob.push_str(&format!("{byte:02x}"));
}
}
blob.push('\'');
let k = neighbours_asked_for(arguments)?;
let measure = arguments.text("measure").unwrap_or("cos");
let function = match measure {
"cos" => "vector_distance_cos",
"l2" => "vector_distance_l2",
"dot" => "vector_dot",
other => {
return Err(Failed::misuse(format!(
"'{other}' is not a measure. Use cos, l2 or dot."
)))
}
};
let shape = searched_table_shape(context, &name);
let rowid = if shape.integer_key { "" } else { "rowid, " };
let sql = format!(
"SELECT {rowid}*, {function}({1}, {blob}) AS distance FROM {0} \
WHERE {1} IS NOT NULL ORDER BY {function}({1}, {blob}) LIMIT {k}",
quoted(&name),
quoted(&column)
);
let mut produced = produce(context, "vector-search", &sql, &[], 0)?;
let offset = usize::from(!shape.integer_key);
let positions: Vec<usize> = shape.vectors.iter().map(|nth| nth + offset).collect();
for row in &mut produced.rows {
for &nth in &positions {
if let Some(cell) = row.get_mut(nth) {
if let Some(numbers) = vector_numbers(cell) {
*cell = numbers;
}
}
}
}
if !positions.is_empty() {
let names: Vec<String> = produced.columns.iter().map(|c| c.name.clone()).collect();
produced.columns = columns_from(&names, &produced.rows);
produced.text = table(&produced.columns, &produced.rows, &context.null);
}
Ok(produced)
}
struct SearchedTable {
integer_key: bool,
vectors: Vec<usize>,
}
fn searched_table_shape(context: &mut Context, name: &str) -> SearchedTable {
let info = context
.shell()
.collect(&format!("PRAGMA table_info({})", quoted(name)))
.map(|(_, rows)| rows)
.unwrap_or_default();
let text_of = |value: Option<&Value<'static>>| match value {
Some(Value::Text(text)) => String::from_utf8_lossy(text.raw()).to_ascii_uppercase(),
_ => String::new(),
};
let key_of = |row: &Vec<Value<'static>>| match row.get(5) {
Some(Value::Integer(key)) => *key,
_ => 0,
};
let keys: Vec<&Vec<Value<'static>>> = info.iter().filter(|row| key_of(row) > 0).collect();
let integer_key = matches!(keys.as_slice(), [only] if text_of(only.get(2)) == "INTEGER");
let vectors = info
.iter()
.enumerate()
.filter(|(_, row)| text_of(row.get(2)).starts_with("VECTOR"))
.map(|(nth, _)| nth)
.collect();
SearchedTable {
integer_key,
vectors,
}
}
fn vector_numbers(cell: &Json) -> Option<Json> {
let hex = cell.get("blob").and_then(Json::text)?;
let bytes: Vec<u8> = hex
.as_bytes()
.chunks(2)
.map(|pair| {
std::str::from_utf8(pair)
.ok()
.and_then(|digits| u8::from_str_radix(digits, 16).ok())
})
.collect::<Option<Vec<u8>>>()?;
if !bytes.len().is_multiple_of(4) {
return None;
}
let numbers = bytes
.chunks_exact(4)
.map(|word| {
let mut four = [0u8; 4];
four.copy_from_slice(word);
Json::Real(f64::from(f32::from_le_bytes(four)))
})
.collect();
Some(Json::Array(numbers))
}
pub fn capabilities(_context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let wanted = arguments.text("name");
let rows: Vec<Vec<Json>> = inillucent_driver::CAPABILITIES
.iter()
.filter(|entry| wanted.is_none_or(|name| entry.name == name))
.map(|entry| {
vec![
json::text(entry.name),
json::text(support_name(entry.support)),
json::text(entry.note),
]
})
.collect();
if rows.is_empty() {
return Err(Failed::said(
Status::NotFound,
format!(
"no capability named \"{}\". An unknown name means no, never yes: a capability \
that was never declared was never checked.",
wanted.unwrap_or_default()
),
));
}
let names = vec![
"capability".to_string(),
"support".to_string(),
"note".to_string(),
];
let columns = columns_from(&names, &rows);
let text = wrapped_notes(&rows);
Ok(Outcome {
command: "capabilities".to_string(),
total: rows.len(),
rows,
columns,
more: false,
changes: 0,
last_insert_rowid: 0,
elapsed_ms: 0.0,
text,
extra: Vec::new(),
})
}
fn wrapped_notes(rows: &[Vec<Json>]) -> String {
let mut lines = Vec::with_capacity(rows.len() * 3);
for row in rows {
let name = row.first().and_then(Json::text).unwrap_or_default();
let support = row.get(1).and_then(Json::text).unwrap_or_default();
let note = row.get(2).and_then(Json::text).unwrap_or_default();
lines.push(format!("{name:<22} {support}"));
for line in wrap(note, 74) {
lines.push(format!(" {line}"));
}
}
lines.join(
"
",
)
}
fn wrap(text: &str, width: usize) -> Vec<String> {
let mut lines = Vec::new();
let mut current = String::new();
for word in text.split_whitespace() {
if !current.is_empty() && current.chars().count() + 1 + word.chars().count() > width {
lines.push(std::mem::take(&mut current));
}
if !current.is_empty() {
current.push(' ');
}
current.push_str(word);
}
if !current.is_empty() {
lines.push(current);
}
lines
}
fn support_name(support: Support) -> &'static str {
match support {
Support::Yes => "yes",
Support::Partial => "partial",
Support::No => "no",
}
}
pub fn functions(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let mut sql = String::from("PRAGMA function_list");
let produced = listing(context, "functions", &sql);
let mut produced = produced?;
if let Some(pattern) = arguments.text("pattern") {
produced.rows.retain(|row| {
row.first()
.and_then(Json::text)
.is_some_and(|name| like_matches(pattern, name))
});
produced.total = produced.rows.len();
produced.text = table(&produced.columns, &produced.rows, &context.null);
}
sql.clear();
Ok(produced)
}
fn like_matches(pattern: &str, name: &str) -> bool {
let pattern: Vec<char> = pattern.chars().map(|c| c.to_ascii_lowercase()).collect();
let name: Vec<char> = name.chars().map(|c| c.to_ascii_lowercase()).collect();
let (mut p, mut n) = (0usize, 0usize);
let mut star: Option<(usize, usize)> = None;
while n < name.len() {
match pattern.get(p) {
Some('%') => {
star = Some((p, n));
p += 1;
}
Some(&c) if c == '_' || name.get(n) == Some(&c) => {
p += 1;
n += 1;
}
_ => match star {
Some((star_p, star_n)) => {
p = star_p + 1;
n = star_n + 1;
star = Some((star_p, star_n + 1));
}
None => return false,
},
}
}
pattern
.get(p..)
.is_some_and(|rest| rest.iter().all(|&c| c == '%'))
}
pub fn migrate(context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
let destination = arguments.required_text("destination")?.to_string();
let source = resolve_source(context, arguments.text("source"))?;
let kind = arguments
.text("kind")
.map(str::to_string)
.unwrap_or_else(|| kind_of_source(&source));
if kind == "postgres" || kind == "mysql" {
return migrate_remote(context, &source, &destination, arguments);
}
let from = context.confine(&source)?;
let to = context.confine(&destination)?;
if !from.exists() {
return Err(Failed::said(
Status::NotFound,
format!("there is no \"{source}\" to migrate from."),
));
}
if to.exists() {
return Err(Failed::said(
Status::InvalidState,
format!("\"{destination}\" already exists. This tool never overwrites."),
));
}
match kind.as_str() {
"sqlite" => migrate_sqlite_file(&from, &to),
"index" => Err(Failed::unsupported(
"migrate --kind index",
"the retrieval-index migration runs in inillucent-migrate, which links the retrieval \
engine. Run: inillucent-migrate <source-index-dir> <destination.db>",
)),
other => Err(Failed::misuse(format!(
"'{other}' is not a migration kind. Use sqlite, postgres, mysql or index."
))),
}
}
const SOURCE_URL_VARIABLE: &str = "INILLUCENT_SOURCE_URL";
fn resolve_source(context: &Context, argument: Option<&str>) -> Result<String, Failed> {
if let Some(source) = argument {
if source != "-" {
return Ok(source.to_string());
}
}
if let Ok(held) = std::env::var(SOURCE_URL_VARIABLE) {
if !held.trim().is_empty() {
return Ok(held);
}
}
if argument == Some("-") {
if context.confined() {
return Err(Failed::said(
Status::InvalidState,
"this surface is confined to a directory with --root, and '-' reads the source \
from standard input, which such a surface does not have to itself. Set \
INILLUCENT_SOURCE_URL instead.",
));
}
let mut line = String::new();
std::io::stdin()
.read_line(&mut line)
.map_err(|error| Failed::said(Status::Io, format!("standard input: {error}")))?;
let line = line.trim_end_matches(['\r', '\n']).to_string();
if line.is_empty() {
return Err(Failed::misuse(
"standard input held no source. Write the file path or the connection URL on one \
line.",
));
}
return Ok(line);
}
Err(Failed::misuse(format!(
"migrate needs a source: a database file, or a postgres:// or mysql:// URL. Pass it as \
the first argument, set {SOURCE_URL_VARIABLE}, or pass '-' to read one line from \
standard input."
)))
}
fn kind_of_source(source: &str) -> String {
match inillucent_remote::ConnectionUrl::parse(source) {
Ok(url) => url.scheme.name().to_string(),
Err(_) => "sqlite".to_string(),
}
}
fn migrate_remote(
context: &mut Context,
source: &str,
destination: &str,
arguments: &Arguments,
) -> Result<Outcome, Failed> {
if context.confined() {
return Err(Failed::said(
Status::InvalidState,
"this surface is confined to a directory with --root, and a migration from a server \
reaches a host and a port rather than a path. Run it from an unconfined command \
line.",
));
}
let url = inillucent_remote::ConnectionUrl::parse(source)
.map_err(|error| Failed::misuse(error.detail().unwrap_or_else(|| error.message())))?;
let to = context.confine(destination)?;
if to.exists() {
return Err(Failed::said(
Status::InvalidState,
format!("\"{destination}\" already exists. This tool never overwrites."),
));
}
let mut plan = inillucent_remote::Plan::new(url, &to);
plan.limits = Some(context.limits());
if let Some(batch) = arguments.integer("batch") {
plan.batch = (batch.max(1)) as u64;
}
plan.insecure_plaintext = arguments.flag("insecure-plaintext");
plan.transport().map_err(|error| {
Failed::said(
Status::InvalidState,
error.detail().unwrap_or_else(|| error.message()),
)
})?;
let report = inillucent_remote::migrate::migrate(&plan).map_err(|error| {
Failed::said(
Status::Io,
error.detail().unwrap_or_else(|| error.message()),
)
})?;
let checks: Vec<json::Json> = report
.checks
.iter()
.map(|check| {
json::object(vec![
("name", json::text(&check.name)),
("passed", json::Json::Bool(check.passed)),
("detail", json::text(&check.detail)),
])
})
.collect();
let tables: Vec<json::Json> = report
.tables
.iter()
.map(|table| {
json::object(vec![
("source", json::text(&table.source)),
("destination", json::text(&table.target)),
("rows", json::Json::Int(table.rows as i64)),
("digest", json::text(&table.digest)),
])
})
.collect();
let not_carried: Vec<json::Json> = report
.not_carried
.iter()
.map(|(kind, name)| {
json::object(vec![("kind", json::text(kind)), ("name", json::text(name))])
})
.collect();
let mut text = format!(
"{} -> {}\n{}, {} tables, {} rows\ntransport: {}\n",
report.source,
to.display(),
report.server,
report.tables.len(),
report.rows(),
report.transport
);
for check in &report.checks {
text.push_str(&format!(" {}\n", check.line()));
}
if !report.passed() {
text.push_str(&format!(
"verification failed; nothing was published. The staging file is at {}",
report.staged.display()
));
return Err(Failed::said(Status::Io, text));
}
text.push_str(&format!("published: {}", to.display()));
Ok(Outcome::said("migrate", text)
.with("destination", json::text(to.to_string_lossy()))
.with("transport", json::text(&report.transport))
.with("source", json::text(&report.source))
.with("server", json::text(&report.server))
.with("rows", json::Json::Int(report.rows() as i64))
.with("tables", json::Json::Array(tables))
.with("checks", json::Json::Array(checks))
.with("notCarried", json::Json::Array(not_carried)))
}
fn migrate_sqlite_file(from: &std::path::Path, to: &std::path::Path) -> Result<Outcome, Failed> {
let report = inillucent_migrate::sqlite::migrate(from, to)
.map_err(|error| Failed::from_engine(&error))?;
let failures: Vec<String> = report
.failures()
.iter()
.map(|check| format!("{}: {}", check.name, check.detail))
.collect();
let checks = Json::Array(
report
.checks
.iter()
.map(|check| {
json::object(vec![
("name", json::text(&check.name)),
("passed", Json::Bool(check.passed)),
("detail", json::text(&check.detail)),
])
})
.collect(),
);
if !report.passed() {
return Err(Failed::said(
Status::Corrupt,
format!(
"{} was not published: {}",
to.display(),
failures.join("; ")
),
));
}
Ok(Outcome::said(
"migrate",
format!("imported {} into {}", from.display(), to.display()),
)
.with("destination", json::text(to.to_string_lossy()))
.with("checks", checks))
}
pub fn version(context: &mut Context, _arguments: &Arguments) -> Result<Outcome, Failed> {
let printed = context.collect_output(".version");
context.shell().failed = false;
context.shell().first_error = None;
let text = format!(
"{}
inillucent-cli {}
{}",
printed.trim_end(),
env!("CARGO_PKG_VERSION"),
inillucent_driver::version()
);
Ok(Outcome::said("version", text)
.with("cli", json::text(env!("CARGO_PKG_VERSION")))
.with("driver", json::text(inillucent_driver::version())))
}
pub fn help(_context: &mut Context, arguments: &Arguments) -> Result<Outcome, Failed> {
match arguments.text("topic") {
None => {
let rows: Vec<Vec<Json>> = super::COMMANDS
.iter()
.map(|command| vec![json::text(command.name), json::text(command.summary)])
.collect();
let names = vec!["command".to_string(), "what it does".to_string()];
let columns = columns_from(&names, &rows);
let text = table(&columns, &rows, "");
Ok(Outcome {
command: "help".to_string(),
total: rows.len(),
rows,
columns,
more: false,
changes: 0,
last_insert_rowid: 0,
elapsed_ms: 0.0,
text,
extra: Vec::new(),
})
}
Some(topic) => {
let Some(command) = super::find(topic) else {
return Err(Failed::said(
Status::NotFound,
format!("there is no '{topic}' command. Run 'inillucent help' for the list."),
));
};
let mut text = format!(
"{}\n\n{}\n\n{}",
command.usage(),
command.summary,
command.detail
);
if !command.params.is_empty() {
text.push_str("\n\nParameters:");
for param in command.params {
text.push_str(&format!(
"\n {:<12} {}{}",
param.name,
if param.required { "(required) " } else { "" },
param.description
));
}
}
Ok(Outcome::said("help", text))
}
}
}
fn front_end_only(name: &'static str) -> Failed {
Failed::misuse(format!(
"'{name}' is run by the inillucent binary itself and cannot be dispatched here."
))
}
pub fn shell_placeholder(
_context: &mut Context,
_arguments: &Arguments,
) -> Result<Outcome, Failed> {
Err(front_end_only("shell"))
}
pub fn mcp_placeholder(_context: &mut Context, _arguments: &Arguments) -> Result<Outcome, Failed> {
Err(front_end_only("mcp"))
}
#[cfg(test)]
mod source_tests {
use super::*;
use crate::shell::Shell;
fn env_guard() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn context(root: Option<std::path::PathBuf>) -> Context {
Context::for_test(
Shell::open(":memory:").expect("a memory database opens"),
root,
)
}
#[test]
fn an_argument_is_the_source() {
let context = context(None);
let held = resolve_source(&context, Some("postgres://user@host/db"))
.expect("the argument is accepted");
assert_eq!(held, "postgres://user@host/db");
}
#[test]
fn the_environment_supplies_a_source_that_was_not_an_argument() {
let _held = env_guard();
let context = context(None);
std::env::set_var(SOURCE_URL_VARIABLE, "postgres://user:secret@host/db");
let held = resolve_source(&context, None).expect("the variable is read");
std::env::remove_var(SOURCE_URL_VARIABLE);
assert_eq!(held, "postgres://user:secret@host/db");
}
#[test]
fn no_source_anywhere_is_refused_by_name() {
let _held = env_guard();
let context = context(None);
std::env::remove_var(SOURCE_URL_VARIABLE);
let error = resolve_source(&context, None).expect_err("there is no source");
let said = format!("{error:?}");
assert!(said.contains(SOURCE_URL_VARIABLE), "{said}");
assert!(said.contains("standard input"), "{said}");
}
#[test]
fn a_confined_surface_refuses_to_read_standard_input() {
let _held = env_guard();
let context = context(Some(std::env::temp_dir()));
std::env::remove_var(SOURCE_URL_VARIABLE);
let error = resolve_source(&context, Some("-")).expect_err("a confined surface refuses");
let said = format!("{error:?}");
assert!(said.contains("--root"), "{said}");
assert!(said.contains(SOURCE_URL_VARIABLE), "{said}");
}
#[test]
fn export_refuses_table_and_sql_together() {
let mut arguments = Arguments::default();
arguments.set("table", crate::json::text("expected"));
arguments.set("sql", crate::json::text("SELECT 'other' AS v"));
let failure = export(&mut context(None), &arguments)
.expect_err("export must require one data source");
assert!(failure.message.contains("not both"), "{}", failure.message);
}
#[test]
fn the_environment_wins_over_reading_standard_input() {
let _held = env_guard();
let context = context(None);
std::env::set_var(SOURCE_URL_VARIABLE, "mysql://user@host/db");
let held = resolve_source(&context, Some("-")).expect("the variable is read");
std::env::remove_var(SOURCE_URL_VARIABLE);
assert_eq!(held, "mysql://user@host/db");
}
}