use std::{
io::{BufRead, Write},
path::{Path, PathBuf},
sync::Arc,
};
use axum::{
Router,
extract::State,
http::StatusCode,
response::Json,
routing::{get, post},
};
use clap::{Parser, Subcommand, ValueEnum};
use mq_db::{DocumentStore, MqEngine, SqlEngine, block::BlockType, sql::html_escape};
use serde::Deserialize;
#[derive(Parser)]
#[command(
name = "mq-db",
about = "Markdown-specialised embedded database",
version
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
Index {
#[arg(required = true)]
paths: Vec<PathBuf>,
#[arg(short, long, default_value = "store.mq-db")]
output: PathBuf,
#[arg(short, long)]
recursive: bool,
#[arg(long)]
no_spans: bool,
#[arg(long)]
prune: bool,
},
List {
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
#[arg(long, short = 'F', default_value = "table")]
format: OutputFormat,
},
Mq {
code: String,
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
#[arg(long, short = 'F', default_value = "table")]
format: OutputFormat,
},
Sql {
query: Option<String>,
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
#[arg(short, long)]
file: Option<PathBuf>,
#[arg(long, short = 'F', default_value = "table")]
format: OutputFormat,
#[arg(long)]
write_back: bool,
},
Repl {
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
#[arg(short, long, default_value = "sql")]
mode: ReplMode,
#[arg(long)]
write_back: bool,
},
Lint {
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
#[arg(long, default_value_t = 2)]
depth: u8,
},
Stats {
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
},
Show {
doc_id: u32,
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
},
Tui {
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
},
Serve {
#[arg(short, long, default_value = "store.mq-db")]
db: PathBuf,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(short, long, default_value_t = 7878)]
port: u16,
},
}
#[derive(Clone, ValueEnum, Debug, Default)]
enum OutputFormat {
#[default]
Table,
Json,
Csv,
Tsv,
Markdown,
Html,
}
#[derive(Clone, ValueEnum, Debug)]
enum ReplMode {
Mq,
Sql,
}
impl std::fmt::Display for ReplMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ReplMode::Mq => write!(f, "mq"),
ReplMode::Sql => write!(f, "sql"),
}
}
}
fn load_store(db: &Path) -> anyhow::Result<DocumentStore> {
if !db.exists() {
anyhow::bail!(
"Store file not found: {}\nRun `mq-db index <files...>` to create it.",
db.display()
);
}
DocumentStore::load(db).map_err(|e| anyhow::anyhow!("Failed to load store: {}", e))
}
fn open_store_for_sql(db: &Path) -> anyhow::Result<DocumentStore> {
if !db.exists() {
anyhow::bail!(
"Store file not found: {}\nRun `mq-db index <files...>` to create it.",
db.display()
);
}
let mut store =
DocumentStore::open(db).map_err(|e| anyhow::anyhow!("Failed to open store: {}", e))?;
store
.load_all_blocks()
.map_err(|e| anyhow::anyhow!("Failed to load blocks: {}", e))?;
store
.load_all_indexes()
.map_err(|e| anyhow::anyhow!("Failed to load indexes: {}", e))?;
Ok(store)
}
fn load_catalog_store(db: &Path) -> anyhow::Result<DocumentStore> {
if !db.exists() {
anyhow::bail!(
"Store file not found: {}\nRun `mq-db index <files...>` to create it.",
db.display()
);
}
DocumentStore::load_catalog_only(db).map_err(|e| anyhow::anyhow!("Failed to load store: {}", e))
}
fn bar(count: usize, max: usize, width: usize) -> String {
if max == 0 {
return " ".repeat(width);
}
let filled = (count * width / max).min(width);
let empty = width - filled;
format!("{}{}", "█".repeat(filled), "░".repeat(empty))
}
fn block_type_icon(bt: &BlockType) -> &'static str {
match bt {
BlockType::Heading => "#",
BlockType::Paragraph => "¶",
BlockType::Code => "{}",
BlockType::List => "•",
BlockType::TableCell | BlockType::TableRow | BlockType::TableAlign => "▦",
BlockType::Blockquote => "❝",
BlockType::HorizontalRule => "─",
BlockType::Html => "<>",
BlockType::Yaml | BlockType::Toml => "≡",
BlockType::Math => "∑",
BlockType::Definition => "§",
BlockType::Footnote => "†",
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Index {
paths,
output,
recursive,
no_spans,
prune,
} => {
let files = mq_db::discover::collect_markdown_files(&paths, recursive);
if files.is_empty() {
anyhow::bail!("No Markdown files found in the specified paths.");
}
let is_new_store = !output.exists();
let mut store = if is_new_store {
let mut store = DocumentStore::new();
if no_spans {
store.set_store_spans(false);
}
store
} else {
DocumentStore::open(&output)
.map_err(|e| anyhow::anyhow!("Failed to open store: {}", e))?
};
let report = store
.reindex_paths(&files, prune)
.map_err(|e| anyhow::anyhow!("Failed to reindex: {}", e))?;
if is_new_store {
store
.save(&output)
.map_err(|e| anyhow::anyhow!("Failed to save store: {}", e))?;
}
for path in &report.added {
eprintln!(" + {}", path.display());
}
for path in &report.updated {
eprintln!(" ~ {}", path.display());
}
for path in &report.removed {
eprintln!(" - {}", path.display());
}
for (path, err) in &report.failed {
eprintln!(" ✗ {}: {}", path.display(), err);
}
println!(
"\n{} added, {} updated, {} unchanged, {} removed{} → {}",
report.added.len(),
report.updated.len(),
report.unchanged,
report.removed.len(),
if report.failed.is_empty() {
String::new()
} else {
format!(", {} failed", report.failed.len())
},
output.display()
);
}
Commands::List { db, format } => {
let store = load_catalog_store(&db)?;
if store.is_empty() {
println!("(no documents indexed)");
return Ok(());
}
match format {
OutputFormat::Json
| OutputFormat::Csv
| OutputFormat::Tsv
| OutputFormat::Markdown
| OutputFormat::Html => {
let engine = SqlEngine::new(&store).map_err(|e| anyhow::anyhow!("{}", e))?;
let out = engine
.execute("SELECT id, path, title, tags FROM documents")
.map_err(|e| anyhow::anyhow!("{}", e))?;
match format {
OutputFormat::Json => print!("{}", out.to_json()),
OutputFormat::Csv => print!("{}", out.to_csv()),
OutputFormat::Tsv => print!("{}", out.to_tsv()),
OutputFormat::Markdown => print!("{}", out.to_markdown_table()),
OutputFormat::Html => print!("{}", out.to_html_table()),
OutputFormat::Table => unreachable!(),
}
}
OutputFormat::Table => {
let path_width = store
.documents()
.iter()
.map(|d| {
d.path
.as_ref()
.map(|p| p.to_string_lossy().len())
.unwrap_or(10)
.min(52)
})
.max()
.unwrap_or(10)
.max(12); let tag_width = store
.documents()
.iter()
.map(|d| d.zone_maps.tags.join(", ").len())
.max()
.unwrap_or(0)
.max(4);
let sep_id = "──────";
let sep_path = "─".repeat(path_width + 2);
let sep_blocks = "────────";
let sep_tags = "─".repeat(tag_width.max(4) + 2);
println!("┌{}┬{}┬{}┬{}┐", sep_id, sep_path, sep_blocks, sep_tags);
println!(
"│ {:<4} │ {:<path_width$} │ {:>6} │ {:<tag_w$} │",
"ID",
"Path / Title",
"Blocks",
"Tags",
path_width = path_width,
tag_w = tag_width.max(4),
);
println!("├{}┼{}┼{}┼{}┤", sep_id, sep_path, sep_blocks, sep_tags);
for doc in store.documents() {
let path_str = doc
.path
.as_ref()
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_else(|| {
doc.zone_maps
.title
.clone()
.unwrap_or_else(|| format!("<doc {}>", doc.id))
});
let path_display = if path_str.len() > path_width {
format!("…{}", &path_str[path_str.len() - path_width + 1..])
} else {
path_str.clone()
};
let tags = doc.zone_maps.tags.join(", ");
println!(
"│ {:>4} │ {:<path_width$} │ {:>6} │ {:<tag_w$} │",
doc.id,
path_display,
doc.block_count,
tags,
path_width = path_width,
tag_w = tag_width.max(4),
);
}
println!("└{}┴{}┴{}┴{}┘", sep_id, sep_path, sep_blocks, sep_tags);
println!(
"{} document{}",
store.len(),
if store.len() == 1 { "" } else { "s" }
);
}
}
}
Commands::Mq { code, db, format } => {
let store = load_store(&db)?;
let results =
MqEngine::eval_store(&code, &store).map_err(|e| anyhow::anyhow!("{}", e))?;
if results.is_empty() {
println!("(no results)");
} else {
match format {
OutputFormat::Json => {
let items: Vec<String> = results
.iter()
.map(|s| {
format!(
"\"{}\"",
s.replace('\\', "\\\\")
.replace('"', "\\\"")
.replace('\n', "\\n")
)
})
.collect();
println!("[{}]", items.join(","));
}
OutputFormat::Csv => {
println!("content");
for line in &results {
let cell = if line.contains(',')
|| line.contains('"')
|| line.contains('\n')
{
format!("\"{}\"", line.replace('"', "\"\""))
} else {
line.clone()
};
println!("{}", cell);
}
}
OutputFormat::Tsv => {
println!("content");
for line in &results {
println!("{}", line);
}
}
OutputFormat::Markdown => {
println!("{}", mq_to_markdown(&results));
}
OutputFormat::Html => {
print!("{}", mq_to_html(&results));
}
OutputFormat::Table => {
for line in &results {
println!("{}", line);
}
}
}
}
}
Commands::Sql {
query,
db,
file,
format,
write_back,
} => {
let sql = if let Some(f) = file {
std::fs::read_to_string(&f)
.map_err(|e| anyhow::anyhow!("Cannot read file {}: {}", f.display(), e))?
} else if let Some(q) = query {
q
} else {
anyhow::bail!("Provide a query argument or --file <path>");
};
if !write_back && is_write_statement(&sql) {
anyhow::bail!(
"UPDATE/DELETE would write back to the source Markdown file; pass --write-back to allow this."
);
}
let mut store = open_store_for_sql(&db)?;
let out = if write_back {
store.execute_sql_mut(&sql)
} else {
SqlEngine::new(&store).and_then(|e| e.execute(&sql))
}
.map_err(|e| anyhow::anyhow!("{}", e))?;
match format {
OutputFormat::Table => print!("{}", out.to_table()),
OutputFormat::Json => print!("{}", out.to_json()),
OutputFormat::Csv => print!("{}", out.to_csv()),
OutputFormat::Tsv => print!("{}", out.to_tsv()),
OutputFormat::Markdown => print!("{}", out.to_markdown_table()),
OutputFormat::Html => print!("{}", out.to_html_table()),
}
}
Commands::Repl {
db,
mode,
write_back,
} => {
let store = open_store_for_sql(&db)?;
run_repl(store, mode, write_back)?;
}
Commands::Lint { db, depth } => {
let store = load_store(&db)?;
let q = store.query();
let violations = q.lint_heading_followed_by(depth, &[BlockType::List]);
if violations.is_empty() {
println!(
"✓ No violations (H{} must not be immediately followed by a list)",
depth
);
} else {
let n = violations.len();
println!(
"✗ {} violation{} (H{} immediately followed by list)\n",
n,
if n == 1 { "" } else { "s" },
depth
);
println!(" {:<40} heading", "file");
println!(" {} {}", "─".repeat(40), "─".repeat(30));
for v in &violations {
let path = v
.document
.path
.as_ref()
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_else(|| format!("<doc {}>", v.document.id));
let path_display = if path.len() > 40 {
format!("…{}", &path[path.len() - 39..])
} else {
path
};
println!(" {:<40} \"{}\"", path_display, v.heading.content);
}
}
}
Commands::Stats { db } => {
let store = load_store(&db)?;
let stats = store.stats();
println!(" Documents {}", stats.documents);
println!(" Blocks {}", stats.blocks);
let max_type = stats
.block_type_counts
.first()
.map(|(_, v)| *v)
.unwrap_or(1);
println!("\n Block types");
println!(" {}", "─".repeat(56));
for (bt, count) in &stats.block_type_counts {
let pct = count * 100 / stats.blocks.max(1);
let b = bar(*count, max_type, 20);
let icon = block_type_icon(bt);
println!(
" {:>2} {:<12} {} {:>5} ({:>2}%)",
icon,
bt.as_str(),
b,
count,
pct,
);
}
if !stats.code_lang_counts.is_empty() {
let max_lang = stats.code_lang_counts.first().map(|(_, v)| *v).unwrap_or(1);
let total_code: usize = stats.code_lang_counts.iter().map(|(_, v)| v).sum();
println!("\n Code languages");
println!(" {}", "─".repeat(56));
for (lang, count) in &stats.code_lang_counts {
let pct = count * 100 / total_code.max(1);
let b = bar(*count, max_lang, 20);
println!(" {{}} {:<12} {} {:>5} ({:>2}%)", lang, b, count, pct);
}
}
}
Commands::Show { doc_id, db } => {
let store = load_store(&db)?;
let doc = store
.get_document(doc_id)
.ok_or_else(|| anyhow::anyhow!("Document {} not found", doc_id))?;
let path = doc
.path
.as_ref()
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_else(|| format!("<doc {}>", doc.id));
println!(" {}", path);
if let Some(title) = &doc.zone_maps.title {
println!(" title {}", title);
}
println!(" blocks {}", doc.blocks.len());
if !doc.zone_maps.tags.is_empty() {
println!(" tags {}", doc.zone_maps.tags.join(", "));
}
println!();
let pre_w = doc
.blocks
.iter()
.map(|b| digits(b.pre))
.max()
.unwrap_or(3)
.max(3);
let post_w = doc
.blocks
.iter()
.map(|b| digits(b.post))
.max()
.unwrap_or(4)
.max(4);
println!(
" {:<pre_w$} {:<post_w$} {:<16} content",
"pre",
"post",
"type",
pre_w = pre_w,
post_w = post_w,
);
println!(
" {} {} {} {}",
"─".repeat(pre_w),
"─".repeat(post_w),
"─".repeat(16),
"─".repeat(40),
);
for block in &doc.blocks {
let depth = block.heading_depth().unwrap_or(0) as usize;
let indent = if depth > 1 {
" ".repeat(depth - 1).to_string()
} else {
String::new()
};
let type_label = match block.block_type {
BlockType::Heading => {
format!("heading H{}", block.heading_depth().unwrap_or(0))
}
ref bt => bt.as_str().to_string(),
};
let preview: String = block.content.chars().take(48).collect();
let preview = if block.content.chars().count() > 48 {
format!("{}…", preview)
} else {
preview
};
let preview = preview.replace('\n', " ");
println!(
" {:<pre_w$} {:<post_w$} {:<16} {}{}",
block.pre,
block.post,
type_label,
indent,
preview,
pre_w = pre_w,
post_w = post_w,
);
}
}
Commands::Tui { db } => {
let store = if db.exists() {
DocumentStore::load(&db).map_err(|e| anyhow::anyhow!("{}", e))?
} else {
eprintln!(
"No store found at {}. Starting with empty store.",
db.display()
);
DocumentStore::new()
};
mq_db::tui::run(store).map_err(|e| anyhow::anyhow!("{}", e))?;
}
Commands::Serve { db, host, port } => {
let store = Arc::new(load_store(&db)?);
let addr = format!("{}:{}", host, port);
let app = Router::new()
.route("/sql", post(serve_sql))
.route("/mq", post(serve_mq))
.route("/health", get(serve_health))
.with_state(store);
let listener = tokio::net::TcpListener::bind(&addr)
.await
.map_err(|e| anyhow::anyhow!("Cannot bind {}: {}", addr, e))?;
println!("mq-db listening on http://{}", addr);
axum::serve(listener, app)
.await
.map_err(|e| anyhow::anyhow!("{}", e))?;
}
}
Ok(())
}
type SharedStore = Arc<DocumentStore>;
#[derive(Deserialize)]
struct SqlRequest {
query: String,
}
#[derive(Deserialize)]
struct MqRequest {
code: String,
}
type ApiResult = Result<Json<serde_json::Value>, (StatusCode, Json<serde_json::Value>)>;
async fn serve_sql(State(store): State<SharedStore>, Json(req): Json<SqlRequest>) -> ApiResult {
let engine = SqlEngine::new(&store).map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, e))?;
let out = engine
.execute(&req.query)
.map_err(|e| err(StatusCode::BAD_REQUEST, e))?;
let v: serde_json::Value =
serde_json::from_str(&out.to_json()).unwrap_or(serde_json::json!([]));
Ok(Json(v))
}
async fn serve_mq(State(store): State<SharedStore>, Json(req): Json<MqRequest>) -> ApiResult {
let results =
MqEngine::eval_store(&req.code, &store).map_err(|e| err(StatusCode::BAD_REQUEST, e))?;
Ok(Json(serde_json::json!({ "results": results })))
}
async fn serve_health(State(store): State<SharedStore>) -> Json<serde_json::Value> {
Json(serde_json::json!({ "status": "ok", "documents": store.len() }))
}
fn err(status: StatusCode, e: impl std::fmt::Display) -> (StatusCode, Json<serde_json::Value>) {
(status, Json(serde_json::json!({ "error": e.to_string() })))
}
fn digits(n: u32) -> usize {
if n == 0 { 1 } else { n.ilog10() as usize + 1 }
}
fn mq_to_markdown(results: &[String]) -> String {
let mut out = String::new();
for (i, block) in results.iter().enumerate() {
if i > 0 {
let prev = &results[i - 1];
let prev_is_list = prev.trim_start().starts_with("- ")
|| prev.trim_start().starts_with("* ")
|| prev
.trim_start()
.chars()
.next()
.is_some_and(|c| c.is_ascii_digit());
let curr_is_list = block.trim_start().starts_with("- ")
|| block.trim_start().starts_with("* ")
|| block
.trim_start()
.chars()
.next()
.is_some_and(|c| c.is_ascii_digit());
if prev_is_list && curr_is_list {
out.push('\n');
} else {
out.push_str("\n\n");
}
}
out.push_str(block);
}
out
}
fn mq_to_html(results: &[String]) -> String {
let mut out = String::new();
for block in results {
out.push_str(&md_block_to_html(block));
out.push('\n');
}
out
}
fn md_block_to_html(s: &str) -> String {
let trimmed = s.trim();
for depth in (1u8..=6).rev() {
let prefix = "#".repeat(depth as usize);
if let Some(rest) = trimmed.strip_prefix(&prefix)
&& (rest.starts_with(' ') || rest.is_empty())
{
let text = html_escape(rest.trim());
return format!("<h{depth}>{text}</h{depth}>");
}
}
if trimmed.starts_with("```") {
let first_line = trimmed.lines().next().unwrap_or("");
let lang = first_line.trim_start_matches('`').trim();
let code: String = trimmed
.lines()
.skip(1)
.take_while(|l| !l.trim_start().starts_with("```"))
.collect::<Vec<_>>()
.join("\n");
let escaped = html_escape(&code);
return if lang.is_empty() {
format!("<pre><code>{escaped}</code></pre>")
} else {
format!("<pre><code class=\"language-{lang}\">{escaped}</code></pre>")
};
}
if trimmed.starts_with("> ") {
let inner = trimmed
.lines()
.map(|l| l.strip_prefix("> ").unwrap_or(l))
.collect::<Vec<_>>()
.join("\n");
return format!("<blockquote><p>{}</p></blockquote>", html_escape(&inner));
}
if matches!(trimmed, "---" | "***" | "___") {
return "<hr>".to_string();
}
if trimmed.lines().all(|l| {
let l = l.trim();
l.is_empty() || l.starts_with("- ") || l.starts_with("* ")
}) && trimmed
.lines()
.any(|l| l.trim().starts_with("- ") || l.trim().starts_with("* "))
{
let items: String = trimmed
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| {
let text = l.trim().trim_start_matches("- ").trim_start_matches("* ");
format!("<li>{}</li>", html_escape(text))
})
.collect::<Vec<_>>()
.join("\n");
return format!("<ul>\n{items}\n</ul>");
}
if trimmed.lines().all(|l| {
let l = l.trim();
l.is_empty() || l.chars().next().is_some_and(|c| c.is_ascii_digit())
}) && trimmed
.lines()
.any(|l| l.trim().chars().next().is_some_and(|c| c.is_ascii_digit()))
{
let items: String = trimmed
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| {
let text = l.trim().split_once(". ").map(|x| x.1).unwrap_or(l.trim());
format!("<li>{}</li>", html_escape(text))
})
.collect::<Vec<_>>()
.join("\n");
return format!("<ol>\n{items}\n</ol>");
}
format!("<p>{}</p>", html_escape(trimmed))
}
fn is_write_statement(sql: &str) -> bool {
let trimmed = sql.trim().trim_end_matches(';');
let upper = trimmed.to_ascii_uppercase();
upper.starts_with("UPDATE ") || upper.starts_with("DELETE ")
}
fn run_repl(
mut store: DocumentStore,
initial_mode: ReplMode,
write_back: bool,
) -> anyhow::Result<()> {
let stdin = std::io::stdin();
let mut mode = initial_mode;
println!("mq-db (.help for commands .quit to exit)");
println!(
"mode: {} (.mode mq | .mode sql){}\n",
mode,
if write_back { " [write-back: on]" } else { "" }
);
loop {
print!("{}> ", mode);
std::io::stdout().flush()?;
let mut line = String::new();
match stdin.lock().read_line(&mut line) {
Ok(0) => break,
Ok(_) => {}
Err(e) => anyhow::bail!("Read error: {}", e),
}
let input = line.trim();
if input.is_empty() {
continue;
}
match input {
".quit" | ".exit" | "\\q" => break,
".help" => print_repl_help(),
".mode mq" => {
mode = ReplMode::Mq;
println!("→ mq mode");
}
".mode sql" => {
mode = ReplMode::Sql;
println!("→ sql mode");
}
_ => match mode {
ReplMode::Sql if !write_back && is_write_statement(input) => {
eprintln!(
"error: UPDATE/DELETE would write back to the source Markdown file; restart with --write-back to allow this."
);
}
ReplMode::Sql if write_back => match store.execute_sql_mut(input) {
Ok(out) => print!("{}", out.to_table()),
Err(e) => eprintln!("error: {}", e),
},
ReplMode::Sql => match SqlEngine::new(&store).and_then(|e| e.execute(input)) {
Ok(out) => print!("{}", out.to_table()),
Err(e) => eprintln!("error: {}", e),
},
ReplMode::Mq => match MqEngine::eval_store(input, &store) {
Ok(results) => {
if results.is_empty() {
println!("(no results)");
} else {
for r in results {
println!("{}", r);
}
}
}
Err(e) => eprintln!("error: {}", e),
},
},
}
}
println!("bye");
Ok(())
}
fn print_repl_help() {
println!(
r#"
.mode sql switch to SQL mode
.mode mq switch to mq mode
.quit exit
SQL examples
SELECT block_type, count(*) FROM blocks GROUP BY block_type;
SELECT content FROM blocks WHERE block_type = 'heading' ORDER BY pre;
SELECT b.content FROM blocks b
WHERE under(b.pre, b.post,
(SELECT pre FROM blocks WHERE content = 'Architecture'),
(SELECT post FROM blocks WHERE content = 'Architecture'));
mq examples
.h1
.code
select(.block_type == "heading")
"#
);
}