use std::collections::HashMap;
use std::io::{self, BufRead, BufReader, Write};
use std::ops::Deref;
use serde::{Deserialize, Serialize, ser::SerializeSeq};
use serde_json::Value;
use syntaqlite::Diagnostic;
use syntaqlite::analysis::{
CatalogLayer, DiagnosticMessage, Help, ModuleResolver, RelationKind, Severity as SemSeverity,
StatementAnalysis,
};
use syntaqlite::any::{AnyDialect, AnyParser, AnyTokenizer, ParseOutcome};
use syntaqlite::fmt::KeywordCase;
use syntaqlite::util::DiagnosticRenderer;
use syntaqlite::{AnalysisConfig, AnalysisContext, Analyzer, Catalog, FormatConfig, Formatter};
pub(crate) fn run(dialect: &AnyDialect) -> Result<(), String> {
let stdin = io::stdin();
let stdout = io::stdout();
let mut reader = BufReader::new(stdin.lock());
let mut writer = io::BufWriter::new(stdout.lock());
writeln!(writer, "READY").map_err(|e| format!("write READY: {e}"))?;
writer.flush().map_err(|e| format!("flush READY: {e}"))?;
let parser = AnyParser::new(dialect.deref().clone());
let tokenizer = AnyTokenizer::new(dialect.deref().clone());
let mut analyzer = Analyzer::with_dialect(dialect.clone());
let mut formatter_cache: Option<(FormatConfig, Formatter)> = None;
let mut line = String::new();
loop {
line.clear();
let n = reader
.read_line(&mut line)
.map_err(|e| format!("read: {e}"))?;
if n == 0 {
return Ok(()); }
let trimmed = line.trim_end_matches(['\n', '\r']);
if trimmed.is_empty() {
continue;
}
let req: Value = match serde_json::from_str(trimmed) {
Ok(v) => v,
Err(e) => {
write_err(&mut writer, &format!("invalid JSON: {e}"))?;
continue;
}
};
let Some(op) = req.get("op").and_then(Value::as_str) else {
write_err(&mut writer, "missing \"op\" field")?;
continue;
};
if op == "quit" {
return Ok(());
}
let outcome = match op {
"parse" => handle_parse(&parser, &req).and_then(|r| write_ok(&mut writer, &r)),
"format" => handle_format(dialect, &mut formatter_cache, &req)
.and_then(|r| write_ok(&mut writer, &r)),
"tokenize" => {
handle_tokenize(dialect, &tokenizer, &req).and_then(|r| write_ok(&mut writer, &r))
}
"analyze" => handle_analyze(dialect, &mut analyzer, &req, &mut writer),
other => Err(format!("unknown op: {other}")),
};
if let Err(msg) = outcome {
write_err(&mut writer, &msg)?;
}
}
}
#[derive(Serialize)]
struct OkFrame<'a, T: Serialize> {
ok: bool,
result: &'a T,
}
#[derive(Serialize)]
struct ErrFrame<'a> {
ok: bool,
error: &'a str,
}
fn write_ok<W: Write, T: Serialize>(w: &mut W, result: &T) -> Result<(), String> {
let frame = OkFrame { ok: true, result };
serde_json::to_writer(&mut *w, &frame).map_err(|e| format!("write: {e}"))?;
w.write_all(b"\n").map_err(|e| format!("write: {e}"))?;
w.flush().map_err(|e| format!("flush: {e}"))
}
fn write_err<W: Write>(w: &mut W, msg: &str) -> Result<(), String> {
let frame = ErrFrame {
ok: false,
error: msg,
};
serde_json::to_writer(&mut *w, &frame).map_err(|e| format!("write: {e}"))?;
w.write_all(b"\n").map_err(|e| format!("write: {e}"))?;
w.flush().map_err(|e| format!("flush: {e}"))
}
#[derive(Deserialize)]
struct ParseReq {
sql: String,
}
#[derive(Serialize)]
struct ParseResp {
statements: Vec<Value>,
errors: Vec<ParseErrView>,
}
#[derive(Serialize)]
struct ParseErrView {
severity: &'static str,
message: String,
start_offset: usize,
end_offset: usize,
code: u32,
}
fn handle_parse(parser: &AnyParser, req: &Value) -> Result<ParseResp, String> {
let req: ParseReq = serde_json::from_value(req.clone()).map_err(|e| format!("parse: {e}"))?;
let mut session = parser.parse(&req.sql);
let mut statements: Vec<Value> = Vec::new();
let mut errors: Vec<ParseErrView> = Vec::new();
loop {
match session.next() {
ParseOutcome::Ok(stmt) => {
let erased = stmt.erase();
let node = erased
.root_node()
.and_then(|n| serde_json::to_value(n).ok())
.unwrap_or(Value::Null);
statements.push(node);
}
ParseOutcome::Err(err) => {
let base = err.statement_base();
let stmt_offset = err.offset().unwrap_or_default();
let length = err.length().unwrap_or_default();
let start = stmt_offset.to_doc(base).as_usize();
errors.push(ParseErrView {
severity: "error",
message: err.message().to_string(),
start_offset: start,
end_offset: start + length.as_usize(),
code: DIAG_CODE_PARSE_ERROR,
});
}
ParseOutcome::Done => break,
}
}
Ok(ParseResp { statements, errors })
}
#[derive(Deserialize)]
struct FormatReq {
sql: String,
line_width: Option<usize>,
indent_width: Option<usize>,
keyword_case: Option<String>,
semicolons: Option<bool>,
}
#[derive(Serialize)]
struct FormatResp {
formatted: String,
}
fn handle_format(
dialect: &AnyDialect,
cache: &mut Option<(FormatConfig, Formatter)>,
req: &Value,
) -> Result<FormatResp, String> {
let req: FormatReq = serde_json::from_value(req.clone()).map_err(|e| format!("format: {e}"))?;
let mut config = FormatConfig::default();
if let Some(w) = req.line_width {
config = config.with_line_width(w);
}
if let Some(w) = req.indent_width {
config = config.with_indent_width(w);
}
if let Some(case) = req.keyword_case.as_deref() {
let kc = match case {
"upper" => KeywordCase::Upper,
"lower" => KeywordCase::Lower,
other => {
return Err(format!(
"keyword_case must be 'upper' or 'lower', got {other:?}"
));
}
};
config = config.with_keyword_case(kc);
}
if let Some(s) = req.semicolons {
config = config.with_semicolons(s);
}
let formatter = match cache {
Some((cfg, f)) if format_config_eq(cfg, &config) => f,
slot => {
*slot = Some((
config.clone(),
Formatter::with_dialect_config(dialect.clone(), &config),
));
&mut slot.as_mut().expect("just inserted").1
}
};
formatter
.format(&req.sql)
.map(|formatted| FormatResp { formatted })
.map_err(|e| format!("{e}"))
}
fn format_config_eq(a: &FormatConfig, b: &FormatConfig) -> bool {
a.line_width() == b.line_width()
&& a.indent_width() == b.indent_width()
&& a.keyword_case() == b.keyword_case()
&& a.semicolons() == b.semicolons()
}
#[derive(Serialize)]
struct TokenizeResp<'a> {
tokens: TokensView<'a>,
}
struct TokensView<'a> {
dialect: &'a AnyDialect,
tokenizer: &'a AnyTokenizer,
sql: &'a str,
}
impl Serialize for TokensView<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let base = self.sql.as_ptr() as usize;
let mut seq = serializer.serialize_seq(None)?;
for tok in self.tokenizer.tokenize(self.sql) {
let text = tok.text();
let offset = (text.as_ptr() as usize).saturating_sub(base);
let length = text.len();
let tt = tok.token_type();
seq.serialize_element(&TokenView {
text,
offset,
length,
token_type: u32::from(tt),
category: token_category_name(self.dialect.token_category(tt)),
})?;
}
seq.end()
}
}
#[derive(Serialize)]
struct TokenView<'a> {
text: &'a str,
offset: usize,
length: usize,
#[serde(rename = "type")]
token_type: u32,
category: &'static str,
}
fn handle_tokenize<'a>(
dialect: &'a AnyDialect,
tokenizer: &'a AnyTokenizer,
req: &'a Value,
) -> Result<TokenizeResp<'a>, String> {
let sql = req
.get("sql")
.and_then(Value::as_str)
.ok_or_else(|| "tokenize: missing 'sql'".to_string())?;
Ok(TokenizeResp {
tokens: TokensView {
dialect,
tokenizer,
sql,
},
})
}
fn token_category_name(c: syntaqlite_syntax::any::TokenCategory) -> &'static str {
use syntaqlite_syntax::any::TokenCategory as T;
match c {
T::Keyword => "keyword",
T::Identifier => "identifier",
T::String => "string",
T::Number => "number",
T::Operator => "operator",
T::Punctuation => "punctuation",
T::Comment => "comment",
T::Parameter => "parameter",
T::Function => "function",
T::Type => "type",
T::Other => "other",
}
}
#[derive(Deserialize)]
struct TableRec {
name: String,
columns: Option<Vec<String>>,
}
#[derive(Deserialize)]
struct ViewRec {
name: String,
columns: Option<Vec<String>>,
}
#[derive(Deserialize)]
struct ValidateReq {
sql: String,
tables: Option<Vec<TableRec>>,
views: Option<Vec<ViewRec>>,
schema_ddl: Option<String>,
modules: Option<HashMap<String, String>>,
output: Option<String>,
render_options: Option<RenderOptions>,
}
#[derive(Deserialize, Default)]
struct RenderOptions {
#[serde(default)]
source_name: String,
}
fn build_module_resolver(req: &ValidateReq) -> Option<Box<dyn ModuleResolver>> {
let modules = req.modules.as_ref()?;
if modules.is_empty() {
return None;
}
Some(Box::new(HashMapResolver(modules.clone())))
}
struct HashMapResolver(HashMap<String, String>);
impl ModuleResolver for HashMapResolver {
fn resolve(&self, module_path: &str) -> Option<String> {
self.0.get(module_path).cloned()
}
}
#[derive(Serialize)]
struct RenderedResp {
rendered: String,
}
#[derive(Serialize)]
struct ValidateResp<'a> {
diagnostics: AllDiagnostics<'a>,
statements: StatementsView<'a>,
lineage: Option<LineageView<'a>>,
}
fn handle_analyze<W: Write>(
dialect: &AnyDialect,
analyzer: &mut Analyzer,
req: &Value,
writer: &mut W,
) -> Result<(), String> {
let req: ValidateReq =
serde_json::from_value(req.clone()).map_err(|e| format!("validate: {e}"))?;
let mut catalog = if let Some(ddl) = req.schema_ddl.as_deref() {
let (cat, errs) = Catalog::from_ddl(dialect.clone(), &[ddl]);
if !errs.is_empty() {
return Err(format!("schema_ddl: {}", errs.join("; ")));
}
cat
} else {
Catalog::new(dialect.clone())
};
if let Some(tables) = req.tables.as_deref() {
let layer = catalog.layer_mut(CatalogLayer::Database);
for t in tables {
layer.insert_table(t.name.clone(), t.columns.clone(), false);
}
}
if let Some(views) = req.views.as_deref() {
let layer = catalog.layer_mut(CatalogLayer::Database);
for v in views {
layer.insert_view(v.name.clone(), v.columns.clone());
}
}
let config = AnalysisConfig::default();
let resolver = build_module_resolver(&req);
let mut ctx = AnalysisContext::new(&mut catalog).with_config(config);
if let Some(r) = resolver.as_deref() {
ctx = ctx.with_resolver(r);
}
let model = analyzer.analyze(&req.sql, &mut ctx);
match req.output.as_deref().unwrap_or("structured") {
"structured" => {} "text" => {
let opts = req.render_options.unwrap_or_default();
let r = DiagnosticRenderer::new(&req.sql, &opts.source_name);
let mut buf: Vec<u8> = Vec::new();
for d in model.diagnostics() {
r.render_diagnostic(d, &mut buf)
.map_err(|e| format!("render: {e}"))?;
}
let text = String::from_utf8(buf).map_err(|e| format!("utf-8: {e}"))?;
return write_ok(writer, &RenderedResp { rendered: text });
}
other => return Err(format!("unknown output format: {other:?}")),
}
let top_lineage = model.statements().iter().rev().find_map(lineage_view);
let resp = ValidateResp {
diagnostics: AllDiagnostics { model: &model },
statements: StatementsView { model: &model },
lineage: top_lineage,
};
write_ok(writer, &resp)
}
struct AllDiagnostics<'a> {
model: &'a syntaqlite::analysis::Analysis,
}
impl Serialize for AllDiagnostics<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut seq = serializer.serialize_seq(Some(self.model.diagnostic_count()))?;
for d in self.model.diagnostics() {
seq.serialize_element(&DiagView::from(d))?;
}
seq.end()
}
}
struct StatementsView<'a> {
model: &'a syntaqlite::analysis::Analysis,
}
impl Serialize for StatementsView<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let stmts = self.model.statements();
let mut seq = serializer.serialize_seq(Some(stmts.len()))?;
for s in stmts {
seq.serialize_element(&StatementView { inner: s })?;
}
seq.end()
}
}
struct StatementView<'a> {
inner: &'a StatementAnalysis,
}
impl Serialize for StatementView<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let s = self.inner;
let mut m = serializer.serialize_map(Some(4))?;
m.serialize_entry("source", s.source())?;
m.serialize_entry("diagnostics", &DiagsOfStatement(s))?;
m.serialize_entry("defined_relations", &DefinedRelationsView(s))?;
match lineage_view(s) {
Some(l) => m.serialize_entry("lineage", &l)?,
None => m.serialize_entry("lineage", &Value::Null)?,
}
m.end()
}
}
struct DiagsOfStatement<'a>(&'a StatementAnalysis);
impl Serialize for DiagsOfStatement<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let diags = self.0.diagnostics();
let mut seq = serializer.serialize_seq(Some(diags.len()))?;
for d in diags {
seq.serialize_element(&DiagView::from(d))?;
}
seq.end()
}
}
struct DefinedRelationsView<'a>(&'a StatementAnalysis);
impl Serialize for DefinedRelationsView<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let rels = self.0.defined_relations();
let mut seq = serializer.serialize_seq(Some(rels.len()))?;
for d in rels {
seq.serialize_element(&DefinedRelationRec {
name: &d.name,
is_view: d.is_view,
})?;
}
seq.end()
}
}
#[derive(Serialize)]
struct DefinedRelationRec<'a> {
name: &'a str,
is_view: bool,
}
struct DiagView<'a> {
inner: &'a Diagnostic,
}
impl<'a> From<&'a Diagnostic> for DiagView<'a> {
fn from(inner: &'a Diagnostic) -> Self {
Self { inner }
}
}
impl Serialize for DiagView<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let d = self.inner;
let help = d.help().map(|h| match h {
Help::Suggestion(s) => s.as_str(),
});
let len = if help.is_some() { 6 } else { 5 };
let mut m = serializer.serialize_map(Some(len))?;
m.serialize_entry(
"severity",
match d.severity() {
SemSeverity::Error => "error",
SemSeverity::Warning => "warning",
SemSeverity::Info => "info",
SemSeverity::Hint => "hint",
},
)?;
m.serialize_entry("message", &d.message().to_string())?;
m.serialize_entry("start_offset", &d.start().as_usize())?;
m.serialize_entry("end_offset", &d.end().as_usize())?;
m.serialize_entry("code", &diag_code(d.message()))?;
if let Some(h) = help {
m.serialize_entry("help", h)?;
}
m.end()
}
}
struct LineageView<'a> {
stmt: &'a StatementAnalysis,
}
fn lineage_view(s: &StatementAnalysis) -> Option<LineageView<'_>> {
s.lineage()?;
Some(LineageView { stmt: s })
}
impl Serialize for LineageView<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let s = self.stmt;
let cols = s.lineage().expect("guarded by lineage_view");
let rels = s
.relations_accessed()
.expect("relations_accessed present when lineage is present");
let tables = s
.physical_tables_accessed()
.expect("physical_tables_accessed present when lineage is present");
let mut m = serializer.serialize_map(Some(5))?;
m.serialize_entry("complete", &cols.is_complete())?;
m.serialize_entry("columns", &ColumnsSeq(cols.into_inner()))?;
m.serialize_entry("relations", &RelationsSeq(rels.into_inner()))?;
m.serialize_entry("physical_tables", &PhysicalTablesSeq(tables.into_inner()))?;
m.serialize_entry(
"unexpanded_views",
&UnexpandedViewsSeq(s.unexpanded_views()),
)?;
m.end()
}
}
struct ColumnsSeq<'a>(&'a [syntaqlite::analysis::ColumnLineage]);
impl Serialize for ColumnsSeq<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for c in self.0 {
let origin = c.origin.as_ref().map(|o| OriginRec {
table: &o.table,
column: &o.column,
});
seq.serialize_element(&ColumnRec {
name: &c.name,
index: c.index,
origin,
})?;
}
seq.end()
}
}
#[derive(Serialize)]
struct ColumnRec<'a> {
name: &'a str,
index: u32,
origin: Option<OriginRec<'a>>,
}
#[derive(Serialize)]
struct OriginRec<'a> {
table: &'a str,
column: &'a str,
}
struct RelationsSeq<'a>(&'a [syntaqlite::analysis::RelationAccess]);
impl Serialize for RelationsSeq<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for r in self.0 {
seq.serialize_element(&RelationRec {
name: &r.name,
kind: match r.kind {
RelationKind::Table => "table",
RelationKind::View => "view",
},
})?;
}
seq.end()
}
}
#[derive(Serialize)]
struct RelationRec<'a> {
name: &'a str,
kind: &'static str,
}
struct PhysicalTablesSeq<'a>(&'a [syntaqlite::analysis::PhysicalTableAccess]);
impl Serialize for PhysicalTablesSeq<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for t in self.0 {
seq.serialize_element(t.name.as_str())?;
}
seq.end()
}
}
struct UnexpandedViewsSeq<'a>(&'a [String]);
impl Serialize for UnexpandedViewsSeq<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for v in self.0 {
seq.serialize_element(v.as_str())?;
}
seq.end()
}
}
const DIAG_CODE_PARSE_ERROR: u32 = 0;
const DIAG_CODE_UNKNOWN_TABLE: u32 = 1;
const DIAG_CODE_UNKNOWN_COLUMN: u32 = 2;
const DIAG_CODE_UNKNOWN_FUNCTION: u32 = 3;
const DIAG_CODE_UNKNOWN_MODULE: u32 = 4;
const DIAG_CODE_FUNCTION_ARITY: u32 = 5;
const DIAG_CODE_CTE_COLUMN_COUNT_MISMATCH: u32 = 6;
fn diag_code(msg: &DiagnosticMessage) -> u32 {
match msg {
DiagnosticMessage::ParseError(_) => DIAG_CODE_PARSE_ERROR,
DiagnosticMessage::UnknownTable { .. } => DIAG_CODE_UNKNOWN_TABLE,
DiagnosticMessage::UnknownColumn { .. } => DIAG_CODE_UNKNOWN_COLUMN,
DiagnosticMessage::UnknownFunction { .. } => DIAG_CODE_UNKNOWN_FUNCTION,
DiagnosticMessage::UnknownModule { .. } => DIAG_CODE_UNKNOWN_MODULE,
DiagnosticMessage::FunctionArity { .. } => DIAG_CODE_FUNCTION_ARITY,
DiagnosticMessage::CteColumnCountMismatch { .. } => DIAG_CODE_CTE_COLUMN_COUNT_MISMATCH,
}
}