use serde_json::{json, Value};
use super::errors::{err, Category, Error, Result};
use super::types::{MAX_SQL_BYTES, MAX_WAL_SEGMENT_BYTES};
fn encode_line(sql: &str) -> String {
json!({ "sql": sql }).to_string()
}
pub(crate) fn line_bytes(sql: &str) -> u64 {
encode_line(sql).len() as u64 + 1
}
pub(crate) fn assert_statement_within_limit(sql: &str) -> Result<()> {
let size = sql.len() as u64;
if size > MAX_SQL_BYTES {
return Err(err(
Category::LimitExceeded,
format!(
"durable: statement is {size} UTF-8 bytes, over the V1 per-statement limit of \
{MAX_SQL_BYTES}"
),
)
.with_limit(MAX_SQL_BYTES, size));
}
Ok(())
}
pub(crate) fn encode_segment(statements: &[String]) -> Result<Vec<u8>> {
let mut out = String::new();
for sql in statements {
assert_statement_within_limit(sql)?;
out.push_str(&encode_line(sql));
out.push('\n');
}
let size = out.len() as u64;
if size > MAX_WAL_SEGMENT_BYTES {
return Err(err(
Category::LimitExceeded,
format!(
"durable: WAL segment would be {size} bytes, over the V1 limit of \
{MAX_WAL_SEGMENT_BYTES}; flush more often"
),
)
.with_limit(MAX_WAL_SEGMENT_BYTES, size));
}
Ok(out.into_bytes())
}
pub(crate) fn decode_segment(data: &[u8], key: &str) -> Result<Vec<String>> {
let size = data.len() as u64;
if size > MAX_WAL_SEGMENT_BYTES {
return Err(err(
Category::LimitExceeded,
format!(
"durable: WAL segment {key} is {size} bytes, over the V1 limit of \
{MAX_WAL_SEGMENT_BYTES}"
),
)
.with_limit(MAX_WAL_SEGMENT_BYTES, size));
}
if data.is_empty() {
return Ok(Vec::new());
}
if data[data.len() - 1] != b'\n' {
return Err(err(
Category::Corrupt,
format!("durable: WAL segment {key} does not end with a newline; it may be truncated"),
));
}
let text = std::str::from_utf8(&data[..data.len() - 1]).map_err(|e| {
Error::wrap(
Category::Corrupt,
e,
format!("durable: WAL segment {key} is not valid UTF-8"),
)
})?;
let mut statements = Vec::new();
for (index, line) in text.split('\n').enumerate() {
let lineno = index + 1;
let record: Value = serde_json::from_str(line).map_err(|e| {
Error::wrap(
Category::Corrupt,
e,
format!("durable: WAL segment {key} line {lineno} is not a JSON object"),
)
})?;
let sql = record
.as_object()
.and_then(|record| record.get("sql"))
.and_then(Value::as_str)
.ok_or_else(|| {
err(
Category::Corrupt,
format!("durable: WAL segment {key} line {lineno} has no string \"sql\" field"),
)
})?;
assert_statement_within_limit(sql)?;
statements.push(sql.to_string());
}
Ok(statements)
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn a_segment_round_trips_through_the_frozen_shape() -> Result<()> {
let statements = vec![
"INSERT INTO t VALUES (1)".to_string(),
"INSERT INTO t VALUES ('a\nb')".to_string(),
"ALTER TABLE t UPDATE n = 2 WHERE id < 5 AND tag = 'a&b'".to_string(),
];
let segment = encode_segment(&statements)?;
assert_eq!(segment.last(), Some(&b'\n'));
assert_eq!(segment.iter().filter(|b| **b == b'\n').count(), 3);
let text = String::from_utf8(segment.clone()).unwrap();
assert!(text.contains(r#"'a\nb'"#), "{text}");
assert!(text.contains("a&b"), "{text}");
assert_eq!(
decode_segment(&segment, "wal/1-1-aaaaaaaa.jsonl")?,
statements
);
Ok(())
}
#[test]
fn the_budget_agrees_with_the_encoder_byte_for_byte() -> Result<()> {
let statements = vec![
"INSERT INTO t VALUES (1)".to_string(),
"INSERT INTO t VALUES ('ünïcøde \" quoted')".to_string(),
];
let budgeted: u64 = statements.iter().map(|sql| line_bytes(sql)).sum();
assert_eq!(budgeted, encode_segment(&statements)?.len() as u64);
Ok(())
}
#[test]
fn an_empty_segment_replays_nothing() -> Result<()> {
assert!(encode_segment(&[])?.is_empty());
assert!(decode_segment(b"", "wal/1-1-aaaaaaaa.jsonl")?.is_empty());
Ok(())
}
#[test]
fn a_segment_that_does_not_parse_is_corrupt_rather_than_skipped() {
for body in [
&b"{\"sql\":\"SELECT 1\"}"[..], &b"{\"sql\":\"SELECT 1\"}\nnot json\n"[..],
&b"{\"statement\":\"SELECT 1\"}\n"[..], &b"{\"sql\":42}\n"[..], &b"[\"SELECT 1\"]\n"[..], ] {
let error = decode_segment(body, "wal/1-1-aaaaaaaa.jsonl")
.expect_err("a segment that does not parse cannot be replayed");
assert_eq!(error.category(), Category::Corrupt, "{body:?}");
}
}
#[test]
fn an_oversized_statement_is_refused_on_both_sides() {
let huge = "x".repeat(MAX_SQL_BYTES as usize + 1);
let error = assert_statement_within_limit(&huge).expect_err("over the per-statement limit");
assert_eq!(error.category(), Category::LimitExceeded);
assert_eq!(error.limit(), Some((MAX_SQL_BYTES, MAX_SQL_BYTES + 1)));
let error = encode_segment(std::slice::from_ref(&huge)).expect_err("nor may it be encoded");
assert_eq!(error.category(), Category::LimitExceeded);
}
}