kevy-cli 7.0.0

kevy operations CLI: redis-cli-style REPL plus the migration toolchain (import/export/verify/diff/inspect/digest).
Documentation
//! `import-csv <file> (--prefix p --pk col | --table t | --key-column c)
//! [--header] [--columns a,b] [--delimiter c] [--null-marker s] [--resume]
//! [--strict]`: one hash per record, `<prefix><pk>`, written in pipelined
//! batches.
//!
//! `--table` reads the prefix, the primary key and the columns from
//! `TABLE.DESCRIBE`, and writes only declared columns. `--key-column`
//! takes each record's whole key from that column (what `export-csv`
//! writes as `key`) and does not store it as a field.
//!
//! An empty cell (or the null marker) writes no field: a missing field is
//! NULL. Progress is kept in `<file>.progress` as `import` keeps it.

use super::csv;
use super::options::Common;
use crate::migrate::{ImportStart, OnErrorReply};
use crate::rcli::send::write_out;
use crate::rcli::session::{Session, eprint_bytes};
use kevy_resp::Reply;
use std::path::Path;

/// Records per pipelined batch.
const BATCH: usize = 512;

struct Plan {
    file: Vec<u8>,
    prefix: Vec<u8>,
    pk: Vec<u8>,
    table: Option<Vec<u8>>,
    key_column: Option<Vec<u8>>,
    /// With `--table`: the declared columns, the only ones written.
    declared: Option<Vec<Vec<u8>>>,
    header: bool,
    columns: Vec<Vec<u8>>,
    delimiter: u8,
    null_marker: Option<Vec<u8>>,
    start: ImportStart,
    on_error: OnErrorReply,
}

/// Run `import-csv`; the exit code.
pub(crate) fn run(s: &mut Session, common: &Common) -> u8 {
    let Some(mut plan) = parse(&common.args) else { return 1 };
    if plan.table.is_some() && !from_table(s, &mut plan) {
        return 1;
    }
    let path = std::path::PathBuf::from(String::from_utf8_lossy(&plan.file).into_owned());
    let text = match std::fs::read(&path) {
        Ok(t) => t,
        Err(e) => {
            return fail(&[
                b"cannot read '",
                &plan.file,
                b"': ",
                crate::rcli::conn::strerror(&e).as_bytes(),
            ]);
        }
    };
    let mut at = 0;
    if plan.header {
        let Some(head) = csv::next(&text, 0, plan.delimiter) else {
            return fail(&[b"the file has no header record"]);
        };
        plan.columns = head.fields;
        at = head.end;
    }
    let key = plan.key_column.as_ref().unwrap_or(&plan.pk);
    let Some(pk_at) = plan.columns.iter().position(|c| c == key) else {
        return fail(&[b"no column named '", key, b"' (give --header or --columns)"]);
    };
    warn_if_declared(s, &plan.prefix);
    import(s, &plan, &path, &text, at, pk_at)
}

fn import(
    s: &mut Session,
    plan: &Plan,
    path: &Path,
    text: &[u8],
    header_end: usize,
    pk_at: usize,
) -> u8 {
    let (mut progress, start) = match crate::migrate::open_progress(path, plan.start) {
        Ok(p) => p,
        Err(e) => {
            return fail(&[
                b"cannot write the progress file: ",
                crate::rcli::conn::strerror(&e).as_bytes(),
            ]);
        }
    };
    let (mut at, mut rows, mut errors) = (start.max(header_end as u64) as usize, 0u64, 0u64);
    loop {
        let (batch, end) = next_batch(plan, text, at, pk_at);
        if batch.is_empty() {
            break;
        }
        match send(s, plan, &batch) {
            Ok((sent, failed)) => (rows, errors) = (rows + sent, errors + failed),
            Err(code) => return code,
        }
        at = end;
        if let Err(e) = crate::migrate::write_progress(&mut progress, at as u64) {
            return fail(&[
                b"cannot write the progress file: ",
                crate::rcli::conn::strerror(&e).as_bytes(),
            ]);
        }
    }
    let prefix = String::from_utf8_lossy(&plan.prefix);
    write_out(format!("imported {rows} rows into {prefix} ({errors} errors)\n").as_bytes());
    if errors > 0 { 3 } else { 0 }
}

/// Up to BATCH commands from the records at `at`, and where they end.
fn next_batch(plan: &Plan, text: &[u8], at: usize, pk_at: usize) -> (Vec<Vec<Vec<u8>>>, usize) {
    let (mut batch, mut end) = (Vec::new(), at);
    while batch.len() < BATCH {
        let Some(record) = csv::next(text, end, plan.delimiter) else { break };
        end = record.end;
        if let Some(cmd) = command(plan, &record.fields, pk_at) {
            batch.push(cmd);
        }
    }
    (batch, end)
}

/// One pipelined batch: rows sent and error replies; `Err(exit code)` when
/// the link is lost or `--strict` stops at an error.
fn send(s: &mut Session, plan: &Plan, batch: &[Vec<Vec<u8>>]) -> Result<(u64, u64), u8> {
    let commands: Vec<Vec<&[u8]>> =
        batch.iter().map(|c| c.iter().map(Vec::as_slice).collect()).collect();
    let replies = s.pipeline(&commands).map_err(|e| {
        eprint_bytes(&[b"Error: ", e.text().as_bytes(), b"\n"]);
        2u8
    })?;
    let mut errors = 0;
    for msg in replies.iter().filter_map(|r| if let Reply::Error(m) = r { Some(m) } else { None }) {
        errors += 1;
        eprint_bytes(&[b"(error) ", msg, b"\n"]);
        if plan.on_error == OnErrorReply::Abort {
            return Err(3);
        }
    }
    Ok((replies.len() as u64, errors))
}

/// `HSET <key> col val …` for the non-empty cells; `None` for a record
/// without a key. The key is `<prefix><pk>`, or with `--key-column` that
/// column's value, which is then not a field.
fn command(plan: &Plan, fields: &[Vec<u8>], key_at: usize) -> Option<Vec<Vec<u8>>> {
    let key = fields.get(key_at).filter(|v| !v.is_empty())?;
    let key = match plan.key_column {
        Some(_) => key.clone(),
        None => [plan.prefix.as_slice(), key].concat(),
    };
    let mut cmd = vec![b"HSET".to_vec(), key];
    for (i, (name, value)) in plan.columns.iter().zip(fields).enumerate() {
        let null = value.is_empty() || plan.null_marker.as_ref() == Some(value);
        let skipped = (plan.key_column.is_some() && i == key_at)
            || plan.declared.as_ref().is_some_and(|d| !d.contains(name));
        if !null && !skipped {
            cmd.extend([name.clone(), value.clone()]);
        }
    }
    Some(cmd)
}

/// `--table`: the prefix, primary key and declared columns from
/// `TABLE.DESCRIBE`; `false` after saying why not.
fn from_table(s: &mut Session, plan: &mut Plan) -> bool {
    let table = plan.table.clone().unwrap_or_default();
    let d = match super::described::fetch(s, super::described::Kind::Table, &table) {
        Ok(Some(d)) => d,
        Ok(None) => {
            fail(&[b"no table named '", &table, b"'"]);
            return false;
        }
        Err(()) => return false,
    };
    plan.prefix = d.text(b"prefix").unwrap_or_default().to_vec();
    plan.pk = d.text(b"pk").unwrap_or_default().to_vec();
    let declared: Vec<Vec<u8>> = d.columns().into_iter().map(|(c, _)| c).collect();
    if !plan.header && plan.columns.is_empty() {
        plan.columns.clone_from(&declared);
    }
    plan.declared = Some(declared);
    true
}

/// Declared indexes on the prefix update on every row written.
fn warn_if_declared(s: &mut Session, prefix: &[u8]) {
    let Ok(Reply::Array(indexes)) = s.request(&[b"IDX.LIST"]) else { return };
    let needle = super::rows::from_pair_rows(&Reply::Array(indexes)).is_some_and(|rows| {
        let at = rows.columns.iter().position(|c| c == b"prefix");
        rows.rows
            .iter()
            .any(|r| at.is_some_and(|i| r[i] == super::rows::Cell::Text(prefix.to_vec())))
    });
    if needle {
        eprint_bytes(&[
            b"kevy-cli: indexes are declared on '",
            prefix,
            b"'; each row updates them as it lands (importing before declaring is cheaper)\n",
        ]);
    }
}

fn fail(parts: &[&[u8]]) -> u8 {
    eprint_bytes(&[b"kevy-cli: import-csv: ", &parts.concat(), b"\n"]);
    1
}

fn parse(args: &[Vec<u8>]) -> Option<Plan> {
    let plan = read_flags(args)?;
    let keyed = plan.table.is_some()
        || plan.key_column.is_some()
        || (!plan.prefix.is_empty() && !plan.pk.is_empty());
    let described = plan.header || !plan.columns.is_empty() || plan.table.is_some();
    if plan.file.is_empty() || !keyed || !described {
        return usage(b"");
    }
    Some(plan)
}

/// The flags and the file, as given; `parse` decides whether they suffice.
fn read_flags(args: &[Vec<u8>]) -> Option<Plan> {
    let mut plan = Plan {
        file: Vec::new(),
        prefix: Vec::new(),
        pk: Vec::new(),
        table: None,
        key_column: None,
        declared: None,
        header: false,
        columns: Vec::new(),
        delimiter: b',',
        null_marker: None,
        start: ImportStart::Fresh,
        on_error: OnErrorReply::Count,
    };
    let mut i = 0;
    while i < args.len() {
        let value = args.get(i + 1).cloned();
        match (args[i].as_slice(), value) {
            (b"--prefix", Some(v)) => plan.prefix = v,
            (b"--pk", Some(v)) => plan.pk = v,
            (b"--table", Some(v)) => plan.table = Some(v),
            (b"--key-column", Some(v)) => plan.key_column = Some(v),
            (b"--columns", Some(v)) => {
                plan.columns = v.split(|&b| b == b',').map(<[u8]>::to_vec).collect()
            }
            (b"--delimiter", Some(v)) if v.len() == 1 => plan.delimiter = v[0],
            (b"--null-marker", Some(v)) => plan.null_marker = Some(v),
            (flag, _) if flag.starts_with(b"--") => {
                match flag {
                    b"--header" => plan.header = true,
                    b"--resume" => plan.start = ImportStart::Resume,
                    b"--strict" => plan.on_error = OnErrorReply::Abort,
                    _ => return usage(flag),
                }
                i += 1;
                continue;
            }
            (file, _) if plan.file.is_empty() => {
                plan.file = file.to_vec();
                i += 1;
                continue;
            }
            (other, _) => return usage(other),
        }
        i += 2;
    }
    Some(plan)
}

fn usage<T>(bad: &[u8]) -> Option<T> {
    if !bad.is_empty() {
        eprint_bytes(&[b"kevy-cli: import-csv: unexpected '", bad, b"'\n"]);
    }
    eprint_bytes(&[b"usage: kevy-cli import-csv <file> (--prefix p --pk col | --table t | --key-column c) (--header | --columns a,b,...) [--delimiter c] [--null-marker s] [--resume] [--strict]\n"]);
    None
}