mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
//! Serving the same invocation against several destinations at once.

use super::open::{bounded, open};
use super::{context, Work};
use crate::cli::Cli;
use crate::config::vault::FileVault;
use crate::config::Config;
use crate::conn::profile::{Profile, SecretRef};
use crate::conn::store::ProfileStore;
use crate::error::{Failure, Result};
use crate::output::{Envelope, Gap};
use crate::runtime::{Gate, Shutdown};
use serde_json::{json, Value};
use std::future::Future;
use std::pin::Pin;
use std::task::Poll;

/// One piece of work in flight, borrowing what the invocation already holds.
///
/// Boxed because the pieces are driven together from one place, and a boxed
/// future is the only way to hold several of them of the same type at once.
type Queued<'a, T> = Pin<Box<dyn Future<Output = T> + 'a>>;

/// Serves the same invocation against every stored destination.
///
/// One failing destination never stops the others: comparing environments
/// loses its value if one bad destination brings everything down, so the run
/// continues and the destination that failed is declared as a gap rather than
/// left out. The exit therefore stays a success while any destination
/// answered. Only when none did is there nothing to answer with, and then the
/// worst of the failures is the outcome.
///
/// # Errors
///
/// Returns a configuration failure when this host has no destination stored,
/// a usage failure when the ceiling handed in admits no work, and the worst
/// failure met when no destination could be served.
pub(super) async fn across_destinations(
    cli: &Cli,
    config: &Config,
    permits: usize,
    shutdown: &Shutdown,
    store: &ProfileStore,
    vault: &FileVault,
) -> Result<Envelope> {
    let stored = store.list();
    if stored.is_empty() {
        return Err(Failure::config(
            "profile.none_stored",
            "there is no destination stored on this host to run against",
        )
        .with_hint("add one with `conn add`"));
    }

    // The ceiling on simultaneous work is the one the caller already settled
    // for the whole invocation, and it is applied by the gate rather than
    // counted again here: a second count would be a second statement of the
    // same rule, which is how the product came to hold two different ones.
    let gate = Gate::new(permits)?;
    let gate = &gate;
    let queued: Vec<Queued<'_, (usize, String, Result<Value>)>> = stored
        .iter()
        .enumerate()
        .map(|(order, profile)| {
            let served: Queued<'_, _> = Box::pin(async move {
                // The licence is held for the whole of this destination and
                // returned when it answers, so what the gate bounds is the
                // number of destinations open at once, which is the cost the
                // server pays.
                let outcome = async {
                    let _licence = gate.acquire(&shutdown.token()).await?;
                    one_destination(cli, config, shutdown, profile, vault).await
                }
                .await;
                (order, profile.name.clone(), outcome)
            });
            served
        })
        .collect();

    let mut served = drive_all(queued).await;
    // Completion order is whichever destination answered first, and an answer
    // whose blocks move between runs cannot be compared against yesterday's.
    served.sort_by_key(|(order, _, _)| *order);

    assemble(
        cli.command.published_name(),
        served.into_iter().map(|(_, name, outcome)| (name, outcome)),
    )
}

/// Builds the one answer that carries every destination that was reached.
///
/// Each destination gets its own block, named, so a caller comparing two
/// environments can tell which answer came from where. A destination that
/// failed is declared as a gap carrying its own cause, because an answer
/// silently missing a destination reads exactly like an answer from a
/// destination that had nothing to say.
///
/// # Errors
///
/// Returns the worst failure met when no destination answered at all, since
/// there is then no partial answer for the gaps to travel on.
fn assemble(
    command: &'static str,
    served: impl Iterator<Item = (String, Result<Value>)>,
) -> Result<Envelope> {
    let mut blocks = Vec::new();
    let mut envelope = Envelope::new(command, Value::Null);
    let mut worst: Option<Failure> = None;

    for (name, outcome) in served {
        match outcome {
            Ok(data) => blocks.push(json!({ "profile": name, "data": data })),
            Err(failure) => {
                envelope = envelope.with_gap(Gap::new(
                    name,
                    failure.code().to_owned(),
                    failure.message().to_owned(),
                ));
                // The worst is the one whose exit number is highest, which is
                // the outcome a caller would have got had it asked for that
                // destination alone.
                worst = Some(match worst {
                    Some(kept) if kept.exit_number().code() >= failure.exit_number().code() => kept,
                    _ => failure,
                });
            }
        }
    }

    // Nothing was served, so there is no partial answer to carry the gaps. The
    // reason of every destination travels on the failure instead, so a caller
    // still learns why each one was lost and not only the worst.
    if let (true, Some(failure)) = (blocks.is_empty(), worst) {
        return Err(envelope.gaps.iter().fold(failure, |carried, gap| {
            carried.with_cause(format!("{}: {} ({})", gap.object, gap.detail, gap.cause))
        }));
    }

    envelope.data = json!({ "results": blocks });
    Ok(envelope)
}

/// Serves one destination of a run across several, answering with its payload.
///
/// # Errors
///
/// Returns whatever reaching or serving this destination returns, which the
/// caller turns into the gap that names it.
async fn one_destination(
    cli: &Cli,
    config: &Config,
    shutdown: &Shutdown,
    profile: &Profile,
    vault: &FileVault,
) -> Result<Value> {
    let secret =
        crate::conn::profile::SecretVault::load(vault, &SecretRef::for_profile(&profile.name))?;
    // The database this invocation names is pinned onto every destination it
    // reaches, exactly as it is for a single one: a run across destinations
    // that opened each session in that destination's own database would be
    // comparing answers read from different databases.
    let profile = &super::open::targeted(profile.clone(), cli);
    let settings = context::connection(config, cli, Some(profile), secret)?;
    let session = open(profile, &settings, cli, config, &shutdown.token()).await?;

    let work = Work {
        cli,
        config,
        scope: profile.database.clone(),
        shutdown,
        many: true,
    };
    // No continuous flow is opened across destinations, so every destination
    // answers with an envelope and the empty case cannot arise.
    let answered = bounded(&work, session, profile, &settings, &mut std::io::sink()).await?;
    Ok(answered.map_or(Value::Null, |envelope| envelope.data))
}

/// Drives all of the given work until every part of it has answered.
///
/// Everything is driven from the one task that awaits this, so nothing has to
/// outlive the invocation to be worked on simultaneously, and no piece of what
/// the invocation already holds has to be copied to be shared.
///
/// There is no ceiling here, on purpose. This used to admit work a few at a
/// time and count what was running, which was a second limiter standing beside
/// the admission gate: one rule, two implementations, free to disagree. Each
/// part now waits at the gate before it begins, so polling all of them is
/// exactly as bounded as the gate is, and the parts still waiting cost a poll
/// each rather than a place in a queue nobody can see.
///
/// Work is polled in the order it was declared, so the first parts through the
/// gate are the first ones declared.
async fn drive_all<T>(mut running: Vec<Queued<'_, T>>) -> Vec<T> {
    let mut finished = Vec::with_capacity(running.len());

    std::future::poll_fn(|context| {
        let mut index = 0;
        while index < running.len() {
            match running[index].as_mut().poll(context) {
                Poll::Ready(value) => {
                    finished.push(value);
                    // Finished work is dropped rather than polled again, which
                    // a future that already answered forbids. What follows it
                    // shifts into this place and is polled on this same pass.
                    drop(running.remove(index));
                }
                Poll::Pending => index += 1,
            }
        }

        if running.is_empty() {
            Poll::Ready(())
        } else {
            // Whatever is still waiting registered its own waker, on the gate
            // or on the network, so polling again now would only spin.
            Poll::Pending
        }
    })
    .await;

    finished
}

#[cfg(test)]
mod tests {
    use super::*;

    /// One destination failing must not cost the caller the others, and the
    /// one that failed must be named rather than quietly left out.
    #[test]
    fn a_destination_that_failed_is_declared_and_the_others_still_answer() {
        let envelope = assemble(
            "query run",
            [
                ("producao".to_owned(), Ok(json!({"rows": [{"id": 1}]}))),
                (
                    "homologacao".to_owned(),
                    Err(Failure::network(
                        "conn.unreachable",
                        "no route to the server",
                    )),
                ),
                ("desenvolvimento".to_owned(), Ok(json!({"rows": []}))),
            ]
            .into_iter(),
        )
        .expect("a partly served run is still an answer");

        let blocks = envelope.data["results"].as_array().expect("one block each");
        assert_eq!(blocks.len(), 2, "the reachable destinations both answered");
        assert_eq!(blocks[0]["profile"], "producao");
        assert_eq!(blocks[1]["profile"], "desenvolvimento");

        assert_eq!(envelope.gaps.len(), 1, "the failure must be declared");
        assert_eq!(envelope.gaps[0].object, "homologacao");
        assert_eq!(envelope.gaps[0].cause, "conn.unreachable");
    }

    /// With nothing reachable there is no partial answer to carry the gaps, so
    /// the outcome is the worst of the failures met.
    #[test]
    fn with_no_destination_reachable_the_worst_failure_is_the_outcome() {
        let failure = assemble(
            "query run",
            [
                (
                    "a".to_owned(),
                    Err(Failure::network("conn.unreachable", "no route")),
                ),
                (
                    "b".to_owned(),
                    Err(Failure::new(
                        crate::error::Category::Timeout,
                        "timeout.connect",
                        "the destination did not answer in time",
                    )),
                ),
            ]
            .into_iter(),
        )
        .expect_err("nothing was served");
        assert_eq!(failure.code(), "timeout.connect");
        // Losing every destination must not lose why each one was lost.
        assert_eq!(
            failure.causes().len(),
            2,
            "each destination carries its own reason"
        );
        assert!(failure
            .causes()
            .iter()
            .any(|cause| cause.starts_with("a: ")));
        assert!(failure
            .causes()
            .iter()
            .any(|cause| cause.starts_with("b: ")));
    }

    /// FR-067. Work runs simultaneously up to the ceiling and never past it,
    /// and the ceiling is the admission gate's, since there is only one.
    #[tokio::test]
    async fn work_runs_up_to_the_ceiling_and_no_further() {
        use std::cell::Cell;
        use tokio_util::sync::CancellationToken;

        let gate = Gate::new(2).expect("a legal ceiling");
        let gate = &gate;
        let token = &CancellationToken::new();
        let running = &Cell::new(0usize);
        let highest = &Cell::new(0usize);
        let queued: Vec<Queued<'_, usize>> = (0..6)
            .map(|index| {
                let work: Queued<'_, usize> = Box::pin(async move {
                    let _licence = gate.acquire(token).await.expect("admitted");
                    running.set(running.get() + 1);
                    highest.set(highest.get().max(running.get()));
                    tokio::task::yield_now().await;
                    running.set(running.get() - 1);
                    index
                });
                work
            })
            .collect();

        let finished = drive_all(queued).await;
        assert_eq!(finished.len(), 6, "every piece of work is finished");
        assert_eq!(highest.get(), 2, "the ceiling is the ceiling");
    }
}