exonum-supervisor 1.0.0

// Copyright 2020 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use exonum::{
    crypto,
    helpers::{Height, ValidatorId},
    merkledb::ObjectHash,
    messages::{AnyTx, Verified},
    runtime::{
        ArtifactId, CommonError, ErrorMatch, InstanceId, RuntimeIdentifier, SnapshotExt,
        SUPERVISOR_INSTANCE_ID,
    },
};
use exonum_rust_runtime::{
    api,
    spec::{JustFactory, Spec},
    RustRuntimeBuilder, ServiceFactory,
};
use exonum_supervisor::{
    ArtifactError, CommonError as SupervisorCommonError, ConfigPropose, DeployRequest,
    DeployResult, ServiceError, Supervisor, SupervisorInterface,
};
use exonum_testkit::{ApiKind, TestKit, TestKitApi, TestKitBuilder};

use crate::{
    inc::{IncInterface, IncService, SERVICE_ID, SERVICE_NAME},
    utils::{build_confirmation_transactions, CFG_CHANGE_HEIGHT},
};

const DEPLOY_HEIGHT: Height = CFG_CHANGE_HEIGHT;
const START_HEIGHT: Height = Height(DEPLOY_HEIGHT.0 * 2 + 1);

mod config;
mod config_api;
mod consensus_config;
mod deploy_failures;
mod inc;
mod migrations;
mod service_lifecycle;
mod supervisor_config;
mod utils;

fn default_artifact() -> ArtifactId {
    IncService.artifact_id()
}

async fn assert_count(api: &TestKitApi, service_name: &'static str, expected_count: u64) {
    let real_count: u64 = api
        .public(ApiKind::Service(service_name))
        .get("v1/counter")
        .await
        .unwrap();
    assert_eq!(real_count, expected_count);
}

/// Check that the service's counter isn't started yet (no Inc txs were received).
async fn assert_count_is_not_set(api: &TestKitApi, service_name: &'static str) {
    let response: api::Result<u64> = api
        .public(ApiKind::Service(service_name))
        .get("v1/counter")
        .await;
    assert!(response.is_err());
}

#[allow(clippy::let_and_return)] // doesn't work otherwise
fn artifact_exists(testkit: &TestKit, name: &str) -> bool {
    let snapshot = testkit.snapshot();
    let artifacts = snapshot.for_dispatcher().service_artifacts();
    let artifact_exists = artifacts.keys().any(|artifact| artifact.name == name);
    artifact_exists
}

fn service_instance_exists(testkit: &TestKit, name: &str) -> bool {
    let snapshot = testkit.snapshot();
    snapshot.for_dispatcher().get_instance(name).is_some()
}

fn find_instance_id(testkit: &TestKit, instance_name: &str) -> InstanceId {
    let snapshot = testkit.snapshot();
    snapshot
        .for_dispatcher()
        .get_instance(instance_name)
        .expect("Can't find the instance")
        .spec
        .id
}

async fn deploy_artifact(api: &TestKitApi, request: DeployRequest) -> crypto::Hash {
    let hash: crypto::Hash = api
        .private(ApiKind::Service("supervisor"))
        .query(&request)
        .post("deploy-artifact")
        .await
        .unwrap();
    hash
}

fn deploy_artifact_manually(
    testkit: &mut TestKit,
    request: &DeployRequest,
    validator_id: ValidatorId,
) -> crypto::Hash {
    let keypair = testkit.validator(validator_id).service_keypair();
    let signed_request =
        keypair.request_artifact_deploy(SUPERVISOR_INSTANCE_ID, request.to_owned());
    let request_hash = signed_request.object_hash();
    testkit.add_tx(signed_request);
    request_hash
}

async fn start_service(api: &TestKitApi, request: ConfigPropose) -> crypto::Hash {
    // Even though this method sends a config proposal, it's *intended* to start
    // services (so the callee-side code will be more readable).
    // However, this convention is up to test writers.

    let hash: crypto::Hash = api
        .private(ApiKind::Service("supervisor"))
        .query(&request)
        .post("propose-config")
        .await
        .unwrap();
    hash
}

fn start_service_manually(
    testkit: &mut TestKit,
    request: &ConfigPropose,
    validator_id: ValidatorId,
) -> crypto::Hash {
    let keypair = testkit.validator(validator_id).service_keypair();
    let signed_request = keypair.propose_config_change(SUPERVISOR_INSTANCE_ID, request.to_owned());
    let request_hash = signed_request.object_hash();
    testkit.add_tx(signed_request);
    request_hash
}

fn deploy_confirmation(
    testkit: &TestKit,
    request: &DeployRequest,
    validator_id: ValidatorId,
) -> Verified<AnyTx> {
    let confirmation = DeployResult::ok(request.to_owned());
    testkit
        .validator(validator_id)
        .service_keypair()
        .report_deploy_result(SUPERVISOR_INSTANCE_ID, confirmation)
}

fn deploy_confirmation_hash(
    testkit: &TestKit,
    request: &DeployRequest,
    validator_id: ValidatorId,
) -> crypto::Hash {
    let confirmation_signed = deploy_confirmation(testkit, request, validator_id);
    confirmation_signed.object_hash()
}

fn deploy_confirmation_hash_default(testkit: &TestKit, request: &DeployRequest) -> crypto::Hash {
    deploy_confirmation_hash(testkit, request, ValidatorId(0))
}

fn deploy_request(artifact: ArtifactId, deadline_height: Height) -> DeployRequest {
    DeployRequest::new(artifact, deadline_height)
}

fn start_service_request(
    artifact: ArtifactId,
    name: impl Into<String>,
    deadline_height: Height,
) -> ConfigPropose {
    ConfigPropose::new(0, deadline_height).start_service(artifact, name, Vec::default())
}

async fn deploy_default(testkit: &mut TestKit) {
    let artifact = default_artifact();
    let api = testkit.api();

    assert!(!artifact_exists(testkit, &artifact.name));

    let request = deploy_request(artifact.clone(), DEPLOY_HEIGHT);
    let deploy_confirmation_hash = deploy_confirmation_hash_default(testkit, &request);
    let hash = deploy_artifact(&api, request).await;
    let block = testkit.create_block();
    block[hash].status().unwrap();

    // Confirmation is ready.
    assert!(testkit.is_tx_in_pool(&deploy_confirmation_hash));
    testkit.create_blocks_until(DEPLOY_HEIGHT);

    // Confirmation is gone now.
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_hash));

    assert!(artifact_exists(&testkit, &artifact.name));
}

async fn start_service_instance(testkit: &mut TestKit, instance_name: &str) -> InstanceId {
    assert!(!service_instance_exists(testkit, instance_name));

    let api = testkit.api();
    let request = start_service_request(default_artifact(), instance_name, START_HEIGHT);
    let hash = start_service(&api, request).await;
    let block = testkit.create_block();
    block[hash].status().unwrap();
    testkit.create_blocks_until(START_HEIGHT);

    assert!(service_instance_exists(testkit, instance_name));
    find_instance_id(testkit, instance_name)
}

fn testkit_with_inc_service() -> TestKit {
    TestKitBuilder::validator()
        .with_logger()
        .with(Supervisor::decentralized())
        .with(JustFactory::new(IncService))
        .build()
}

fn testkit_with_inc_service_and_n_validators(n: u16) -> TestKit {
    TestKitBuilder::validator()
        .with_logger()
        .with(Supervisor::decentralized())
        .with(JustFactory::new(IncService))
        .with_validators(n)
        .build()
}

fn testkit_with_inc_service_and_two_validators() -> TestKit {
    testkit_with_inc_service_and_n_validators(2)
}

fn testkit_with_inc_service_auditor_validator() -> TestKit {
    TestKitBuilder::auditor()
        .with_logger()
        .with(Supervisor::decentralized())
        .with(JustFactory::new(IncService))
        .with_validators(1)
        .build()
}

fn testkit_with_inc_service_and_static_instance() -> TestKit {
    TestKitBuilder::validator()
        .with_logger()
        .with(Supervisor::decentralized())
        .with(Spec::new(IncService).with_default_instance())
        .build()
}

fn available_services() -> RustRuntimeBuilder {
    RustRuntimeBuilder::new()
        .with_factory(IncService)
        .with_factory(Supervisor)
}

/// Just test that the Inc service works as intended.
#[tokio::test]
async fn test_static_service() {
    let mut testkit = testkit_with_inc_service_and_static_instance();
    let api = testkit.api();

    assert_count_is_not_set(&api, SERVICE_NAME).await;

    let keypair = crypto::KeyPair::random();
    api.send(keypair.inc(SERVICE_ID, 0)).await;
    testkit.create_block();
    assert_count(&api, SERVICE_NAME, 1).await;
    api.send(keypair.inc(SERVICE_ID, 1)).await;
    testkit.create_block();
    assert_count(&api, SERVICE_NAME, 2).await;
}

/// Test a normal dynamic service workflow with one validator.
#[tokio::test]
async fn test_dynamic_service_normal_workflow() {
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;
    let instance_name = "test_basics";
    let instance_id = start_service_instance(&mut testkit, instance_name).await;
    let api = testkit.api();

    assert_count_is_not_set(&api, instance_name).await;

    let keypair = crypto::KeyPair::random();
    api.send(keypair.inc(instance_id, 0)).await;
    testkit.create_block();
    assert_count(&api, instance_name, 1).await;

    api.send(keypair.inc(instance_id, 1)).await;
    testkit.create_block();
    assert_count(&api, instance_name, 2).await;
}

#[tokio::test]
async fn test_artifact_deploy_with_already_passed_deadline_height() {
    let mut testkit = testkit_with_inc_service();

    // We skip to Height(1) ...
    testkit.create_block();

    // ... but set Height(0) as a deadline.
    let bad_deadline_height = testkit.height().previous();

    let artifact = default_artifact();
    let api = testkit.api();

    let request = deploy_request(artifact.clone(), bad_deadline_height);
    let deploy_confirmation_hash = deploy_confirmation_hash_default(&testkit, &request);
    let hash = deploy_artifact(&api, request).await;
    let block = testkit.create_block();

    assert!(!artifact_exists(&testkit, &artifact.name));
    // No confirmation was generated
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_hash));

    let expected_err = ErrorMatch::from_fail(&SupervisorCommonError::ActualFromIsPast)
        .for_service(SUPERVISOR_INSTANCE_ID);
    assert_eq!(*block[hash].status().unwrap_err(), expected_err);
}

#[tokio::test]
async fn test_start_service_instance_with_already_passed_deadline_height() {
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;

    let api = testkit.api();
    let artifact = default_artifact();
    let instance_name = "inc_test";
    let bad_deadline_height = testkit.height().previous();
    let request = start_service_request(artifact, instance_name, bad_deadline_height);
    let hash = start_service(&api, request).await;
    let block = testkit.create_block();

    let expected_err = ErrorMatch::from_fail(&SupervisorCommonError::ActualFromIsPast)
        .with_description_containing("height for config proposal (2) is in the past")
        .for_service(SUPERVISOR_INSTANCE_ID);
    assert_eq!(*block[hash].status().unwrap_err(), expected_err);
}

#[tokio::test]
async fn test_try_run_unregistered_service_instance() {
    let mut testkit = testkit_with_inc_service();
    let api = testkit.api();

    // Deliberately missing the DeployRequest step.

    let instance_name = "wont_run";
    let request = start_service_request(default_artifact(), instance_name.to_owned(), Height(1000));
    let hash = start_service(&api, request).await;
    let block = testkit.create_block();

    let expected_err = ErrorMatch::from_fail(&ArtifactError::UnknownArtifact)
        .for_service(SUPERVISOR_INSTANCE_ID)
        .with_any_description();
    assert_eq!(*block[hash].status().unwrap_err(), expected_err);
}

#[tokio::test]
async fn test_bad_artifact_name() {
    let mut testkit = testkit_with_inc_service();
    let api = testkit.api();

    let bad_artifact = ArtifactId::from_raw_parts(
        RuntimeIdentifier::Rust as _,
        "does-not-exist".to_owned(),
        "1.0.0".parse().unwrap(),
    );
    let request = deploy_request(bad_artifact.clone(), DEPLOY_HEIGHT);
    let deploy_confirmation_hash = deploy_confirmation_hash_default(&testkit, &request);
    let hash = deploy_artifact(&api, request).await;

    let block = testkit.create_block();
    // The deploy request transaction was executed...
    block[hash].status().unwrap();
    // ... but no confirmation was generated ...
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_hash));
    testkit.create_block();

    // ...and no artifact was deployed.
    assert!(!artifact_exists(&testkit, &bad_artifact.name));
}

#[tokio::test]
async fn test_bad_runtime_id() {
    let mut testkit = testkit_with_inc_service();
    let api = testkit.api();
    let bad_runtime_id = 10_000;

    let mut artifact = IncService.artifact_id();
    artifact.runtime_id = bad_runtime_id;
    let request = deploy_request(artifact.clone(), DEPLOY_HEIGHT);
    let deploy_confirmation_hash = deploy_confirmation_hash_default(&testkit, &request);
    let hash = deploy_artifact(&api, request).await;
    let block = testkit.create_block();

    // The deploy request transaction was executed...
    block[hash].status().unwrap();
    // ... but no confirmation was generated ...
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_hash));

    testkit.create_block();
    // ...and no artifact was deployed.
    assert!(!artifact_exists(&testkit, &artifact.name));
}

#[tokio::test]
async fn test_empty_service_instance_name() {
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;

    let api = testkit.api();
    let artifact = default_artifact();
    let empty_instance_name = "";
    let deadline_height = testkit.height().next();
    let request = start_service_request(artifact, empty_instance_name, deadline_height);
    let hash = start_service(&api, request).await;
    let block = testkit.create_block();

    let expected_err = ErrorMatch::from_fail(&ServiceError::InvalidInstanceName)
        .with_description_containing("Service name is empty")
        .for_service(SUPERVISOR_INSTANCE_ID);
    assert_eq!(*block[hash].status().unwrap_err(), expected_err);
}

#[tokio::test]
async fn test_bad_service_instance_name() {
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;

    let api = testkit.api();
    let artifact = default_artifact();
    let bad_instance_name = "\u{2764}";

    let deadline_height = testkit.height().next();
    let request = start_service_request(artifact, bad_instance_name, deadline_height);
    let hash = start_service(&api, request).await;
    let block = testkit.create_block();

    let expected_msg = "Service name `\u{2764}` is invalid";
    let expected_err = ErrorMatch::from_fail(&ServiceError::InvalidInstanceName)
        .with_description_containing(expected_msg)
        .for_service(SUPERVISOR_INSTANCE_ID);
    assert_eq!(*block[hash].status().unwrap_err(), expected_err);
}

#[tokio::test]
async fn test_start_service_instance_twice() {
    let instance_name = "inc";
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;

    // Start the first instance
    {
        assert!(!service_instance_exists(&testkit, instance_name));

        let api = testkit.api();
        let deadline = testkit.height().next();
        let request = start_service_request(default_artifact(), instance_name, deadline);
        let hash = start_service(&api, request).await;
        let block = testkit.create_block();
        block[hash].status().unwrap();

        assert!(service_instance_exists(&testkit, instance_name));
    }

    // Try to start another instance with the same name
    {
        let api = testkit.api();

        let deadline = testkit.height().next();
        let request = start_service_request(default_artifact(), instance_name, deadline);
        let hash = start_service(&api, request).await;
        let block = testkit.create_block();

        let expected_err = ErrorMatch::from_fail(&ServiceError::InstanceExists)
            .for_service(SUPERVISOR_INSTANCE_ID)
            .with_any_description();
        assert_eq!(*block[hash].status().unwrap_err(), expected_err);
    }
}

/// Checks that we can start several service instances in one request.
#[tokio::test]
async fn test_start_two_services_in_one_request() {
    let instance_name_1 = "inc";
    let instance_name_2 = "inc2";
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;

    assert!(!service_instance_exists(&testkit, instance_name_1));
    assert!(!service_instance_exists(&testkit, instance_name_2));

    let artifact = default_artifact();
    let deadline = testkit.height().next();

    let request = ConfigPropose::new(0, deadline)
        .start_service(artifact.clone(), instance_name_1, Vec::default())
        .start_service(artifact, instance_name_2, Vec::default());

    let api = testkit.api();
    let hash = start_service(&api, request).await;
    let block = testkit.create_block();
    block[hash].status().unwrap();

    assert!(service_instance_exists(&testkit, instance_name_1));
    assert!(service_instance_exists(&testkit, instance_name_2));
}

#[tokio::test]
async fn test_restart_node_and_start_service_instance() {
    let mut testkit = TestKitBuilder::validator()
        .with_logger()
        .with(Supervisor::decentralized())
        .with(JustFactory::new(IncService))
        .build();
    deploy_default(&mut testkit).await;

    // Stop the node.
    let stopped_testkit = testkit.stop();
    // ...and start it again with the same service factory.
    let mut testkit = stopped_testkit.resume(available_services());

    // Ensure that the deployed artifact still exists.
    assert!(artifact_exists(&testkit, &default_artifact().name));

    let instance_name = "test_basics";
    let keypair = crypto::KeyPair::random();

    // Start IncService's instance now.
    let instance_id = start_service_instance(&mut testkit, instance_name).await;
    let api = testkit.api(); // update the API

    // Check that the service instance actually works.
    {
        assert_count_is_not_set(&api, instance_name).await;

        api.send(keypair.inc(instance_id, 0)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 1).await;

        api.send(keypair.inc(instance_id, 1)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 2).await;
    }

    // Restart the node again.
    let stopped_testkit = testkit.stop();
    let mut testkit = stopped_testkit.resume(available_services());
    let api = testkit.api();

    // Ensure that the started service instance still exists.
    assert!(service_instance_exists(&testkit, instance_name));

    // Check that the service instance still works.
    {
        assert_count(&api, instance_name, 2).await;
        api.send(keypair.inc(instance_id, 2)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 3).await;
    }
}

#[tokio::test]
async fn test_restart_node_during_artifact_deployment_with_two_validators() {
    let mut testkit = testkit_with_inc_service_and_two_validators();
    let artifact = default_artifact();
    let api = testkit.api();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact.clone(), DEPLOY_HEIGHT.next());
    let deploy_confirmation_0 = deploy_confirmation(&testkit, &request_deploy, ValidatorId(0));
    let deploy_confirmation_1 = deploy_confirmation(&testkit, &request_deploy, ValidatorId(1));

    // Send an artifact deploy request from this validator.
    deploy_artifact(&api, request_deploy.clone()).await;
    // Emulate an artifact deploy request from the second validator.
    deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(1));

    let block = testkit.create_block();
    block.iter().for_each(|tx| tx.status().unwrap());

    // Confirmation is ready.
    assert!(testkit.is_tx_in_pool(&deploy_confirmation_0.object_hash()));
    testkit.create_block();

    // Restart the node again after the first block was created.
    let testkit = testkit.stop();
    let mut testkit = testkit.resume(available_services());

    // Emulate a confirmation from the second validator.
    testkit.add_tx(deploy_confirmation_1.clone());
    assert!(testkit.is_tx_in_pool(&deploy_confirmation_1.object_hash()));
    testkit.create_block();
    // Both confirmations are gone now.
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_0.object_hash()));
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_1.object_hash()));

    assert!(artifact_exists(&testkit, &artifact.name));
}

/// This test emulates a normal workflow with two validators.
#[tokio::test]
async fn test_two_validators() {
    let mut testkit = testkit_with_inc_service_and_two_validators();
    let artifact = default_artifact();
    let api = testkit.api();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact.clone(), DEPLOY_HEIGHT);
    let deploy_confirmation_0 = deploy_confirmation(&testkit, &request_deploy, ValidatorId(0));
    let deploy_confirmation_1 = deploy_confirmation(&testkit, &request_deploy, ValidatorId(1));

    // Send an artifact deploy request from this validator.
    deploy_artifact(&api, request_deploy.clone()).await;
    // Emulate an artifact deploy request from the second validator.
    deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(1));
    let block = testkit.create_block();
    block.iter().for_each(|tx| tx.status().unwrap());

    // Emulate a confirmation from the second validator.
    testkit.add_tx(deploy_confirmation_1.clone());

    // Both confirmations are ready.
    assert!(testkit.is_tx_in_pool(&deploy_confirmation_0.object_hash()));
    assert!(testkit.is_tx_in_pool(&deploy_confirmation_1.object_hash()));
    testkit.create_block();

    // Both confirmations are gone now.
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_0.object_hash()));
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_1.object_hash()));

    let api = testkit.api(); // update the API
    assert!(artifact_exists(&testkit, &artifact.name));
    let instance_name = "inc";

    // Start the service now
    {
        assert!(!service_instance_exists(&testkit, instance_name));
        // Add two heights to the deadline: one for block with config proposal and one for confirmation.
        let deadline = DEPLOY_HEIGHT.next();
        let request_start = start_service_request(default_artifact(), instance_name, deadline);
        let propose_hash = request_start.object_hash();

        // Send a start instance request from this node.
        start_service(&api, request_start).await;
        testkit.create_block();

        // Confirm changes.
        let signed_txs = build_confirmation_transactions(&testkit, propose_hash, ValidatorId(0));
        testkit
            .create_block_with_transactions(signed_txs)
            .transactions[0]
            .status()
            .expect("Transaction with confirmations discarded.");

        assert!(service_instance_exists(&testkit, instance_name));
    }

    let api = testkit.api(); // Update the API
    let instance_id = find_instance_id(&testkit, instance_name);
    // Basic check that service works.
    {
        assert_count_is_not_set(&api, instance_name).await;
        let keypair = crypto::KeyPair::random();
        api.send(keypair.inc(instance_id, 0)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 1).await;

        api.send(keypair.inc(instance_id, 1)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 2).await;
    }
}

/// This test emulates the case when the second validator doesn't send DeployRequest.
#[tokio::test]
async fn test_multiple_validators_no_confirmation() {
    let mut testkit = testkit_with_inc_service_and_two_validators();

    let artifact = default_artifact();
    let api = testkit.api();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact.clone(), DEPLOY_HEIGHT);
    let deploy_confirmation_0 = deploy_confirmation(&testkit, &request_deploy, ValidatorId(0));

    // Send an artifact deploy request from this validator.
    deploy_artifact(&api, request_deploy).await;
    // Deliberately not sending an artifact deploy request from the second validator.
    let block = testkit.create_block();
    block.iter().for_each(|tx| tx.status().unwrap());

    // Deliberately not sending a confirmation from the second validator.

    // No confirmation was generated ...
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation_0.object_hash()));
    testkit.create_block();
    // ...and no artifact was deployed.
    assert!(!artifact_exists(&testkit, &artifact.name));
}

// Test that auditor can't send any requests.
#[tokio::test]
async fn test_auditor_cant_send_requests() {
    let mut testkit = testkit_with_inc_service_auditor_validator();

    let artifact = default_artifact();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact, DEPLOY_HEIGHT);

    // Try to send an artifact deploy request from the auditor.
    let deploy_request_from_auditor = {
        // Manually signing the tx with auditor's keypair.
        let confirmation = DeployResult::ok(request_deploy.clone());
        testkit
            .us()
            .service_keypair()
            .report_deploy_result(SUPERVISOR_INSTANCE_ID, confirmation)
    };
    testkit.add_tx(deploy_request_from_auditor.clone());

    // Emulate an artifact deploy request from the second validator.
    let deploy_artifact_validator_tx_hash =
        deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(0));

    let block = testkit.create_block();
    for tx in &block {
        if tx.message().object_hash() == deploy_artifact_validator_tx_hash {
            // Emulated request executed as fine...
            tx.status().unwrap();
        } else if *tx.message() == deploy_request_from_auditor {
            // ... but the auditor's request is failed as expected.
            let expected_err = ErrorMatch::from_fail(&CommonError::UnauthorizedCaller)
                .for_service(SUPERVISOR_INSTANCE_ID);
            assert_eq!(*tx.status().unwrap_err(), expected_err);
        } else {
            panic!("Unexpected transaction in block: {:?}", tx);
        }
    }
}

/// This test emulates a normal workflow with a validator and an auditor.
#[tokio::test]
async fn test_auditor_normal_workflow() {
    let mut testkit = testkit_with_inc_service_auditor_validator();
    let artifact = default_artifact();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact.clone(), DEPLOY_HEIGHT);
    let deploy_confirmation = deploy_confirmation(&testkit, &request_deploy, ValidatorId(0));

    // Emulate an artifact deploy request from the validator.
    deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(0));
    let block = testkit.create_block();
    block.iter().for_each(|tx| tx.status().unwrap());

    // Emulate a confirmation from the validator.
    testkit.add_tx(deploy_confirmation.clone());
    // The confirmation is in the pool.
    assert!(testkit.is_tx_in_pool(&deploy_confirmation.object_hash()));
    testkit.create_block();

    // The confirmation is gone.
    assert!(!testkit.is_tx_in_pool(&deploy_confirmation.object_hash()));
    // The artifact is deployed.
    assert!(artifact_exists(&testkit, &artifact.name));

    let instance_name = "inc";
    // Start the service now
    {
        assert!(!service_instance_exists(&testkit, instance_name));
        let deadline = DEPLOY_HEIGHT;
        let request_start = start_service_request(default_artifact(), instance_name, deadline);

        // Emulate a start instance request from the validator.
        start_service_manually(&mut testkit, &request_start, ValidatorId(0));
        let block = testkit.create_block();
        block.iter().for_each(|tx| tx.status().unwrap());
        assert!(service_instance_exists(&testkit, instance_name));
    }

    let api = testkit.api(); // Update the API
    let instance_id = find_instance_id(&testkit, instance_name);

    // Check that service still works.
    {
        assert_count_is_not_set(&api, instance_name).await;
        let keypair = crypto::KeyPair::random();
        api.send(keypair.inc(instance_id, 0)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 1).await;
        api.send(keypair.inc(instance_id, 1)).await;
        testkit.create_block();
        assert_count(&api, instance_name, 2).await;
    }
}

/// This test emulates a deploy confirmation with 12 validators.
/// Here we send confirmations by every validator and expect deploy to start.
#[tokio::test]
async fn test_multiple_validators_deploy_confirm() {
    let validators_count = 12;
    let mut testkit = testkit_with_inc_service_and_n_validators(validators_count);
    let artifact = default_artifact();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact.clone(), DEPLOY_HEIGHT);

    // Send deploy requests by every validator.
    for i in 0..validators_count {
        deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(i));
    }

    // Verify that every transaction succeeded (even for confirmations
    // sent after the quorum was achieved).
    let block = testkit.create_block();
    assert_eq!(block.len(), validators_count as usize);
    block.iter().for_each(|tx| tx.status().unwrap());

    // Send deploy confirmations by every validator.
    let deploy_confirmations: Vec<Verified<AnyTx>> = (0..validators_count)
        .map(|i| deploy_confirmation(&testkit, &request_deploy, ValidatorId(i)))
        .collect();

    testkit.create_block_with_transactions(deploy_confirmations);

    // Check that artifact is deployed now.
    assert!(artifact_exists(&testkit, &artifact.name));
}

/// This test emulates a deploy confirmation with 12 validators.
/// Here we send confirmations by the byzantine majority (2/3+1) validators
/// and expect deploy to start.
#[test]
fn test_multiple_validators_deploy_confirm_byzantine_majority() {
    let validators_count = 12;
    let byzantine_majority = (validators_count * 2 / 3) + 1;
    let mut testkit = testkit_with_inc_service_and_n_validators(validators_count);
    let artifact = default_artifact();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact.clone(), DEPLOY_HEIGHT);

    // Send deploy requests by byzantine majority of validators.
    for i in 0..byzantine_majority {
        deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(i));
    }
    let block = testkit.create_block();
    assert_eq!(block.len(), byzantine_majority as usize);
    block.iter().for_each(|tx| tx.status().unwrap());

    // Send deploy confirmations by every validator.
    let deploy_confirmations: Vec<Verified<AnyTx>> = (0..validators_count)
        .map(|i| deploy_confirmation(&testkit, &request_deploy, ValidatorId(i)))
        .collect();
    testkit.create_block_with_transactions(deploy_confirmations);

    // Check that artifact is deployed now.
    assert!(artifact_exists(&testkit, &artifact.name));
}

/// This test emulates a deploy confirmation with 12 validators.
/// Here we send confirmations by the byzantine minority (2/3) validators
/// and expect deploy to not start.
#[test]
fn test_multiple_validators_deploy_confirm_byzantine_minority() {
    let validators_count = 12;
    let byzantine_minority = validators_count * 2 / 3;
    let mut testkit = testkit_with_inc_service_and_n_validators(validators_count);
    let artifact = default_artifact();
    assert!(!artifact_exists(&testkit, &artifact.name));

    let request_deploy = deploy_request(artifact, DEPLOY_HEIGHT);

    // Send deploy requests by byzantine majority of validators.
    for i in 0..byzantine_minority {
        deploy_artifact_manually(&mut testkit, &request_deploy, ValidatorId(i));
    }
    let block = testkit.create_block();
    assert_eq!(block.len(), byzantine_minority as usize);
    block.iter().for_each(|tx| tx.status().unwrap());

    // Try to send confirmation. It should fail, since deploy was not approved
    // and thus not registered.
    let confirmation = deploy_confirmation(&testkit, &request_deploy, ValidatorId(0));
    let block = testkit.create_block_with_transaction(confirmation);
    let expected_err = ErrorMatch::from_fail(&ArtifactError::DeployRequestNotRegistered)
        .with_description_containing("Deploy of artifact `0:inc:1.0.0` is not registered")
        .for_service(SUPERVISOR_INSTANCE_ID);
    assert_eq!(*block[0].status().unwrap_err(), expected_err);
}

/// Checks that service IDs are assigned sequentially starting from the
/// ID next to max builtin ID.
#[tokio::test]
async fn test_id_assignment() {
    let max_builtin_id = SUPERVISOR_INSTANCE_ID;

    // Deploy inc service & start two instances.
    let instance_name_1 = "inc";
    let instance_name_2 = "inc2";
    let mut testkit = testkit_with_inc_service();
    deploy_default(&mut testkit).await;

    let artifact = default_artifact();
    let deadline = testkit.height().next();

    let request = ConfigPropose::new(0, deadline)
        .start_service(artifact.clone(), instance_name_1, Vec::default())
        .start_service(artifact, instance_name_2, Vec::default());

    let api = testkit.api();
    start_service(&api, request).await;
    testkit.create_block();

    // Check that new instances have IDs 1 and 2.
    assert_eq!(
        find_instance_id(&testkit, instance_name_1),
        max_builtin_id + 1
    );
    assert_eq!(
        find_instance_id(&testkit, instance_name_2),
        max_builtin_id + 2
    );
}

/// Checks that if builtin IDs space is sparse (here we have `Supervisor` with ID 0 and
/// `IncService` with ID 100), the ID for the new service will be next to the max
/// builtin ID (101 in our case).
#[tokio::test]
async fn test_id_assignment_sparse() {
    let max_builtin_id = 100;
    let inc_service = Spec::new(IncService).with_instance(max_builtin_id, "inc", ());

    // Create testkit with builtin instance with ID 100.
    let mut testkit = TestKitBuilder::validator()
        .with_logger()
        .with(Supervisor::decentralized())
        .with(inc_service)
        .build();

    let artifact = default_artifact();
    let deadline = testkit.height().next();

    let instance_name = "inc2";
    let request =
        ConfigPropose::new(0, deadline).start_service(artifact, instance_name, Vec::default());

    let api = testkit.api();
    start_service(&api, request).await;
    testkit.create_block();

    // Check that new instance has ID 101.
    assert_eq!(
        find_instance_id(&testkit, instance_name),
        max_builtin_id + 1
    );
}