solvastro 0.1.1

A Rust blind plate solver
Documentation
#![feature(let_else)]

mod config;

use std::path::PathBuf;
use std::time::Instant;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use log::{debug, error, info, trace, warn};

use aws_sdk_dynamodb::Client as DynamodbClient;
use aws_sdk_dynamodb::model::AttributeValue;
use aws_sdk_sns::Client as SnsClient;
use aws_sdk_sqs::Client as SqsClient;
use serde::{Deserialize,Serialize};
use structopt::StructOpt;
use unique_id::Generator;

use solvastro::process_query::process_query;
use solvastro::query::Query;
use solvastro::settings::Settings;
use solvastro::star_index::StarIndex;
use solvastro::verified_solution::VerifiedSolution;
use crate::config::AppConfig;

const MAX_CONSEC_SQS_RECV_FAILS: i32 = 5;

#[derive(Debug, StructOpt)]
struct CliArgs {
    #[structopt(parse(from_os_str))]
    star_index: PathBuf,
}

#[derive(Deserialize, Debug, Serialize)]
struct SnsSolveRequestMessage {
    #[serde(alias="Message")]
    message: String
}

#[derive(Deserialize, Debug, Serialize)]
struct SolveRequest {
    id: String,
    user_id: String,
    connection_id: Option<String>,
    query: Query,
}

#[derive(Deserialize, Debug, Serialize)]
struct SolveResponse {
    id: String,
    user_id: String,
    connection_id: Option<String>,
    response: Option<VerifiedSolution>,
}

impl Default for SnsSolveRequestMessage {
    fn default() -> Self {
        SnsSolveRequestMessage {
            message: "".to_string()
        }
    }
}

#[tokio::main]
async fn main() {
    let start = Instant::now();
    pretty_env_logger::init();
    //env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();

    let cfg = AppConfig::new();
    debug!("cfg: {:?}", &cfg);

    let aws_config = aws_config::load_from_env().await;
    let dynamo_db_client = DynamodbClient::new(&aws_config);
    let sqs_client = SqsClient::new(&aws_config);
    let sns_client = SnsClient::new(&aws_config);

    // Create a unique ID for this instance
    let instance_id = unique_id::string::StringGenerator::default().next_id();

    // track whether termination has been requested
    let running = Arc::new(AtomicBool::new(true));
    let r = running.clone();
    ctrlc::set_handler(move || {
        r.store(false, Ordering::SeqCst);
    }).expect("Error setting termination / CTRL-C handler");

    // register this instance in ddb
    // info!("Registering (id {})", &instance_id);
    // let result = dynamo_db_client.put_item()
    //     .table_name(&cfg.ddb_table_name)
    //     .item("id", AttributeValue::S(String::from(&instance_id)))
    //     .send()
    //     .await;
    // if result.is_err() {
    //     error!("Could not register instance ({:?})", &result);
    //     return;
    // }
    // info!("Registration complete");

    let cli_args = CliArgs::from_args();
    let solver_config = Settings::new();
    let star_index = StarIndex::read_from_file(&cli_args.star_index);

    if solver_config.is_ok() && star_index.is_ok() {
        let duration_startup = start.elapsed();
        debug!("Settings: {:?}", &solver_config);
        info!("Startup time: {:?}", duration_startup);

        let star_index = star_index.expect("Could not load StarIndex");
        let solver_config = solver_config.expect("could not load settings");

        let mut consec_fail_count = 0;
        while running.load(Ordering::SeqCst) {
            let rcv_message = sqs_client.receive_message()
                .queue_url(&cfg.sqs_queue_solver_request)
                .visibility_timeout(15)
                .wait_time_seconds(20)
                .send().await;

            debug!("rcv_message: {:?}", &rcv_message);

            if let Ok(rcv_message_output) = rcv_message {
                consec_fail_count = 0;

                for message in rcv_message_output.messages.unwrap_or_default() {
                    trace!("SQS message: {:#?}", &message);
                    info!("Received SQS msg {:?}", &message.message_id);
                    let start_message = Instant::now();
                    if let Some(body_str) = &message.body {

                        if let Ok(sns_msg) = serde_json::from_str(body_str) {
                            let sns_msg: SnsSolveRequestMessage = sns_msg;
                            trace!("SNS message: {:#?}", &sns_msg);

                            if let Ok(msg) = serde_json::from_str(&sns_msg.message) {
                                let msg: SolveRequest = msg;
                                let parsing_time = start_message.elapsed();
                                info!("Solve Request {:?} parsed for user_id {:?} (img: {:?}, parse time: {:?})", &msg.id, &msg.user_id, &msg.query.filename, parsing_time);

                                let start_processing = Instant::now();
                                let solution_opt = process_query(&star_index, &msg.query, &solver_config);
                                let solving_time = start_processing.elapsed();

                                let solve_response: SolveResponse = if let Some(solution) = solution_opt {
                                    info!("Request {:?} solved in {:?}, solution logodds = {}", &msg.id, &solving_time, &solution.logodds);
                                    SolveResponse {
                                        id: msg.id,
                                        user_id: msg.user_id,
                                        connection_id: msg.connection_id,
                                        response: Some(solution)
                                    }
                                } else {
                                    warn!("Request {:?} solving complete in {:?}. No solution found", &msg.id, &solving_time);
                                    SolveResponse {
                                        id: msg.id,
                                        user_id: msg.user_id,
                                        connection_id: msg.connection_id,
                                        response: None
                                    }
                                };

                                if let Ok(solve_response_str) = serde_json::to_string(&solve_response) {
                                    info!("Response: {:?}", &solve_response_str);

                                    let pub_result = sns_client.publish()
                                        .topic_arn(&cfg.sns_topic_solver_response)
                                        .message(solve_response_str)
                                        .send().await;

                                    if let Ok(pub_output) = pub_result {
                                        let response_time = start_message.elapsed();
                                        info!("Result published to SNS (Msg Id: {:?}), response time {:?}", &pub_output.message_id.unwrap(), response_time);

                                        let delete_result = sqs_client.delete_message()
                                            .queue_url(&cfg.sqs_queue_solver_request)
                                            .receipt_handle(message.receipt_handle.unwrap())
                                            .send().await;

                                        if let Ok(_delete_output) = delete_result {
                                            let processing_time = start_message.elapsed();
                                            info!("Message {:?} deleted from SQS. Total processing time: {:?}", message.message_id, processing_time);

                                        } else {
                                            error!("Cold not delete message from SQS (Msg id: {:?}), {:?}", &message.message_id, &delete_result);
                                        }
                                    } else {
                                        error!("Cold not publish response to SNS ({:?})", &pub_result);
                                    }
                                } else {
                                    error!("Could not serialise response into JSON: {:?}", &solve_response);
                                }
                            } else {
                                error!("could not parse SolveRequest from SNS message body JSON");
                            }
                        } else {
                            error!("Could not parse encapsulated SNS message JSON from SQS message")
                        }
                    } else {
                        warn!("SQS Message received with no msg body.");
                    }
                }

            } else {
                warn!("SQS ReceiveMessage failed: {:?}", rcv_message);
                consec_fail_count += 1;
                if consec_fail_count > MAX_CONSEC_SQS_RECV_FAILS {
                    error!("Hit max consecutive SQS RecieveMessage limit of {:?} times. Exiting.", consec_fail_count);
                    running.clone().store(false, Ordering::SeqCst);
                }
            }
        }
    }

    // SIGTERM / SIGINT received. Deregister and exit
    // info!("Deregistering (id {})", &instance_id);
    // let result = dynamo_db_client.delete_item()
    //     .table_name(&cfg.ddb_table_name)
    //     .key("id", AttributeValue::S(String::from(&instance_id)))
    //     .send()
    //     .await;
    // if result.is_err() {
    //     error!("Could not deregister instance ({:?})", &result);
    // } else {
    //     info!("Deregistration complete. exiting");
    // }
}