use crate::bos::template::http_client::v2::types::BosSessionTemplate;
use crate::cfs::configuration::http_client::v3::types::{
cfs_configuration_request::CfsConfigurationRequest,
cfs_configuration_response::CfsConfigurationResponse,
};
use crate::hsm::group::http_client::{create_new_group, delete_group};
use crate::hsm::group::types::Group;
use crate::ims::image::{
http_client::{
patch,
types::{Image, ImsImageRecord2Update, Link},
},
utils::get_by_name,
utils::get_fuzzy,
};
use crate::ims::s3_client::BAR_FORMAT;
use crate::{bos, cfs, ims};
use chrono::Local;
use humansize::DECIMAL;
use indicatif::{ProgressBar, ProgressStyle};
use md5::Digest;
use serde::{Deserialize, Serialize};
use std::fs;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::path::PathBuf;
use crate::error::Error;
#[derive(Serialize, Deserialize, Debug, Clone)]
struct Artifact {
pub link: Link,
pub md5: String,
#[serde(rename = "type")]
pub r#type: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
struct ImageManifest {
pub created: String,
#[serde(default = "default_version")]
pub version: String,
pub artifacts: Vec<Artifact>,
}
fn default_version() -> String {
"1.0".to_string()
}
pub async fn exec(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
bos_file: Option<&String>,
cfs_file: Option<&String>,
hsm_file: Option<&String>,
ims_file: Option<&String>,
image_dir: Option<&String>,
overwrite_group: bool,
overwrite_configuration: bool,
overwrite_image: bool,
overwrite_template: bool,
) -> Result<(), Error> {
if !PathBuf::from(&bos_file.unwrap()).exists() {
return Err(Error::Message(format!(
"Error, file {} does not exist or cannot be open.",
&bos_file.unwrap()
)));
}
if !PathBuf::from(&cfs_file.unwrap()).exists() {
return Err(Error::Message(format!(
"Error, file {} does not exist or cannot be open.",
&cfs_file.unwrap()
)));
}
if !PathBuf::from(&ims_file.unwrap()).exists() {
return Err(Error::Message(format!(
"Error, file {} does not exist or cannot be open.",
&ims_file.unwrap()
)));
}
if !PathBuf::from(&hsm_file.unwrap()).exists() {
return Err(Error::Message(format!(
"Error, file {} does not exist or cannot be open.",
&hsm_file.unwrap()
)));
}
let current_timestamp = Local::now().format("%Y-%m-%d %H:%M:%S");
let mut ims_image_manifest = ImageManifest {
created: current_timestamp.to_string(),
version: "1.0".to_string(),
artifacts: vec![],
};
let backup_ims_file = ims_file.unwrap().to_string();
let backup_cfs_file = cfs_file.unwrap().to_string();
let backup_bos_file = bos_file.unwrap().to_string();
let backup_hsm_file = hsm_file.unwrap().to_string();
let ims_image_name: String = get_image_name_from_ims_file(&backup_ims_file);
println!(" Image name: {}", ims_image_name);
println!(
"\tinitrd file: {}",
image_dir.unwrap().to_string() + "/initrd"
);
println!(
"\tkernel file: {}",
image_dir.unwrap().to_string() + "/kernel"
);
println!(
"\trootfs file: {}",
image_dir.unwrap().to_string() + "/rootfs"
);
let vec_backup_image_files = vec![
image_dir.unwrap().to_string() + "/initrd",
image_dir.unwrap().to_string() + "/kernel",
image_dir.unwrap().to_string() + "/rootfs",
];
for file in &vec_backup_image_files {
if !PathBuf::from(&file).exists() {
return Err(Error::Message(format!(
"Error, file {} does not exist or cannot be open.",
&file
)));
}
}
println!("Calculating image artifact checksum...");
calculate_image_checksums(&mut ims_image_manifest, &vec_backup_image_files);
println!("\n\nRegistering image with IMS...");
let ims_image_id_rslt = ims_register_image(
shasta_token,
shasta_base_url,
shasta_root_cert,
&ims_image_name,
overwrite_image,
)
.await;
let ims_image_id: String = match ims_image_id_rslt {
Ok(value) => value,
Err(e) => {
return Err(Error::Message(format!("{}", e)));
}
};
println!("Ok, IMS image ID: {}", &ims_image_id);
println!("\nUploading image artifacts to s3...");
s3_upload_image_artifacts(
shasta_token,
shasta_base_url,
shasta_root_cert,
&ims_image_id,
&mut ims_image_manifest,
&vec_backup_image_files,
)
.await;
println!("\nUpdating IMS image record with the new location in s3...");
log::debug!("Updating image record with location of the newly generated manifest.json data");
ims_update_image_add_manifest(
shasta_token,
shasta_base_url,
shasta_root_cert,
&ims_image_name,
&ims_image_id,
)
.await;
println!("Ok");
println!("\nCreating HSM group...");
create_hsm_group_from_file(
shasta_token,
shasta_base_url,
shasta_root_cert,
&backup_hsm_file,
overwrite_group,
)
.await;
println!("Ok");
println!("\nUploading CFS configuration...");
create_cfs_config(
shasta_token,
shasta_base_url,
shasta_root_cert,
&backup_cfs_file,
overwrite_configuration,
)
.await;
println!("\nUploading BOS sessiontemplate...");
create_bos_sessiontemplate(
shasta_token,
shasta_base_url,
shasta_root_cert,
&backup_bos_file,
&ims_image_id,
overwrite_template,
)
.await;
println!("\nDone, the image bundle, HSM group, CFS configuration and BOS sessiontemplate have been restored.");
Ok(())
}
async fn create_bos_sessiontemplate(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
bos_file: &String,
ims_image_id: &String,
overwrite: bool,
) {
let file_content = File::open(bos_file)
.expect(&format!("Unable to read BOS JSON file '{}'", bos_file));
let bos_json: BosSessionTemplate =
serde_json::from_reader(BufReader::new(file_content))
.expect("BOS JSON file does not have correct format.");
let bos_sessiontemplate_name = bos_json.name.unwrap();
log::debug!("BOS sessiontemplate name: {}", &bos_sessiontemplate_name);
let vector = bos::template::http_client::v2::get(
shasta_token,
shasta_base_url,
shasta_root_cert,
Some(&bos_sessiontemplate_name),
)
.await
.unwrap_or_else(|error| {
eprint!(
"Error: unable to query CSM to get list of BOS sessiontemplates. Error returned: {}",
error
);
std::process::exit(1);
});
log::debug!("BOS sessiontemplate filtered: {:#?}", vector);
if !vector.is_empty() {
if !overwrite {
println!("Looks like you do not want to continue, bailing out.");
std::process::exit(2)
} else {
match bos::template::http_client::v2::delete(
shasta_token,
shasta_base_url,
shasta_root_cert,
&bos_sessiontemplate_name,
)
.await
{
Ok(_) => log::debug!(
"Ok BOS session template {}, deleted.",
&bos_sessiontemplate_name
),
Result::Err(err1) => panic!(
"Error, unable to delete BOS session template. Cannot continue. Error: {}",
err1
),
};
}
}
let file_content = File::open(bos_file)
.expect(&format!("Unable to read BOS JSON file '{}'", bos_file));
let mut bos_sessiontemplate: BosSessionTemplate =
serde_json::from_reader(BufReader::new(file_content))
.expect("BOS JSON file does not have correct format.");
let path_modified =
format!("s3://boot-images/{}/manifest.json", ims_image_id);
bos_sessiontemplate
.boot_sets
.as_mut()
.unwrap()
.get_mut("compute")
.unwrap()
.path = Some(path_modified);
log::debug!("BOS sessiontemplate loaded:\n{:#?}", bos_sessiontemplate);
log::debug!("BOS sessiontemplate modified:\n{:#?}", &bos_sessiontemplate);
match bos::template::http_client::v2::put(
shasta_token,
shasta_base_url,
shasta_root_cert,
&bos_sessiontemplate,
&bos_sessiontemplate_name,
)
.await
{
Ok(_result) => println!(
"Ok, BOS session template {} created successfully.",
&bos_sessiontemplate_name
),
Err(e1) => panic!(
"Error, unable to create BOS sesiontemplate. Error returned by CSM API: {}",
e1
),
}
}
async fn create_cfs_config(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
cfs_file: &String,
overwrite: bool,
) {
let file_content = File::open(cfs_file)
.expect(&format!("Unable to read CFS JSON file '{}'", cfs_file));
let cfs_configuration: CfsConfigurationResponse =
serde_json::from_reader(BufReader::new(file_content))
.expect("CFS JSON file does not have correct format.");
let cfs_config_name = cfs_configuration.name;
let cfs_config_vec = cfs::configuration::http_client::v3::get(
shasta_token,
shasta_base_url,
shasta_root_cert,
Some(&cfs_config_name),
)
.await
.unwrap_or_else(|error| {
eprint!(
"Error: Unable to fetch CFS configuration. Error returned by CSM API: {}",
error
);
std::process::exit(1);
});
if !cfs_config_vec.is_empty() {
if !overwrite {
println!("Looks like you do not want to continue, bailing out.");
std::process::exit(2)
}
match cfs::configuration::http_client::v3::delete(
shasta_token,
shasta_base_url,
shasta_root_cert,
cfs_config_name.as_str(),
)
.await
{
Ok(_) => {
log::debug!("Ok CFS configuration {}, deleted.", cfs_config_name)
}
Result::Err(error) => panic!(
"Error, unable to delete configuration. Cannot continue. Error: {}",
error
),
};
}
let file_content = File::open(cfs_file)
.expect(&format!("Unable to read CFS JSON file '{}'", cfs_file));
let cfs_configuration: CfsConfigurationRequest =
serde_json::from_reader(BufReader::new(file_content))
.expect("CFS JSON file does not have correct format.");
log::debug!("CFS config:\n{:#?}", &cfs_configuration);
match cfs::configuration::http_client::v3::put(
shasta_token,
shasta_base_url,
shasta_root_cert,
&cfs_configuration,
cfs_config_name.as_str(),
)
.await
{
Ok(result) => {
log::debug!("Ok, result: {:#?}", result);
println!(
"Ok, CFS configuration {} created successfully.",
&cfs_config_name
);
}
Err(e1) => panic!(
"Error, unable to create CFS configuration. Error returned by CSM API: {}",
e1
),
}
}
async fn ims_update_image_add_manifest(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
ims_image_name: &String,
ims_image_id: &String,
) {
match get_fuzzy(shasta_token,
shasta_base_url,
shasta_root_cert,
&["".to_string()], Some(ims_image_name.clone().as_str()),
None).await {
Ok(_vector) => {
if _vector.is_empty() {
panic!("Error: there are no images stored with id {} in IMS. Unable to update the image manifest", &ims_image_id);
}
},
Err(error) => panic!("Error: Unable to determine if there are other images in IMS with the name {}. Error code: {}", &ims_image_name, &error),
};
let _ims_record = ims::image::http_client::types::Image {
name: ims_image_name.clone().to_string(),
id: Some(ims_image_id.clone().to_string()),
created: None,
arch: None,
link: Some(ims::image::http_client::types::Link {
etag: None,
path: format!(
"s3://boot-images/{}/manifest.json",
&ims_image_id.to_string()
),
r#type: "s3".to_string(),
}),
};
let ims_link = Link {
etag: None,
path: format!(
"s3://boot-images/{}/manifest.json",
&ims_image_id.to_string()
),
r#type: "s3".to_string(),
};
let rec = ImsImageRecord2Update {
link: ims_link,
arch: None,
};
match patch(
shasta_token,
shasta_base_url,
shasta_root_cert,
&ims_image_id.to_string(),
&rec,
)
.await
{
Ok(_returned) => log::debug!("Returned json: {}", _returned),
Err(e) => panic!(
"Error, unable to modify the record of the image. Err msg: {}",
e
),
};
}
async fn s3_upload_image_artifacts(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
ims_image_id: &String,
ims_image_manifest: &mut ImageManifest,
vec_image_files: &Vec<String>,
) {
let bucket_name = "boot-images";
let object_path = ims_image_id;
let sts_value = match ims::s3_client::s3_auth(
shasta_token,
shasta_base_url,
shasta_root_cert,
)
.await
{
Ok(sts_value) => {
log::debug!("STS token:\n{:#?}", sts_value);
sts_value
}
Err(error) => panic!(
"Unable to authenticate with s3 when uploading images. Error: {}",
error
),
};
for file in vec_image_files {
let filename = Path::new(file).file_name().unwrap();
let file_size = match fs::metadata(file) {
Ok(_file_metadata) => {
let res: String = humansize::format_size(_file_metadata.len(), DECIMAL);
res
}
Err(e) => {
eprintln!(
"Unable to fetch file metadata info, faking the value. Error: {}",
e
);
"-1".to_string()
}
};
let full_object_path =
format!("{}/{}", &object_path, &filename.to_string_lossy());
println!(
"File {:?} ({}) to s3://{}/{}.",
&file, &file_size, &bucket_name, &full_object_path
);
let etag: String;
if fs::metadata(file).unwrap().len() > 1024 * 1024 * 5 {
etag = match ims::s3_client::s3_multipart_upload_object(
&sts_value,
&full_object_path,
bucket_name,
file,
)
.await
{
Ok(result) => {
log::debug!("Artifact uploaded successfully.");
result
}
Err(error) => panic!("Unable to upload file to s3. Error {}", error),
};
} else {
etag = match ims::s3_client::s3_upload_object(
&sts_value,
&full_object_path,
bucket_name,
file,
)
.await
{
Ok(result) => {
println!("Ok");
result
}
Err(error) => panic!("Unable to upload file to s3. Error {}", error),
};
}
if file.contains("kernel") {
for artifact in ims_image_manifest.artifacts.iter_mut() {
if !etag.is_empty() {
artifact.link.etag = Some(etag.clone());
}
if artifact.r#type.contains("kernel") {
artifact.link.path = "s3://".to_string()
+ bucket_name
+ "/"
+ &object_path.to_string()
+ "/kernel";
break;
}
}
} else if file.contains("rootfs") {
for artifact in ims_image_manifest.artifacts.iter_mut() {
if !etag.is_empty() {
artifact.link.etag = Some(etag.clone());
}
if artifact.r#type.contains("rootfs") {
artifact.link.path = "s3://".to_string()
+ bucket_name
+ "/"
+ &object_path.to_string()
+ "/rootfs";
break;
}
}
} else if file.contains("initrd") {
for artifact in ims_image_manifest.artifacts.iter_mut() {
if !etag.is_empty() {
artifact.link.etag = Some(etag.clone());
}
if artifact.r#type.contains("initrd") {
artifact.link.path = "s3://".to_string()
+ bucket_name
+ "/"
+ &object_path.to_string()
+ "/initrd";
break;
}
}
}
}
log::debug!("Writing the new manifest.json file with the correct new ID");
let new_manifest_file_name = String::from("new-manifest.json");
let new_manifest_file_path = Path::new(vec_image_files.first().unwrap())
.parent()
.unwrap()
.join(&new_manifest_file_name);
let new_manifest_file = File::create(&new_manifest_file_path)
.expect("new manifest.json file could not be created.");
serde_json::to_writer_pretty(&new_manifest_file, &ims_image_manifest)
.expect("Unable to write new manifest.json file");
log::debug!("Uploading the new manifest.json file");
let manifest_full_object_path = format!("{}/manifest.json", &object_path);
println!(
"File {:?} -> s3://{}/{}.",
&new_manifest_file_name, &bucket_name, &manifest_full_object_path
);
match ims::s3_client::s3_upload_object(
&sts_value,
&manifest_full_object_path,
bucket_name,
&new_manifest_file_path.to_owned().to_string_lossy(),
)
.await
{
Ok(_result) => {
println!("OK");
}
Err(error) => panic!("Unable to upload file to s3. Error {}", error),
};
}
fn file_md5sum(filename: PathBuf) -> Digest {
log::debug!("File {:?}...", &filename);
let f = File::open(filename).unwrap();
let len = f.metadata().unwrap().len();
let buf_len = len.min(100_000_000) as usize;
let mut buf = BufReader::with_capacity(buf_len, f);
let mut context = md5::Context::new();
let bar = ProgressBar::new(len);
bar.set_style(ProgressStyle::with_template(BAR_FORMAT).unwrap());
loop {
let part = buf.fill_buf().unwrap();
if part.is_empty() {
break;
}
context.consume(part);
let part_len = part.len();
buf.consume(part_len);
bar.inc(part_len as u64);
}
let digest = context.compute();
bar.finish();
digest
}
fn calculate_image_checksums(
image_manifest: &mut ImageManifest,
vec_backup_image_files: &Vec<String>,
) {
for file in vec_backup_image_files {
let file_size = match fs::metadata(file) {
Ok(_file_metadata) => {
let res: String = humansize::format_size(_file_metadata.len(), DECIMAL);
res
}
Err(e) => {
eprintln!(
"Unable to fetch file metadata info, faking the value. Error: {}",
e
);
"-1".to_string()
}
};
println!("File {:?} ({})...", &file, &file_size);
let artifact;
let mut fp = PathBuf::new();
fp.push(file);
let digest = file_md5sum(fp);
if file.contains("kernel") {
artifact = Artifact {
md5: format!("{:x}", digest),
link: Link {
path: "path".to_string(),
r#type: "s3".to_string(),
etag: None,
},
r#type: "application/vnd.cray.image.kernel".to_string(),
};
} else if file.contains("rootfs") {
artifact = Artifact {
md5: format!("{:x}", digest),
link: Link {
path: "path".to_string(),
r#type: "s3".to_string(),
etag: None,
},
r#type: "application/vnd.cray.image.rootfs.squashfs".to_string(),
};
} else {
artifact = Artifact {
md5: format!("{:x}", digest),
link: Link {
path: "path".to_string(),
r#type: "s3".to_string(),
etag: None,
},
r#type: "application/vnd.cray.image.initrd".to_string(),
};
}
image_manifest.artifacts.push(artifact);
}
}
async fn ims_register_image(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
ims_image_name: &String,
overwrite: bool,
) -> anyhow::Result<String> {
let ims_record = Image {
name: ims_image_name.clone().to_string(),
id: None,
created: None,
link: None,
arch: None,
};
let list_images_with_same_name = get_by_name(
shasta_token,
shasta_base_url,
shasta_root_cert,
&["".to_string()], Some(ims_image_name.clone().as_str()),
None,
)
.await?;
if !list_images_with_same_name.is_empty() {
if !overwrite {
println!("Looks like you do not want to continue, bailing out.");
std::process::exit(2)
}
}
let json_response = ims::image::http_client::post(
shasta_token,
shasta_base_url,
shasta_root_cert,
&ims_record,
)
.await?;
Ok(json_response["id"].to_string().replace('"', ""))
}
pub fn get_image_name_from_ims_file(ims_file: &String) -> String {
let ims_data = fs::read_to_string(PathBuf::from(&ims_file))
.expect("Unable to read IMS file file");
let ims_json: serde_json::Value = serde_json::from_str(&ims_data)
.expect("HSM JSON file does not have correct format.");
ims_json[0]["name"].clone().to_string().replace('"', "")
}
pub async fn create_hsm_group_from_file(
shasta_token: &str,
shasta_base_url: &str,
shasta_root_cert: &[u8],
hsm_file: &String,
overwrite: bool,
) {
let hsm_data = fs::read_to_string(PathBuf::from(hsm_file))
.expect("Unable to read HSM JSON file");
let group_vec: Vec<Group> = serde_json::from_str(&hsm_data)
.expect("HSM JSON file does not have correct format.");
for group in group_vec {
let group_members_opt =
group.members.clone().and_then(|members| members.ids);
match create_new_group(
shasta_token,
shasta_base_url,
shasta_root_cert,
&group.label,
&group_members_opt.unwrap_or_default(),
&group.exclusive_group.clone().unwrap(),
&group.description.clone().unwrap(),
&group.tags.clone().unwrap(),
)
.await
{
Ok(group) => {
println!(
"The HSM group {} has been created successfully.",
&group.label
);
}
Err(error) => {
if error.to_string().to_lowercase().contains("409") {
if overwrite {
println!("Looks like you want to continue");
match delete_group(
shasta_token,
shasta_base_url,
shasta_root_cert,
&group.label,
)
.await
{
Ok(_) => {
match crate::hsm::group::http_client::post(
shasta_token,
shasta_base_url,
shasta_root_cert,
group.clone(),
)
.await
{
Ok(_json) => {
println!(
"The HSM group {} has been created successfully.",
&group.label
);
}
Err(e) => {
log::error!("Error message {}", e);
panic!("Second error creating a new HSM group. Bailing out. Error returned: '{}'", e)
}
}
}
Err(e) => {
log::error!("Error message {}", e);
panic!(
"Error deleting the HSM group {}. Error returned: '{}'",
&group.label, e
)
}
}
} else {
println!("Not deleting the group, cannot continue the operation.");
std::process::exit(2);
}
} else if error.to_string().to_lowercase().contains("400") {
eprintln!("Unable to create the group, the API returned code 400. This usually means the HSM file is malformed, or has incorrect xnames for this site in it.");
std::process::exit(2);
}
}
};
}
}