use anyhow::{anyhow, bail, ensure};
use chrono::prelude::*;
use json::JsonValue;
use log::{error, info, warn};
use serde::{Deserialize, Serialize};
use serde_json::{self, Map, Value};
use std::{
fs::{self, File},
io::{BufRead, BufReader},
path::{Path, PathBuf},
};
use structopt::StructOpt;
use super::Command;
use crate::{
a_ok_or, app::AppSession, atry, env::require_var, errors::Result, project::Project, write_crlf,
};
#[derive(Debug)]
struct ZenodoService {
token: String,
}
impl ZenodoService {
fn new() -> Result<Self> {
let token = require_var("ZENODO_TOKEN")?;
Ok(ZenodoService { token })
}
fn make_blocking_client(&self) -> Result<reqwest::blocking::Client> {
use reqwest::header;
let mut headers = header::HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
header::HeaderValue::from_str(&format!("Bearer {}", self.token))?,
);
headers.insert(header::USER_AGENT, header::HeaderValue::from_str("cranko")?);
Ok(reqwest::blocking::Client::builder()
.default_headers(headers)
.build()?)
}
fn api_url(&self, rest: &str) -> String {
format!("https://zenodo.org/api/{}", rest)
}
}
#[derive(Debug)]
struct ZenodoWorkflow<'a> {
mode: ZenodoMode,
proj: &'a Project,
}
#[derive(Debug)]
enum ZenodoMode {
Development,
Release(ZenodoService),
}
impl<'a> ZenodoWorkflow<'a> {
fn new(proj: &'a Project, dev_mode: bool) -> Result<Self> {
let mode = if dev_mode {
info!(
"faking Zenodo workflow for project `{}` in development mode",
&proj.user_facing_name
);
if std::env::var_os("ZENODO_TOKEN").is_some() {
error!(
"the environment variable ZENODO_TOKEN is set during this development-mode job"
);
error!("... this could be a security risk given a malicious pull request");
error!("... if this is a CI job, fix your configuration to only provide the variable for trusted release jobs");
bail!("refusing to proceed");
}
ZenodoMode::Development
} else {
info!(
"starting Zenodo workflow for project `{}` in release mode",
&proj.user_facing_name
);
let svc = ZenodoService::new()?;
ZenodoMode::Release(svc)
};
Ok(ZenodoWorkflow { mode, proj })
}
fn preregister(&self, metadata_path: &PathBuf, rewrite_paths: &[PathBuf]) -> Result<()> {
let mut md = ZenodoMetadata::load_for_prereg(metadata_path)?;
md.metadata.insert(
"title".to_owned(),
Value::String(format!(
"{} {}",
&self.proj.user_facing_name, &self.proj.version
)),
);
md.metadata.insert(
"version".to_owned(),
Value::String(self.proj.version.to_string()),
);
let utc: DateTime<Utc> = Utc::now();
md.metadata.insert(
"publication_date".to_owned(),
Value::String(format!(
"{:>04}-{:>02}-{:>02}",
utc.year(),
utc.month(),
utc.day(),
)),
);
let new_concept = match &self.mode {
ZenodoMode::Development => {
md.concept_doi =
format!("xx.xxxx/dev-build.{}.concept", &self.proj.user_facing_name);
md.version_rec_id = format!(
"dev.{}.v{}",
&self.proj.user_facing_name, &self.proj.version
);
md.version_doi = format!(
"xx.xxxx/dev-build.{}.v{}",
&self.proj.user_facing_name, &self.proj.version
);
false
}
ZenodoMode::Release(ref svc) => {
if let Some(target_version) = md.concept_rec_id.strip_prefix("new-for:") {
if target_version != self.proj.version.to_string() {
error!("the Zenodo metadata file specifies that a new \"concept DOI\" should be created");
error!(
"... but for version `{}`, while this run is for version `{}`",
target_version, &self.proj.version
);
error!("... this suggests that you need to update `{}` to include the Zenodo record ID of the \"concept record\"", metadata_path.display());
error!(
"... so that this release will be properly linked to previous releases"
);
error!("If you really want to create a new concept DOI, update the version in the `conceptrecid: \"new-for:...\"` specification");
bail!("refusing to proceed");
}
self.preregister_new_concept(svc, &mut md)?;
true
} else {
self.preregister_existing_concept(svc, &mut md)?;
false
}
}
};
info!(
"DOI for {}@{}: {}",
&self.proj.user_facing_name, &self.proj.version, &md.version_doi
);
info!(
"Zenodo record-id for {}@{}: {}",
&self.proj.user_facing_name, &self.proj.version, &md.version_rec_id
);
if new_concept {
info!(
"Zenodo record-id for {} \"concept\": {}",
&self.proj.user_facing_name, &md.concept_rec_id
);
info!(
"... you should insert this value into the `conceptrecid` field of `{}`",
metadata_path.display()
);
info!("... so that subsequent releases are properly associated with this one");
}
{
let mut f = atry!(
File::create(metadata_path);
["failed to open `{}` for rewriting", metadata_path.display()]
);
atry!(
serde_json::to_writer_pretty(&mut f, &md);
["failed to overwrite JSON file `{}`", metadata_path.display()]
);
atry!(
write_crlf!(f, "");
["failed to overwrite JSON file `{}`", metadata_path.display()]
);
}
let mut rewrites = Vec::new();
rewrites.push((
format!("xx.xxxx/dev-build.{}.concept", &self.proj.user_facing_name),
md.concept_doi.clone(),
));
rewrites.push((
format!("xx.xxxx/dev-build.{}.version", &self.proj.user_facing_name),
md.version_doi.clone(),
));
for rw_path in rewrite_paths {
atry!(
self.rewrite_file(rw_path, &rewrites);
["error while attempting to rewrite `{}`", rw_path.display()]
);
}
Ok(())
}
fn preregister_new_concept(&self, svc: &ZenodoService, md: &mut ZenodoMetadata) -> Result<()> {
let client = svc.make_blocking_client()?;
let url = svc.api_url("deposit/depositions");
self.send_metadata_and_slurp(&client, &url, true, md)
.map(|_info| ())
}
fn preregister_existing_concept(
&self,
svc: &ZenodoService,
md: &mut ZenodoMetadata,
) -> Result<()> {
let client = svc.make_blocking_client()?;
let url = svc.api_url(&format!("records/{}", &md.concept_rec_id));
let resp = client.get(&url).send()?;
let status = resp.status();
let mut parsed = json::parse(&resp.text()?)?;
ensure!(
status.is_success(),
"query of concept record `{}` failed: {}",
&md.concept_rec_id,
parsed
);
let last_rec_id = match parsed["id"].take() {
JsonValue::String(s) => s,
JsonValue::Short(s) => s.as_str().to_owned(),
JsonValue::Number(n) => n.to_string(),
_ => {
error!("Zenodo response: {}", parsed);
bail!("queried Zenodo concept record but got no `id`");
}
};
info!(
"resolved concept rec-id {} to latest release {}",
&md.concept_rec_id, &last_rec_id
);
let url = svc.api_url(&format!(
"deposit/depositions/{}/actions/newversion",
&last_rec_id
));
let resp = client.post(&url).send()?;
let status = resp.status();
let mut parsed = json::parse(&resp.text()?)?;
ensure!(
status.is_success(),
"request for new version failed: {}",
parsed
);
let new_rec_url = a_ok_or!(
parsed["links"]["latest_draft"].take_string();
["Zenodo new version request seems to have worked, but no `links.latest_draft` available: {}", parsed]
);
info!("URL of new version: {}", &new_rec_url);
let mut info = self.send_metadata_and_slurp(&client, &new_rec_url, false, md)?;
if let JsonValue::Array(mut files) = info["files"].take() {
for mut fileinfo in files.drain(..) {
let name = fileinfo["filename"]
.take_string()
.unwrap_or_else(|| "(unknown name)".to_owned());
if let Some(url) = fileinfo["links"]["self"].take_string() {
info!("deleting propagated artifact file `{}` ...", name);
let resp = client.delete(&url).send()?;
let status = resp.status();
if !status.is_success() {
let t = resp.text().unwrap_or_else(|_e| {
"(unable to parse server response body)".to_owned()
});
warn!("failed to delete {}: {}", url, t);
}
} else {
info!("not deleting propagated artifact file `{}` because `links.self` was not specified", name);
}
}
}
Ok(())
}
fn send_metadata_and_slurp(
&self,
client: &reqwest::blocking::Client,
url: &str,
do_post: bool,
md: &mut ZenodoMetadata,
) -> Result<JsonValue> {
let md_body = atry!(
serde_json::to_string(&md.metadata);
["failed to serialize Zenodo metadata to JSON"]
);
let body = format!("{{\"metadata\":{}}}", md_body);
let req = if do_post {
client.post(url)
} else {
client.put(url)
};
let resp = req
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(body)
.send()?;
let status = resp.status();
let mut parsed = json::parse(&resp.text()?)?;
if status.is_success() {
info!("preregistration completed");
} else {
bail!("Zenodo preregistration failed: {}", parsed);
}
if let Some(s) = parsed["conceptrecid"].take_string() {
md.concept_rec_id = s;
} else {
error!("Zenodo response: {}", parsed);
bail!(
"Zenodo preregistration seems to have succeeded, but response was \
missing `conceptrecid` string field; cannot proceed"
);
}
md.version_rec_id = match parsed["record_id"].take() {
JsonValue::String(s) => s,
JsonValue::Short(s) => s.as_str().to_owned(),
JsonValue::Number(n) => n.to_string(),
_ => {
error!("Zenodo response: {}", parsed);
bail!(
"Zenodo preregistration seems to have succeeded, the `record_id`
field had a surprising format; cannot proceed"
);
}
};
if let Some(s) = parsed["metadata"]["prereserve_doi"]["doi"].take_string() {
md.version_doi = s;
} else {
error!("Zenodo response: {}", parsed);
bail!(
"Zenodo preregistration seems to have succeeded, but response was \
missing `metadata.prereserve_doi.doi` string field; cannot proceed"
);
}
if let Some(s) = parsed["links"]["bucket"].take_string() {
md.bucket_link = s;
} else {
error!("Zenodo response: {}", parsed);
bail!(
"Zenodo preregistration seems to have succeeded, but response was \
missing `links.bucket` string field; cannot proceed"
);
}
let mut maybe_concept_doi = parsed["conceptdoi"].take_string().unwrap_or_default();
if maybe_concept_doi.is_empty() {
warn!("fabricating Zenodo concept DOI for first-time registration");
warn!("... it could be incorrect if Zenodo changes their DOI implementation");
maybe_concept_doi = format!("10.5281/zenodo.{}", &md.concept_rec_id);
}
md.concept_doi = maybe_concept_doi;
Ok(parsed)
}
fn rewrite_file<P: AsRef<Path>>(&self, path: P, rewrites: &[(String, String)]) -> Result<()> {
let path = path.as_ref();
let mut did_anything = false;
let cur_f = atry!(
File::open(&path);
["failed to open file `{}` for reading", path.display()]
);
let cur_reader = BufReader::new(cur_f);
let new_af =
atomicwrites::AtomicFile::new(&path, atomicwrites::OverwriteBehavior::AllowOverwrite);
let r = new_af.write(|new_f| {
for line in cur_reader.lines() {
let mut line = atry!(
line;
["error reading data from file `{}`", path.display()]
);
for (ref template, ref replacement) in rewrites {
if line.contains(template) {
line = line.replace(template, replacement);
did_anything = true;
}
}
atry!(
write_crlf!(new_f, "{}", line);
["error writing data to `{}`", new_af.path().display()]
);
}
Ok(())
});
match r {
Err(atomicwrites::Error::Internal(e)) => Err(e.into()),
Err(atomicwrites::Error::User(e)) => Err(e),
Ok(()) => {
if !did_anything {
warn!(
"rewriter for Zenodo DOI file `{}` didn't make any modifications",
path.display()
);
}
Ok(())
}
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct ZenodoMetadata {
#[serde(rename = "conceptrecid")]
pub concept_rec_id: String,
pub metadata: Map<String, Value>,
#[serde(rename = "conceptdoi", default)]
pub concept_doi: String,
#[serde(rename = "record_id", default)]
pub version_rec_id: String,
#[serde(rename = "doi", default)]
pub version_doi: String,
#[serde(default)]
pub bucket_link: String,
}
impl ZenodoMetadata {
fn load_for_prereg<P: AsRef<Path>>(path: P) -> Result<Self> {
let path = path.as_ref();
let md = Self::load_base(path)?;
ensure!(
md.concept_doi.is_empty(),
"`conceptdoi` field of `{}` must not be specified before preregistration",
path.display()
);
ensure!(
md.version_rec_id.is_empty(),
"`record_id` field of `{}` must not be specified before preregistration",
path.display()
);
ensure!(
md.version_doi.is_empty(),
"`doi` field of `{}` must not be specified before preregistration",
path.display()
);
ensure!(
md.bucket_link.is_empty(),
"`bucket_link` field of `{}` must not be specified before preregistration",
path.display()
);
Ok(md)
}
fn load_for_deployment<P: AsRef<Path>>(path: P) -> Result<Self> {
let path = path.as_ref();
let md = Self::load_base(path)?;
ensure!(
!md.version_doi.is_empty(),
"`doi` field of `{}` should be specified for deployment",
path.display()
);
Ok(md)
}
fn load_base<P: AsRef<Path>>(path: P) -> Result<Self> {
let path = path.as_ref();
let t = atry!(
fs::read_to_string(path);
["failed to read file `{}` as text", path.display()]
);
Ok(atry!(
json5::from_str::<ZenodoMetadata>(&t);
["failed to parse file `{}` as JSON5", path.display()]
))
}
}
#[derive(Debug, PartialEq, StructOpt)]
pub enum ZenodoCommands {
Preregister(PreregisterCommand),
Publish(PublishCommand),
#[structopt(name = "upload-artifacts")]
UploadArtifacts(UploadArtifactsCommand),
}
#[derive(Debug, PartialEq, StructOpt)]
pub struct ZenodoCommand {
#[structopt(subcommand)]
command: ZenodoCommands,
}
impl Command for ZenodoCommand {
fn execute(self) -> Result<i32> {
match self.command {
ZenodoCommands::Preregister(o) => o.execute(),
ZenodoCommands::Publish(o) => o.execute(),
ZenodoCommands::UploadArtifacts(o) => o.execute(),
}
}
}
#[derive(Debug, PartialEq, StructOpt)]
pub struct PreregisterCommand {
#[structopt(
short = "f",
long = "force",
help = "Force operation even in unexpected conditions"
)]
force: bool,
#[structopt(
long = "metadata",
help = "The path to a JSON5 file containing Zenodo deposition metadata.",
required = true
)]
metadata_path: PathBuf,
#[structopt(
help = "The name of the project associated with this deposition.",
required = true
)]
proj_name: String,
#[structopt(
help = "The path(s) of file(s) to rewrite with DOI data",
required = true
)]
rewrite_paths: Vec<PathBuf>,
}
impl Command for PreregisterCommand {
fn execute(self) -> Result<i32> {
let mut sess = AppSession::initialize_default()?;
let (dev_mode, rci) = sess.ensure_ci_rc_mode(self.force)?;
sess.apply_versions(&rci)?;
let ident = sess
.graph()
.lookup_ident(&self.proj_name)
.ok_or_else(|| anyhow!("no such project `{}`", self.proj_name))?;
let proj = sess.graph().lookup(ident);
if rci.lookup_project(proj).is_none() {
if self.force {
warn!(
"project `{}` does not seem to be freshly released; ignoring due to --force mode",
self.proj_name
);
} else {
error!(
"project `{}` does not seem to be freshly released",
self.proj_name
);
bail!("refusing to proceed (use `--force` to override)",);
}
}
let wf = ZenodoWorkflow::new(proj, dev_mode)?;
wf.preregister(&self.metadata_path, &self.rewrite_paths[..])?;
Ok(0)
}
}
#[derive(Debug, PartialEq, StructOpt)]
pub struct PublishCommand {
#[structopt(
short = "f",
long = "force",
help = "Force operation even in unexpected conditions"
)]
force: bool,
#[structopt(
long = "metadata",
help = "The path to a JSON5 file containing Zenodo deposition metadata.",
required = true
)]
metadata_path: PathBuf,
}
impl Command for PublishCommand {
fn execute(self) -> Result<i32> {
let sess = AppSession::initialize_default()?;
let (dev_mode, _rci) = sess.ensure_ci_rc_mode(self.force)?;
if dev_mode {
if self.force {
warn!("should not publish to Zenodo in development mode, but you're forcing me to");
} else {
error!("do not publish to Zenodo in development mode");
bail!("refusing to proceed (use `--force` to override)",);
}
}
let md = atry!(
ZenodoMetadata::load_for_deployment(&self.metadata_path);
["failed to load Zenodo metadata file `{}`", &self.metadata_path.display()]
);
let svc = ZenodoService::new()?;
let client = svc.make_blocking_client()?;
let md_body = atry!(
serde_json::to_string(&md.metadata);
["failed to serialize Zenodo metadata to JSON"]
);
let body = format!("{{\"metadata\":{}, \"state\": \"done\"}}", md_body);
let url = svc.api_url(&format!("deposit/depositions/{}", &md.version_rec_id));
let resp = client
.put(&url)
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(body)
.send()?;
let status = resp.status();
let parsed = json::parse(&resp.text()?)?;
if !status.is_success() {
error!("Zenodo metadata API response: {}", parsed);
bail!("publication of record `{}` failed", &md.version_rec_id);
}
let url = svc.api_url(&format!(
"deposit/depositions/{}/actions/publish",
&md.version_rec_id
));
let resp = client.post(&url).send()?;
let status = resp.status();
let parsed = json::parse(&resp.text()?)?;
if !status.is_success() {
error!("Zenodo API response: {}", parsed);
bail!("publication of record `{}` failed", &md.version_rec_id);
}
info!(
"publication successful - view at https://zenodo.org/record/{}",
&md.version_rec_id
);
Ok(0)
}
}
#[derive(Debug, PartialEq, StructOpt)]
pub struct UploadArtifactsCommand {
#[structopt(
short = "f",
long = "force",
help = "Force operation even in unexpected conditions"
)]
force: bool,
#[structopt(
long = "metadata",
help = "The path to a JSON5 file containing Zenodo deposition metadata.",
required = true
)]
metadata_path: PathBuf,
#[structopt(help = "The path(s) to the file(s) to upload", required = true)]
paths: Vec<PathBuf>,
}
impl Command for UploadArtifactsCommand {
fn execute(self) -> Result<i32> {
let sess = AppSession::initialize_default()?;
let (dev_mode, _rci) = sess.ensure_ci_rc_mode(self.force)?;
if dev_mode {
if self.force {
warn!("should not upload artifacts in development mode, but you're forcing me to");
} else {
error!("do not upload artifacts in development mode");
bail!("refusing to proceed (use `--force` to override)",);
}
}
let md = atry!(
ZenodoMetadata::load_for_deployment(&self.metadata_path);
["failed to load Zenodo metadata file `{}`", &self.metadata_path.display()]
);
let svc = ZenodoService::new()?;
let client = svc.make_blocking_client()?;
for path in &self.paths {
let file = File::open(path)?;
let name = path
.file_name()
.ok_or_else(|| anyhow!("input file has no name component??"))?
.to_str()
.ok_or_else(|| anyhow!("input file name cannot be stringified"))?
.to_owned();
let enc =
percent_encoding::utf8_percent_encode(&name, percent_encoding::NON_ALPHANUMERIC);
info!("uploading `{}` => {}", path.display(), &name);
let url = format!("{}/{}", md.bucket_link, enc);
let resp = client
.put(&url)
.header(reqwest::header::CONTENT_TYPE, "application/octet-stream")
.body(file)
.send()?;
let status = resp.status();
let parsed = json::parse(&resp.text()?)?;
if !status.is_success() {
error!("Zenodo API response: {}", parsed);
bail!("creation of asset `{}` failed", name);
}
}
Ok(0)
}
}