use compose_yml::v2 as dc;
use std::{
collections::{btree_map::Entry, BTreeMap, BTreeSet},
env,
fmt::Debug,
fs,
io::{self, Read},
path::PathBuf,
result,
time::{Duration, SystemTime},
};
use vault::client::VaultDuration;
use crate::errors::*;
use crate::plugins;
use crate::plugins::{Operation, PluginGenerate, PluginNew, PluginTransform};
use crate::pod::Pod;
use crate::project::Project;
use crate::serde_helpers::{dump_yaml, load_yaml, seconds_since_epoch};
use crate::util::err;
use crate::Target;
const DEFAULT_TTL: u64 = 30 * 24 * 60 * 60;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
enum AuthType {
#[serde(rename = "token")]
Token,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct ServiceConfig {
#[serde(default)]
no_default_policies: bool,
#[serde(default)]
policies: Vec<dc::RawOr<String>>,
}
type PodConfig = BTreeMap<String, ServiceConfig>;
#[derive(Clone, Debug, Default, Eq, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
struct TargetConfig {
#[serde(default)]
extra_environment: BTreeMap<String, dc::RawOr<String>>,
#[serde(default)]
default_ttl: Option<u64>,
#[serde(default)]
default_policies: Option<Vec<dc::RawOr<String>>>,
}
impl TargetConfig {
fn extended_with(&self, other: &TargetConfig) -> TargetConfig {
let mut extra_environment = self.extra_environment.clone();
extra_environment.extend(other.extra_environment.clone());
TargetConfig {
extra_environment,
default_ttl: other.default_ttl.or(self.default_ttl),
default_policies: other
.default_policies
.clone()
.or_else(|| self.default_policies.clone()),
}
}
}
#[test]
fn target_config_extension() {
let mut e1 = BTreeMap::new();
e1.insert("K1".to_owned(), dc::value("V1".to_owned()));
let c1 = TargetConfig {
extra_environment: e1,
default_ttl: Some(1),
default_policies: Some(vec![dc::value("p1".to_owned())]),
};
let mut e2 = BTreeMap::new();
e2.insert("K2".to_owned(), dc::value("V2".to_owned()));
let c2 = TargetConfig {
extra_environment: e2,
default_ttl: Some(2),
default_policies: Some(vec![dc::value("p2".to_owned())]),
};
let mut e_all = BTreeMap::new();
e_all.insert("K1".to_owned(), dc::value("V1".to_owned()));
e_all.insert("K2".to_owned(), dc::value("V2".to_owned()));
assert_eq!(
c1.extended_with(&c2),
TargetConfig {
extra_environment: e_all.clone(),
default_ttl: Some(2),
default_policies: Some(vec![dc::value("p2".to_owned())]),
}
);
assert_eq!(
c2.extended_with(&c1),
TargetConfig {
extra_environment: e_all,
default_ttl: Some(1),
default_policies: Some(vec![dc::value("p1".to_owned())]),
}
);
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Config {
enable_in_targets: Option<Vec<String>>,
auth_type: AuthType,
#[serde(flatten)]
default_target_config: TargetConfig,
#[serde(default)]
targets: BTreeMap<Target, TargetConfig>,
#[serde(default)]
pods: BTreeMap<String, PodConfig>,
}
impl Config {
fn target_config_for(&self, target: &Target) -> TargetConfig {
let config = self.targets.get(target).cloned().unwrap_or_default();
self.default_target_config.extended_with(&config)
}
}
#[test]
fn can_deserialize_config() {
use std::path::Path;
let path = Path::new("examples/vault_integration/config/vault.yml");
let config: Config = load_yaml(&path).unwrap();
assert_eq!(config.auth_type, AuthType::Token);
}
fn load_vault_token_from_file() -> Result<String> {
let path = dirs::home_dir()
.ok_or_else(|| err("You do not appear to have a home directory"))?
.join(".vault-token");
let mkerr = || ErrorKind::CouldNotReadFile(path.clone());
let f = fs::File::open(&path).chain_err(&mkerr)?;
let mut reader = io::BufReader::new(f);
let mut result = String::new();
reader.read_to_string(&mut result).chain_err(&mkerr)?;
Ok(result.trim().to_owned())
}
fn find_vault_token() -> Result<String> {
env::var("VAULT_MASTER_TOKEN")
.or_else(|_| env::var("VAULT_TOKEN"))
.or_else(|_| load_vault_token_from_file())
.map_err(|e| {
err!(
"{}. You probably want to log in using the vault client or set \
VAULT_MASTER_TOKEN",
e
)
})
}
#[derive(Debug)]
struct ConfigEnvironment<'a> {
ctx: &'a plugins::Context<'a>,
service: &'a str,
}
impl<'a> dc::Environment for ConfigEnvironment<'a> {
fn var(&self, key: &str) -> result::Result<String, env::VarError> {
let result = match key {
"PROJECT" => Ok(self.ctx.project.name()),
"TARGET" => Ok(self.ctx.project.current_target().name()),
"POD" => Ok(self.ctx.pod.name()),
"SERVICE" => Ok(self.service),
_ => Err(env::VarError::NotPresent),
};
result.map(|s| s.to_owned())
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
struct TokenInfo {
token: String,
policies: BTreeSet<String>,
#[serde(with = "seconds_since_epoch")]
expires: SystemTime,
}
impl TokenInfo {
fn should_renew(
&self,
desired_policies: &BTreeSet<String>,
desired_ttl: Duration,
) -> bool {
&self.policies != desired_policies
|| SystemTime::now() + desired_ttl / 2 >= self.expires
}
}
trait GenerateToken: Debug + Sync {
fn addr(&self) -> &str;
fn generate_token(
&self,
display_name: &str,
policies: &BTreeSet<String>,
ttl: Duration,
) -> Result<TokenInfo>;
}
#[derive(Debug)]
struct Vault {
addr: String,
token: String,
}
impl Vault {
fn new() -> Result<Vault> {
let mut addr = env::var("VAULT_ADDR").map_err(|_| {
err(
"Please set the environment variable VAULT_ADDR to the URL of \
your vault server",
)
})?;
if addr.ends_with('/') {
let new_len = addr.len() - 1;
addr.truncate(new_len);
}
let token = find_vault_token()?;
Ok(Vault { addr, token })
}
}
impl GenerateToken for Vault {
fn addr(&self) -> &str {
&self.addr
}
fn generate_token(
&self,
display_name: &str,
policies: &BTreeSet<String>,
ttl: Duration,
) -> Result<TokenInfo> {
let mkerr = || ErrorKind::VaultError(self.addr.clone());
let client =
vault::Client::new(&self.addr[..], &self.token).chain_err(&mkerr)?;
let opts = vault::client::TokenOptions::default()
.display_name(display_name)
.renewable(true)
.ttl(VaultDuration(ttl))
.policies(policies.clone());
let auth = client.create_token(&opts).chain_err(&mkerr)?;
let lease_duration = auth
.lease_duration
.map_or_else(|| Duration::from_secs(30 * 24 * 60 * 60), |d| d.0);
let expires = SystemTime::now() + lease_duration;
Ok(TokenInfo {
token: auth.client_token,
policies: policies.to_owned(),
expires,
})
}
}
type CachedPodTokens = BTreeMap<String, TokenInfo>;
type CachedTargetTokens = BTreeMap<String, CachedPodTokens>;
#[derive(Default, Deserialize, Serialize)]
struct CachedTokens {
targets: BTreeMap<String, CachedTargetTokens>,
}
impl CachedTokens {
fn entry(
&mut self,
target: String,
pod: String,
service: String,
) -> Entry<String, TokenInfo> {
self.targets
.entry(target)
.or_default()
.entry(pod)
.or_default()
.entry(service)
}
}
struct TokenCache<'gen> {
generator: &'gen dyn GenerateToken,
cache_path: PathBuf,
cached: CachedTokens,
}
impl<'gen> TokenCache<'gen> {
fn load_or_create(
project: &Project,
pod: &Pod,
generator: &'gen dyn GenerateToken,
) -> Result<TokenCache<'gen>> {
let cache_path = project
.output_dir()
.join("vault-cache")
.join(format!("{}.yml", pod.name()));
let cached = if cache_path.exists() {
load_yaml(&cache_path)?
} else {
CachedTokens::default()
};
Ok(TokenCache {
generator,
cache_path,
cached,
})
}
fn save(&self) -> Result<()> {
dump_yaml(&self.cache_path, &self.cached)
}
fn get<'cache>(
&'cache mut self,
project: &str,
target: &str,
pod: &str,
service: &str,
policies: &BTreeSet<String>,
ttl: Duration,
) -> Result<&'cache TokenInfo> {
let display_name = || format!("{}_{}_{}_{}", project, target, pod, service);
let mkerr = || format!("could not generate token for '{}'", service);
let entry =
self.cached
.entry(target.to_owned(), pod.to_owned(), service.to_owned());
match entry {
Entry::Vacant(vacancy) => {
trace!("token cache does not contain {}", service);
let info = self
.generator
.generate_token(&display_name(), policies, ttl)
.chain_err(mkerr)?;
Ok(vacancy.insert(info))
}
Entry::Occupied(occupied) => {
trace!("token cache hit for {}", service);
let cached = occupied.into_mut();
if cached.should_renew(&policies, ttl) {
trace!("token cache needs new token for {}", service);
*cached = self
.generator
.generate_token(&display_name(), policies, ttl)
.chain_err(mkerr)?;
}
Ok(cached)
}
}
}
}
#[derive(Debug)]
pub struct Plugin {
config: Option<Config>,
generator: Option<Box<dyn GenerateToken>>,
}
impl Plugin {
fn config_path(project: &Project) -> PathBuf {
project.root_dir().join("config").join("vault.yml")
}
fn new_with_generator<G>(project: &Project, generator: Option<G>) -> Result<Plugin>
where
G: GenerateToken + 'static,
{
let path = Self::config_path(project);
let config = if path.exists() {
Some(load_yaml(&path)?)
} else {
None
};
Ok(Plugin {
config,
generator: generator
.map(|gen: G| -> Box<dyn GenerateToken> { Box::new(gen) }),
})
}
}
impl plugins::Plugin for Plugin {
fn name(&self) -> &'static str {
Self::plugin_name()
}
}
impl PluginNew for Plugin {
fn plugin_name() -> &'static str {
"vault"
}
fn is_configured_for(project: &Project) -> Result<bool> {
let path = Self::config_path(project);
Ok(path.exists())
}
fn new(project: &Project) -> Result<Self> {
let token_gen = if Self::is_configured_for(project)? {
Some(Vault::new()?)
} else {
None
};
Self::new_with_generator(project, token_gen)
}
}
impl PluginGenerate for Plugin {
fn generator_description(&self) -> &'static str {
"Get passwords & other secrets from a Vault server"
}
}
impl PluginTransform for Plugin {
fn transform(
&self,
_op: Operation,
ctx: &plugins::Context<'_>,
file: &mut dc::File,
) -> Result<()> {
let config = self
.config
.as_ref()
.expect("config should always be present for transform");
let generator = self
.generator
.as_ref()
.expect("generator should always be present for transform");
let target = ctx.project.current_target();
if !target.is_enabled_by(&config.enable_in_targets) {
return Ok(());
}
let mut cache =
TokenCache::load_or_create(&ctx.project, &ctx.pod, generator.as_ref())?;
for (name, service) in &mut file.services {
let env = ConfigEnvironment { ctx, service: name };
let interpolated = |raw_val: &dc::RawOr<String>| -> Result<String> {
let mut val = raw_val.to_owned();
Ok(val.interpolate_env(&env)?.to_owned())
};
let service_config = config
.pods
.get(ctx.pod.name())
.and_then(|pod| pod.get(name));
let target_config = config.target_config_for(target);
let mut raw_policies =
if service_config.map_or_else(|| false, |s| s.no_default_policies) {
vec![]
} else {
target_config.default_policies.clone().unwrap_or_default()
};
raw_policies
.extend(service_config.map_or_else(Vec::new, |s| s.policies.clone()));
if raw_policies.is_empty() {
debug!(
"Skipping token generation for {} because it has no policies",
name
);
continue;
}
let mut policies = BTreeSet::new();
for result in raw_policies.iter().map(|p| interpolated(p)) {
policies.insert(result?);
}
debug!(
"Generating token for '{}' with policies {:?}",
name, &policies
);
service
.environment
.insert("VAULT_ADDR".to_owned(), dc::escape(generator.addr())?);
let ttl =
Duration::from_secs(target_config.default_ttl.unwrap_or(DEFAULT_TTL));
let token_info = cache.get(
ctx.project.name(),
ctx.project.current_target().name(),
ctx.pod.name(),
name,
&policies,
ttl,
)?;
service
.environment
.insert("VAULT_TOKEN".to_owned(), dc::escape(&token_info.token)?);
for (var, val) in &target_config.extra_environment {
service
.environment
.insert(var.to_owned(), dc::escape(interpolated(val)?)?);
}
}
cache.save()?;
Ok(())
}
}
#[cfg(test)]
mod test {
use super::*;
use std::{
iter::FromIterator,
sync::{Arc, RwLock},
};
#[cfg(test)]
type MockVaultCalls = Arc<RwLock<Vec<(String, BTreeSet<String>, Duration)>>>;
#[derive(Debug)]
#[cfg(test)]
struct MockVault {
calls: MockVaultCalls,
}
#[cfg(test)]
impl MockVault {
fn new() -> MockVault {
MockVault {
calls: Arc::new(RwLock::new(vec![])),
}
}
fn calls(&self) -> MockVaultCalls {
self.calls.clone()
}
}
#[cfg(test)]
impl GenerateToken for MockVault {
fn addr(&self) -> &str {
"http://example.com:8200/"
}
fn generate_token(
&self,
display_name: &str,
policies: &BTreeSet<String>,
ttl: Duration,
) -> Result<TokenInfo> {
self.calls.write().unwrap().push((
display_name.to_owned(),
policies.to_owned(),
ttl,
));
Ok(TokenInfo {
token: "fake_token".to_owned(),
policies: policies.to_owned(),
expires: SystemTime::now() + ttl,
})
}
}
#[test]
fn do_not_renew_token_if_policies_and_ttl_are_fine() {
let desired_policies = BTreeSet::from_iter(vec!["a".to_owned()]);
let token_info = TokenInfo {
token: "placeholder".to_owned(),
policies: desired_policies.clone(),
expires: SystemTime::now() + Duration::from_secs(60),
};
assert!(!token_info.should_renew(&desired_policies, Duration::from_secs(60)));
}
#[test]
fn renew_token_if_policies_do_not_match() {
let desired_policies = BTreeSet::from_iter(vec!["a".to_owned()]);
let token_info = TokenInfo {
token: "placeholder".to_owned(),
policies: BTreeSet::from_iter(vec!["b".to_owned()]),
expires: SystemTime::now() + Duration::from_secs(60),
};
assert!(token_info.should_renew(&desired_policies, Duration::from_secs(60)));
}
#[test]
fn renew_token_if_half_of_ttl_expired() {
let desired_policies = BTreeSet::from_iter(vec!["a".to_owned()]);
let token_info = TokenInfo {
token: "placeholder".to_owned(),
policies: desired_policies.clone(),
expires: SystemTime::now() + Duration::from_secs(29),
};
assert!(token_info.should_renew(&desired_policies, Duration::from_secs(60)));
}
#[test]
fn interpolates_policies() {
let _ = env_logger::try_init();
env::set_var("VAULT_ADDR", "http://example.com:8200/");
env::set_var("VAULT_MASTER_TOKEN", "fake master token");
let mut proj = Project::from_example("vault_integration").unwrap();
proj.set_current_target_name("development").unwrap();
let vault = MockVault::new();
let calls = vault.calls();
let plugin = Plugin::new_with_generator(&proj, Some(vault)).unwrap();
let frontend = proj.pod("frontend").unwrap();
let ctx = plugins::Context::new(&proj, frontend, "up");
let mut file = frontend.merged_file(proj.current_target()).unwrap();
plugin
.transform(Operation::Output, &ctx, &mut file)
.unwrap();
let web = file.services.get("web").unwrap();
let vault_addr = web.environment.get("VAULT_ADDR").expect("has VAULT_ADDR");
assert_eq!(vault_addr.value().unwrap(), "http://example.com:8200/");
let vault_token = web.environment.get("VAULT_TOKEN").expect("has VAULT_TOKEN");
assert_eq!(vault_token.value().unwrap(), "fake_token");
let vault_env = web.environment.get("VAULT_ENV").expect("has VAULT_ENV");
assert_eq!(vault_env.value().unwrap(), "development");
let db = proj.pod("db").unwrap();
let ctx = plugins::Context::new(&proj, db, "up");
let mut file = db.merged_file(proj.current_target()).unwrap();
plugin
.transform(Operation::Output, &ctx, &mut file)
.unwrap();
let dbs = file.services.get("db").unwrap();
assert!(dbs.environment.get("VAULT_ADDR").is_none());
assert!(dbs.environment.get("VAULT_TOKEN").is_none());
assert!(dbs.environment.get("VAULT_ENV").is_none());
let calls = calls.read().unwrap();
assert_eq!(calls.len(), 1);
let (ref display_name, ref policies, ref ttl) = calls[0];
assert_eq!(display_name, "vault_integration_development_frontend_web");
assert_eq!(
policies,
&BTreeSet::from_iter(vec![
"vault_integration-development".to_owned(),
"vault_integration-development-frontend-web".to_owned(),
"vault_integration-development-ssl".to_owned()
])
);
assert_eq!(ttl, &Duration::from_secs(2592000));
}
#[test]
fn only_applied_in_specified_targets() {
let _ = env_logger::try_init();
let mut proj = Project::from_example("vault_integration").unwrap();
proj.set_current_target_name("test").unwrap();
let target = proj.current_target();
let vault = MockVault::new();
let plugin = Plugin::new_with_generator(&proj, Some(vault)).unwrap();
let frontend = proj.pod("frontend").unwrap();
let ctx = plugins::Context::new(&proj, frontend, "test");
let mut file = frontend.merged_file(target).unwrap();
plugin
.transform(Operation::Output, &ctx, &mut file)
.unwrap();
let web = file.services.get("web").unwrap();
assert_eq!(web.environment.get("VAULT_ADDR"), None);
}
}