use super::http::extract_github_error_message;
use super::models::ProviderErrorResponse;
use futures::StreamExt;
use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, USER_AGENT};
use reqwest::{Client, StatusCode};
use serde::{Deserialize, Serialize};
use std::collections::{hash_map::Entry, HashMap};
use std::time::Duration;
const GITHUB_API_BASE: &str = "https://api.github.com";
const GITHUB_API_VERSION: &str = "2022-11-28";
const VARIABLE_LIST_PAGE_SIZE: usize = 30;
const VARIABLE_WRITE_CONCURRENCY: usize = 2;
const VARIABLE_WRITE_MAX_RATE_LIMIT_RETRIES: u32 = 8;
const VARIABLE_WRITE_DEFAULT_BACKOFF_MS: u64 = 1_000;
const VARIABLE_WRITE_MAX_BACKOFF_MS: u64 = 60_000;
const GITHUB_ENV_PREFIX: &str = "GITHUB_";
const GITHUB_STORAGE_PREFIX: &str = "_GITHUB_";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GitHubSecretVariable {
pub name: String,
pub value: String,
}
#[derive(Debug)]
pub struct GitHubEnvironmentClient {
owner: String,
repo: String,
environment: String,
token: String,
client: Client,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VariableWriteMode {
Create,
Update,
}
impl GitHubEnvironmentClient {
pub fn new(
owner: impl Into<String>,
repo: impl Into<String>,
environment: impl Into<String>,
token: impl Into<String>,
) -> Result<Self, String> {
let client = Client::builder()
.user_agent("xbp")
.build()
.map_err(|error| format!("Failed to build GitHub client: {}", error))?;
Ok(Self {
owner: owner.into(),
repo: repo.into(),
environment: environment.into(),
token: token.into(),
client,
})
}
pub async fn validate_repo_access(&self) -> Result<(), String> {
let response = self
.client
.get(format!(
"{}/repos/{}/{}",
GITHUB_API_BASE, self.owner, self.repo
))
.headers(self.auth_headers()?)
.send()
.await
.map_err(|error| format!("GitHub repository check failed: {}", error))?;
if response.status().is_success() {
return Ok(());
}
let status = response.status();
let body = response.text().await.unwrap_or_default();
let detail =
extract_github_error_message(&body).unwrap_or_else(|| format!("HTTP {}", status));
Err(format!(
"GitHub repository `{}/{}` is not accessible: {}",
self.owner, self.repo, detail
))
}
pub async fn list(&self) -> Result<Vec<GitHubSecretVariable>, String> {
self.list_with_progress(&mut |_| {}).await
}
pub async fn list_with_progress<F>(
&self,
progress: &mut F,
) -> Result<Vec<GitHubSecretVariable>, String>
where
F: FnMut(&str),
{
Ok(self
.list_raw_with_progress(progress)
.await?
.into_iter()
.map(normalize_variable_name_from_github)
.collect())
}
async fn list_raw_with_progress<F>(
&self,
progress: &mut F,
) -> Result<Vec<GitHubSecretVariable>, String>
where
F: FnMut(&str),
{
let mut page = 1usize;
let mut results = Vec::new();
loop {
progress(&format!("Fetching GitHub variables page {}", page));
let url = format!(
"{}/repos/{}/{}/environments/{}/variables?per_page={}&page={}",
GITHUB_API_BASE,
self.owner,
self.repo,
self.environment,
VARIABLE_LIST_PAGE_SIZE,
page
);
let response = self
.client
.get(&url)
.headers(self.auth_headers()?)
.send()
.await
.map_err(|error| format!("GitHub list request failed: {}", error))?;
let status = response.status();
let body = response
.text()
.await
.map_err(|error| format!("GitHub response read failed: {}", error))?;
if !status.is_success() {
let detail = extract_github_error_message(&body)
.unwrap_or_else(|| format!("HTTP {}", status));
return Err(format!(
"GitHub API returned {} when listing variables: {}",
status, detail
));
}
let payload: ListVariablesResponse = serde_json::from_str(&body)
.map_err(|error| format!("GitHub response parsing failed: {}", error))?;
let count = payload.variables.len();
results.extend(
payload
.variables
.into_iter()
.map(raw_variable_entry_to_secret_variable),
);
if count < VARIABLE_LIST_PAGE_SIZE {
break;
}
page += 1;
}
Ok(results)
}
pub async fn upsert(
&self,
variables: &HashMap<String, String>,
progress: &mut dyn FnMut(&str),
) -> Result<(), String> {
self.upsert_with_counts(variables, &mut |completed, total, message| {
if total == 0 {
progress(message);
} else {
progress(&format!("{message} ({completed}/{total})"));
}
})
.await
}
pub async fn upsert_with_counts(
&self,
variables: &HashMap<String, String>,
progress: &mut dyn FnMut(usize, usize, &str),
) -> Result<(), String> {
progress(0, 0, "Ensuring GitHub environment exists");
self.ensure_environment_exists().await?;
progress(0, 0, "Listing existing GitHub variables");
let existing_names = self.list_variable_names().await?;
let provider_variables = variables_for_github(variables)?;
let plan = plan_variable_writes(&provider_variables, &existing_names);
let total = plan.len();
if plan.is_empty() {
progress(0, 0, "Nothing to write to GitHub.");
return Ok(());
}
progress(0, total, "Writing GitHub variables");
let headers = self.auth_headers()?;
let provider = self;
let mut writes = futures::stream::iter(plan.into_iter().map(|(name, value, mode)| {
let headers = headers.clone();
async move {
provider
.write_variable(&name, &value, mode, headers)
.await
.map(|_| name)
}
}))
.buffer_unordered(VARIABLE_WRITE_CONCURRENCY);
let mut completed = 0usize;
while let Some(result) = writes.next().await {
completed += 1;
match result {
Ok(name) => {
progress(
completed,
total,
&format!("Wrote `{name}`"),
);
}
Err(error) => {
progress(completed, total, "Write failed");
return Err(error);
}
}
}
progress(total, total, "All variables written");
Ok(())
}
pub async fn diag(&self) -> Result<ProviderErrorResponse, String> {
self.validate_repo_access().await?;
let variables = self.list().await?;
Ok(ProviderErrorResponse {
provider: "github".to_string(),
message: format!(
"GitHub access ok. Environment `{}` variables reachable ({} found).",
self.environment,
variables.len()
),
})
}
pub async fn ensure_environment_exists(&self) -> Result<(), String> {
let url = format!(
"{}/repos/{}/{}/environments/{}",
GITHUB_API_BASE, self.owner, self.repo, self.environment
);
let response = self
.client
.put(&url)
.headers(self.auth_headers()?)
.json(&serde_json::json!({}))
.send()
.await
.map_err(|error| format!("GitHub environment create failed: {}", error))?;
let status = response.status();
if status.is_success() {
return Ok(());
}
let body = response.text().await.unwrap_or_default();
let detail =
extract_github_error_message(&body).unwrap_or_else(|| format!("HTTP {}", status));
Err(format!(
"GitHub rejected environment `{}`: {}",
self.environment, detail
))
}
async fn list_variable_names(&self) -> Result<HashMap<String, String>, String> {
Ok(self
.list_raw_with_progress(&mut |_| {})
.await?
.into_iter()
.map(|variable| {
(
canonical_github_variable_name(&variable.name),
variable.name,
)
})
.collect())
}
async fn write_variable(
&self,
name: &str,
value: &str,
mode: VariableWriteMode,
headers: HeaderMap,
) -> Result<(), String> {
let mut mode = mode;
let mut mode_flipped = false;
let mut rate_limit_attempts = 0u32;
loop {
let response = self
.send_variable_write(name, value, mode, headers.clone())
.await?;
let status = response.status();
if status.is_success() {
return Ok(());
}
if !mode_flipped {
if let Some(retry_mode) = retry_variable_write_mode(mode, status) {
mode = retry_mode;
mode_flipped = true;
let _ = response.bytes().await;
continue;
}
}
let retry_after_ms = parse_retry_after_ms(response.headers());
let detail = describe_variable_write_failure(response).await;
if is_github_rate_limit_detail(status, &detail) {
if rate_limit_attempts >= VARIABLE_WRITE_MAX_RATE_LIMIT_RETRIES {
return Err(format!(
"GitHub rejected {}: {} (gave up after {} rate-limit retries)",
name, detail, VARIABLE_WRITE_MAX_RATE_LIMIT_RETRIES
));
}
let backoff_ms = rate_limit_backoff_ms(retry_after_ms, rate_limit_attempts);
rate_limit_attempts += 1;
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
continue;
}
return Err(format!("GitHub rejected {}: {}", name, detail));
}
}
async fn send_variable_write(
&self,
name: &str,
value: &str,
mode: VariableWriteMode,
headers: HeaderMap,
) -> Result<reqwest::Response, String> {
let payload = serde_json::json!({
"name": name,
"value": value,
});
Ok(match mode {
VariableWriteMode::Create => self
.client
.post(format!(
"{}/repos/{}/{}/environments/{}/variables",
GITHUB_API_BASE, self.owner, self.repo, self.environment
))
.headers(headers)
.json(&payload)
.send()
.await
.map_err(|error| format!("GitHub create failed for {}: {}", name, error))?,
VariableWriteMode::Update => self
.client
.patch(format!(
"{}/repos/{}/{}/environments/{}/variables/{}",
GITHUB_API_BASE, self.owner, self.repo, self.environment, name
))
.headers(headers)
.json(&payload)
.send()
.await
.map_err(|error| format!("GitHub update failed for {}: {}", name, error))?,
})
}
fn auth_headers(&self) -> Result<HeaderMap, String> {
let mut headers = HeaderMap::new();
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {}", self.token))
.map_err(|error| format!("Invalid authorization header: {}", error))?,
);
headers.insert(
ACCEPT,
HeaderValue::from_static("application/vnd.github+json"),
);
headers.insert(
"X-GitHub-Api-Version",
HeaderValue::from_static(GITHUB_API_VERSION),
);
headers.insert(USER_AGENT, HeaderValue::from_static("xbp"));
Ok(headers)
}
}
fn retry_variable_write_mode(
mode: VariableWriteMode,
status: StatusCode,
) -> Option<VariableWriteMode> {
match (mode, status) {
(VariableWriteMode::Create, StatusCode::UNPROCESSABLE_ENTITY) => {
Some(VariableWriteMode::Update)
}
(VariableWriteMode::Update, StatusCode::NOT_FOUND) => Some(VariableWriteMode::Create),
_ => None,
}
}
async fn describe_variable_write_failure(response: reqwest::Response) -> String {
let status = response.status();
let body = response.text().await.unwrap_or_default();
extract_github_error_message(&body).unwrap_or_else(|| format!("HTTP {}", status))
}
fn is_github_rate_limit_detail(status: StatusCode, detail: &str) -> bool {
if status == StatusCode::TOO_MANY_REQUESTS {
return true;
}
if status == StatusCode::FORBIDDEN {
let lower = detail.to_ascii_lowercase();
return lower.contains("rate limit")
|| lower.contains("secondary rate")
|| lower.contains("abuse detection")
|| lower.contains("too many requests");
}
false
}
fn rate_limit_backoff_ms(retry_after_ms: Option<u64>, attempt: u32) -> u64 {
if let Some(ms) = retry_after_ms {
return ms.clamp(100, VARIABLE_WRITE_MAX_BACKOFF_MS);
}
let shift = attempt.min(6);
VARIABLE_WRITE_DEFAULT_BACKOFF_MS
.saturating_mul(1u64 << shift)
.min(VARIABLE_WRITE_MAX_BACKOFF_MS)
}
fn parse_retry_after_ms(headers: &HeaderMap) -> Option<u64> {
let value = headers.get("retry-after")?.to_str().ok()?.trim();
if value.is_empty() {
return None;
}
if let Ok(secs) = value.parse::<u64>() {
return Some(secs.saturating_mul(1000));
}
None
}
fn github_variable_value_to_env_string(value: serde_json::Value) -> String {
match value {
serde_json::Value::String(value) => value,
other => other.to_string(),
}
}
fn raw_variable_entry_to_secret_variable(variable: VariableEntry) -> GitHubSecretVariable {
GitHubSecretVariable {
name: variable.name,
value: github_variable_value_to_env_string(variable.value),
}
}
fn normalize_variable_name_from_github(variable: GitHubSecretVariable) -> GitHubSecretVariable {
GitHubSecretVariable {
name: github_variable_name_to_env_name(&variable.name),
value: variable.value,
}
}
fn variables_for_github(
variables: &HashMap<String, String>,
) -> Result<HashMap<String, String>, String> {
let mut canonicalized = HashMap::<String, (String, String)>::new();
for (name, value) in variables {
let provider_name = env_name_to_github_variable_name(name);
validate_github_variable_name(&provider_name)?;
let canonical_name = canonical_github_variable_name(&provider_name);
match canonicalized.entry(canonical_name) {
Entry::Vacant(entry) => {
entry.insert((provider_name, value.clone()));
}
Entry::Occupied(mut entry) => {
let (existing_name, existing_value) = entry.get_mut();
if existing_value != value {
return Err(format!(
"Local env keys collapse to the same GitHub variable `{}` but have different values: `{}` and `{}`",
existing_name,
existing_name,
provider_name
));
}
if should_prefer_github_variable_name(&provider_name, existing_name) {
*existing_name = provider_name;
}
}
}
}
Ok(canonicalized.into_values().collect())
}
fn validate_github_variable_name(name: &str) -> Result<(), String> {
let mut chars = name.chars();
let Some(first) = chars.next() else {
return Err("GitHub variable name cannot be empty.".to_string());
};
if !(first.is_ascii_alphabetic() || first == '_') {
return Err(format!(
"Invalid GitHub variable name `{name}`: must start with a letter or underscore \
(got leading character U+{:04X}). If this came from a `.env` file, re-save without a UTF-8 BOM.",
first as u32
));
}
if !chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_') {
return Err(format!(
"Invalid GitHub variable name `{name}`: only letters, digits, and underscores are allowed."
));
}
Ok(())
}
fn env_name_to_github_variable_name(name: &str) -> String {
let name = name
.trim()
.trim_start_matches('\u{feff}')
.chars()
.filter(|ch| {
!matches!(
ch,
'\u{200b}' | '\u{200c}' | '\u{200d}' | '\u{2060}' | '\u{feff}'
)
})
.collect::<String>();
name.strip_prefix(GITHUB_ENV_PREFIX)
.map(|suffix| format!("{}{}", GITHUB_STORAGE_PREFIX, suffix))
.unwrap_or(name)
}
fn github_variable_name_to_env_name(name: &str) -> String {
name.strip_prefix(GITHUB_STORAGE_PREFIX)
.map(|suffix| format!("{}{}", GITHUB_ENV_PREFIX, suffix))
.unwrap_or_else(|| name.to_string())
}
fn canonical_github_variable_name(name: &str) -> String {
name.to_ascii_uppercase()
}
fn should_prefer_github_variable_name(candidate: &str, current: &str) -> bool {
match (has_lowercase_ascii(candidate), has_lowercase_ascii(current)) {
(false, true) => true,
(true, false) => false,
_ => candidate < current,
}
}
fn has_lowercase_ascii(value: &str) -> bool {
value.chars().any(|ch| ch.is_ascii_lowercase())
}
fn plan_variable_writes(
secrets: &HashMap<String, String>,
existing_names: &HashMap<String, String>,
) -> Vec<(String, String, VariableWriteMode)> {
let mut plan = secrets
.iter()
.map(|(name, value)| {
if let Some(existing_name) = existing_names.get(&canonical_github_variable_name(name)) {
(
existing_name.clone(),
value.clone(),
VariableWriteMode::Update,
)
} else {
(name.clone(), value.clone(), VariableWriteMode::Create)
}
})
.collect::<Vec<_>>();
plan.sort_by(|left, right| left.0.cmp(&right.0));
plan
}
#[derive(Debug, Deserialize)]
struct ListVariablesResponse {
variables: Vec<VariableEntry>,
}
#[derive(Debug, Deserialize)]
struct VariableEntry {
name: String,
value: serde_json::Value,
}
#[cfg(test)]
mod tests {
use super::{
is_github_rate_limit_detail, parse_retry_after_ms, plan_variable_writes,
rate_limit_backoff_ms, variables_for_github, VariableWriteMode,
};
use reqwest::header::{HeaderMap, HeaderValue};
use reqwest::StatusCode;
use std::collections::HashMap;
#[test]
fn rate_limit_detects_429() {
assert!(is_github_rate_limit_detail(
StatusCode::TOO_MANY_REQUESTS,
"HTTP 429"
));
}
#[test]
fn rate_limit_detects_secondary_403() {
assert!(is_github_rate_limit_detail(
StatusCode::FORBIDDEN,
"You have exceeded a secondary rate limit. Please wait a few minutes."
));
assert!(!is_github_rate_limit_detail(
StatusCode::FORBIDDEN,
"Resource not accessible by integration"
));
}
#[test]
fn rate_limit_backoff_prefers_retry_after() {
assert_eq!(rate_limit_backoff_ms(Some(5_000), 0), 5_000);
assert_eq!(rate_limit_backoff_ms(Some(10), 0), 100); assert_eq!(rate_limit_backoff_ms(None, 0), 1_000);
assert_eq!(rate_limit_backoff_ms(None, 1), 2_000);
assert_eq!(rate_limit_backoff_ms(None, 2), 4_000);
assert_eq!(rate_limit_backoff_ms(None, 10), 60_000); }
#[test]
fn parse_retry_after_seconds_header() {
let mut headers = HeaderMap::new();
headers.insert("retry-after", HeaderValue::from_static("3"));
assert_eq!(parse_retry_after_ms(&headers), Some(3_000));
}
#[test]
fn variables_for_github_strips_bom_from_keys() {
let mut variables = HashMap::new();
variables.insert("\u{feff}MOLLIE_API_KEY".to_string(), "secret".to_string());
let provider = variables_for_github(&variables).expect("bom key should sanitize");
assert_eq!(provider.get("MOLLIE_API_KEY"), Some(&"secret".to_string()));
assert!(!provider.keys().any(|k| k.contains('\u{feff}')));
}
#[test]
fn variables_for_github_dedupes_case_insensitive_aliases() {
let mut variables = HashMap::new();
variables.insert(
"CLOUDFLARE_OAUTH_CLIENT_PUBLISHER".to_string(),
"xbp".to_string(),
);
variables.insert(
"cloudflare_oauth_client_publisher".to_string(),
"xbp".to_string(),
);
let provider = variables_for_github(&variables).expect("dedupe should succeed");
assert_eq!(provider.len(), 1);
assert_eq!(
provider.get("CLOUDFLARE_OAUTH_CLIENT_PUBLISHER"),
Some(&"xbp".to_string())
);
}
#[test]
fn variables_for_github_rejects_case_insensitive_aliases_with_different_values() {
let mut variables = HashMap::new();
variables.insert(
"CLOUDFLARE_OAUTH_CLIENT_PUBLISHER".to_string(),
"one".to_string(),
);
variables.insert(
"cloudflare_oauth_client_publisher".to_string(),
"two".to_string(),
);
let error = variables_for_github(&variables).expect_err("conflict should be rejected");
assert!(error.contains("collapse to the same GitHub variable"));
}
#[test]
fn plan_variable_writes_updates_existing_name_even_when_case_differs() {
let mut secrets = HashMap::new();
secrets.insert(
"CLOUDFLARE_OAUTH_CLIENT_PUBLISHER".to_string(),
"xbp".to_string(),
);
let mut existing_names = HashMap::new();
existing_names.insert(
"CLOUDFLARE_OAUTH_CLIENT_PUBLISHER".to_string(),
"cloudflare_oauth_client_publisher".to_string(),
);
let plan = plan_variable_writes(&secrets, &existing_names);
assert_eq!(
plan,
vec![(
"cloudflare_oauth_client_publisher".to_string(),
"xbp".to_string(),
VariableWriteMode::Update,
)]
);
}
}