1use std::collections::{
2 HashMap,
3 HashSet,
4};
5
6use indexmap::IndexMap;
7
8use super::schema::ProviderFileConfig;
9use super::state::ConfigState;
10use super::token_ref::TokenReference;
11use crate::domain::{
12 DomainResult,
13 ProviderConfig,
14};
15use crate::infrastructure::{
16 ConfigBackend,
17 TokenStore,
18};
19
20const EXPORT_HEADER: &str = "# Pipedash providers export.\n# Tokens are not included. Set each token variable in the environment or enter the token in Settings > Providers.\n\n";
21
22const EXPORT_TOKEN_PREFIX: &str = "PIPEDASH_TOKEN_";
23
24#[derive(Debug, Clone, Default)]
25pub struct SyncResult {
26 pub added: Vec<String>,
27 pub updated: Vec<String>,
28 pub removed: Vec<String>,
29}
30
31pub struct ProviderSyncService;
32
33impl ProviderSyncService {
34 pub fn new() -> Self {
35 Self
36 }
37
38 pub async fn sync_toml_to_db(
39 &self, toml_providers: IndexMap<String, ProviderFileConfig>,
40 config_backend: &dyn ConfigBackend, token_store: &dyn TokenStore, delete_orphans: bool,
41 ) -> DomainResult<SyncResult> {
42 let mut result = SyncResult::default();
43
44 let db_providers = config_backend.list_providers().await?;
45
46 let toml_ids: HashSet<String> = toml_providers.keys().cloned().collect();
47
48 for (id, toml_provider) in toml_providers {
49 let existing_db = db_providers.iter().find(|p| p.name == id);
50
51 match existing_db {
52 None => {
53 let provider_config =
54 Self::provider_config(&id, &toml_provider, toml_provider.token.clone());
55
56 let new_provider_id = config_backend.create_provider(&provider_config).await?;
57
58 if let Some(resolved_token) = Self::resolve_token_ref(&toml_provider.token) {
59 if let Err(e) = token_store
60 .store_token(new_provider_id, &resolved_token)
61 .await
62 {
63 tracing::warn!(
64 provider = %id,
65 error = %e,
66 "Failed to store resolved token - provider may not work until token is re-added"
67 );
68 }
69 }
70
71 result.added.push(id.clone());
72
73 tracing::info!(
74 provider = %id,
75 "Synced provider from TOML to database (new)"
76 );
77 }
78 Some(db_provider) => {
79 if self.needs_update(&toml_provider, db_provider) {
80 let provider_config = ProviderConfig {
81 id: db_provider.id,
82 version: db_provider.version,
83 ..Self::provider_config(
84 &id,
85 &toml_provider,
86 toml_provider.token.clone(),
87 )
88 };
89
90 if let Some(db_id) = db_provider.id {
91 config_backend
92 .update_provider(db_id, &provider_config)
93 .await?;
94
95 if let Some(resolved_token) =
96 Self::resolve_token_ref(&toml_provider.token)
97 {
98 if let Err(e) =
99 token_store.store_token(db_id, &resolved_token).await
100 {
101 tracing::warn!(
102 provider = %id,
103 error = %e,
104 "Failed to store resolved token - provider may not work until token is re-added"
105 );
106 }
107 }
108
109 result.updated.push(id.clone());
110
111 tracing::info!(
112 provider = %id,
113 "Synced provider from TOML to database (updated)"
114 );
115 }
116 }
117 }
118 }
119 }
120
121 if delete_orphans {
122 for db_provider in db_providers {
123 if !toml_ids.contains(&db_provider.name) {
124 if let Some(id) = db_provider.id {
125 config_backend.delete_provider(id).await?;
126 result.removed.push(db_provider.name.clone());
127
128 tracing::info!(
129 provider = %db_provider.name,
130 "Removed provider from database (not in TOML)"
131 );
132 }
133 }
134 }
135 }
136
137 Ok(result)
138 }
139
140 pub(crate) fn provider_config(
141 key: &str, file: &ProviderFileConfig, token: String,
142 ) -> ProviderConfig {
143 let mut config = file.config.clone();
144 if let Some(display_name) = &file.name {
145 config.insert("display_name".to_string(), display_name.clone());
146 }
147
148 ProviderConfig {
149 id: None,
150 name: key.to_string(),
151 provider_type: file.provider_type.clone(),
152 token,
153 config,
154 refresh_interval: file.refresh_interval as i64,
155 version: None,
156 }
157 }
158
159 fn needs_update(&self, toml: &ProviderFileConfig, db: &ProviderConfig) -> bool {
160 if toml.provider_type != db.provider_type
161 || toml.token != db.token
162 || toml.refresh_interval as i64 != db.refresh_interval
163 {
164 return true;
165 }
166
167 let db_display_name = db.config.get("display_name").map(|s| s.as_str());
168 let toml_display_name = toml.name.as_deref();
169 if db_display_name != toml_display_name {
170 return true;
171 }
172
173 let db_config_without_display: std::collections::HashMap<_, _> = db
174 .config
175 .iter()
176 .filter(|(k, _)| *k != "display_name")
177 .map(|(k, v)| (k.clone(), v.clone()))
178 .collect();
179
180 toml.config != db_config_without_display
181 }
182
183 fn resolve_token_ref(token_ref: &str) -> Option<String> {
184 if token_ref.is_empty() {
185 return None;
186 }
187
188 match TokenReference::parse(token_ref) {
189 Ok(TokenReference::EnvVar(var_name)) => match std::env::var(&var_name) {
190 Ok(value) => {
191 tracing::debug!(var_name = %var_name, "Resolved token from environment variable");
192 Some(value)
193 }
194 Err(_) => {
195 tracing::warn!(
196 var_name = %var_name,
197 "Environment variable not found for token reference"
198 );
199 None
200 }
201 },
202 Ok(TokenReference::None) => None,
203 Ok(_) => {
204 tracing::warn!(
205 token_ref = %token_ref,
206 "Unsupported token reference format in TOML config"
207 );
208 None
209 }
210 Err(_) => {
211 tracing::debug!("Using token value directly (not a reference)");
212 Some(token_ref.to_string())
213 }
214 }
215 }
216}
217
218impl Default for ProviderSyncService {
219 fn default() -> Self {
220 Self::new()
221 }
222}
223
224impl ProviderSyncService {
225 pub fn export_toml(db_providers: &[ProviderConfig]) -> DomainResult<String> {
226 let mut sorted: Vec<&ProviderConfig> = db_providers.iter().collect();
227 sorted.sort_by_key(|p| p.id);
228
229 let mut used_vars: HashSet<String> = sorted
230 .iter()
231 .filter_map(|db| Self::preserved_token_var(&db.token))
232 .map(str::to_string)
233 .collect();
234 let mut providers = IndexMap::new();
235
236 for db in sorted {
237 let mut config: HashMap<String, String> = db.config.clone();
238 let name = config.remove("display_name");
239
240 let token = match Self::preserved_token_var(&db.token) {
241 Some(var_name) => format!("${{{var_name}:-}}"),
242 None => Self::generated_token_ref(&db.name, &mut used_vars),
243 };
244
245 providers.insert(
246 db.name.clone(),
247 ProviderFileConfig {
248 name,
249 provider_type: db.provider_type.clone(),
250 token,
251 refresh_interval: u32::try_from(db.refresh_interval)
252 .unwrap_or(super::schema::DEFAULT_REFRESH_INTERVAL_SECS),
253 config,
254 },
255 );
256 }
257
258 let mut doc = toml_edit::DocumentMut::new();
259 ConfigState::write_providers(&mut doc, &providers);
260
261 Ok(format!("{EXPORT_HEADER}{doc}"))
262 }
263
264 fn preserved_token_var(stored: &str) -> Option<&str> {
265 let inner = stored.trim().strip_prefix("${")?.strip_suffix('}')?;
266 let var_name = inner.split(":-").next().unwrap_or(inner);
267 let mut chars = var_name.chars();
268 let first = chars.next()?;
269
270 if !(first.is_ascii_alphabetic() || first == '_')
271 || !chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
272 {
273 return None;
274 }
275
276 Some(var_name)
277 }
278
279 fn generated_token_ref(key: &str, used_vars: &mut HashSet<String>) -> String {
280 let mut suffix = String::new();
281 for c in key.chars() {
282 if c.is_ascii_alphanumeric() {
283 suffix.push(c.to_ascii_uppercase());
284 } else if !suffix.ends_with('_') {
285 suffix.push('_');
286 }
287 }
288 let suffix = suffix.trim_matches('_');
289 let base = format!(
290 "{EXPORT_TOKEN_PREFIX}{}",
291 if suffix.is_empty() {
292 "PROVIDER"
293 } else {
294 suffix
295 }
296 );
297
298 let mut var_name = base.clone();
299 let mut n = 2;
300 while !used_vars.insert(var_name.clone()) {
301 var_name = format!("{base}_{n}");
302 n += 1;
303 }
304
305 format!("${{{var_name}:-}}")
306 }
307}
308
309#[cfg(test)]
310mod tests {
311 use std::sync::Arc;
312
313 use tempfile::TempDir;
314
315 use super::*;
316 use crate::infrastructure::database::SqliteConfigBackend;
317 use crate::infrastructure::MemoryTokenStore;
318
319 async fn create_test_backend() -> (SqliteConfigBackend, Arc<MemoryTokenStore>, TempDir) {
320 let temp_dir = TempDir::new().unwrap();
321 let db_path = temp_dir.path().join("test.db");
322 let pool = crate::infrastructure::database::init_database(db_path)
323 .await
324 .unwrap();
325 let backend = SqliteConfigBackend::new(pool);
326 let token_store = Arc::new(MemoryTokenStore::new());
327 (backend, token_store, temp_dir)
328 }
329
330 #[tokio::test]
331 async fn test_sync_toml_to_db_new_provider() {
332 let (backend, token_store, _temp) = create_test_backend().await;
333 let sync_service = ProviderSyncService::new();
334
335 let mut toml_providers = IndexMap::new();
336 toml_providers.insert(
337 "test-github".to_string(),
338 ProviderFileConfig {
339 name: None,
340 provider_type: "github".to_string(),
341 token: "${GITHUB_TOKEN}".to_string(),
342 refresh_interval: 30,
343 config: Default::default(),
344 },
345 );
346
347 let result = sync_service
348 .sync_toml_to_db(toml_providers, &backend, token_store.as_ref(), true)
349 .await
350 .unwrap();
351
352 assert_eq!(result.added.len(), 1);
353 assert_eq!(result.added[0], "test-github");
354 assert!(result.updated.is_empty());
355 assert!(result.removed.is_empty());
356
357 let db_providers = backend.list_providers().await.unwrap();
358 assert_eq!(db_providers.len(), 1);
359 assert_eq!(db_providers[0].name, "test-github");
360 }
361
362 #[tokio::test]
363 async fn test_sync_toml_to_db_update_provider() {
364 let (backend, token_store, _temp) = create_test_backend().await;
365 let sync_service = ProviderSyncService::new();
366
367 let initial_provider = ProviderConfig {
368 id: None,
369 name: "test-github".to_string(),
370 provider_type: "github".to_string(),
371 token: "${OLD_TOKEN}".to_string(),
372 config: Default::default(),
373 refresh_interval: 30,
374 version: None,
375 };
376 backend.create_provider(&initial_provider).await.unwrap();
377
378 let mut updated_toml = IndexMap::new();
379 updated_toml.insert(
380 "test-github".to_string(),
381 ProviderFileConfig {
382 name: None,
383 provider_type: "github".to_string(),
384 token: "${NEW_TOKEN}".to_string(),
385 refresh_interval: 60,
386 config: Default::default(),
387 },
388 );
389
390 let result = sync_service
391 .sync_toml_to_db(updated_toml, &backend, token_store.as_ref(), true)
392 .await
393 .unwrap();
394
395 assert!(result.added.is_empty());
396 assert_eq!(result.updated.len(), 1);
397 assert_eq!(result.updated[0], "test-github");
398 assert!(result.removed.is_empty());
399
400 let db_providers = backend.list_providers().await.unwrap();
401 assert_eq!(db_providers[0].token, "${NEW_TOKEN}");
402 assert_eq!(db_providers[0].refresh_interval, 60);
403 }
404
405 #[tokio::test]
406 async fn test_sync_toml_to_db_remove_provider() {
407 let (backend, token_store, _temp) = create_test_backend().await;
408 let sync_service = ProviderSyncService::new();
409
410 let initial_provider = ProviderConfig {
411 id: None,
412 name: "test-github".to_string(),
413 provider_type: "github".to_string(),
414 token: "${TOKEN}".to_string(),
415 config: Default::default(),
416 refresh_interval: 30,
417 version: None,
418 };
419 backend.create_provider(&initial_provider).await.unwrap();
420
421 let result = sync_service
422 .sync_toml_to_db(IndexMap::new(), &backend, token_store.as_ref(), true)
423 .await
424 .unwrap();
425
426 assert!(result.added.is_empty());
427 assert!(result.updated.is_empty());
428 assert_eq!(result.removed.len(), 1);
429 assert_eq!(result.removed[0], "test-github");
430
431 let db_providers = backend.list_providers().await.unwrap();
432 assert!(db_providers.is_empty());
433 }
434
435 #[tokio::test]
436 async fn test_sync_toml_to_db_keep_orphans_for_postgres() {
437 let (backend, token_store, _temp) = create_test_backend().await;
438 let sync_service = ProviderSyncService::new();
439
440 let db_only_provider = ProviderConfig {
441 id: None,
442 name: "db-only-provider".to_string(),
443 provider_type: "github".to_string(),
444 token: "${DB_TOKEN}".to_string(),
445 config: Default::default(),
446 refresh_interval: 30,
447 version: None,
448 };
449 backend.create_provider(&db_only_provider).await.unwrap();
450
451 let result = sync_service
452 .sync_toml_to_db(IndexMap::new(), &backend, token_store.as_ref(), false)
453 .await
454 .unwrap();
455
456 assert!(result.added.is_empty());
457 assert!(result.updated.is_empty());
458 assert!(result.removed.is_empty());
459
460 let db_providers = backend.list_providers().await.unwrap();
461 assert_eq!(db_providers.len(), 1);
462 assert_eq!(db_providers[0].name, "db-only-provider");
463 }
464
465 #[tokio::test]
466 async fn test_sync_toml_to_db_postgres_mode_adds_from_toml() {
467 let (backend, token_store, _temp) = create_test_backend().await;
468 let sync_service = ProviderSyncService::new();
469
470 let db_only_provider = ProviderConfig {
471 id: None,
472 name: "db-only-provider".to_string(),
473 provider_type: "github".to_string(),
474 token: "${DB_TOKEN}".to_string(),
475 config: Default::default(),
476 refresh_interval: 30,
477 version: None,
478 };
479 backend.create_provider(&db_only_provider).await.unwrap();
480
481 let mut toml_providers = IndexMap::new();
482 toml_providers.insert(
483 "toml-provider".to_string(),
484 ProviderFileConfig {
485 name: None,
486 provider_type: "gitlab".to_string(),
487 token: "${TOML_TOKEN}".to_string(),
488 refresh_interval: 60,
489 config: Default::default(),
490 },
491 );
492
493 let result = sync_service
494 .sync_toml_to_db(toml_providers, &backend, token_store.as_ref(), false)
495 .await
496 .unwrap();
497
498 assert_eq!(result.added.len(), 1);
499 assert_eq!(result.added[0], "toml-provider");
500 assert!(result.updated.is_empty());
501 assert!(result.removed.is_empty());
502
503 let db_providers = backend.list_providers().await.unwrap();
504 assert_eq!(db_providers.len(), 2);
505 }
506
507 #[test]
508 fn test_export_toml_generated_var_never_reuses_preserved_var() {
509 let provider = |id: i64, name: &str, token: &str| ProviderConfig {
510 id: Some(id),
511 name: name.to_string(),
512 provider_type: "github".to_string(),
513 token: token.to_string(),
514 config: HashMap::new(),
515 refresh_interval: 30,
516 version: None,
517 };
518 let preserved = provider(1, "other", "${PIPEDASH_TOKEN_PROD}");
519 let literal = provider(2, "prod", "ghp_literal_secret");
520
521 for providers in [
522 [preserved.clone(), literal.clone()],
523 [
524 literal.clone(),
525 ProviderConfig {
526 id: Some(3),
527 ..preserved
528 },
529 ],
530 ] {
531 let exported = ProviderSyncService::export_toml(&providers).unwrap();
532
533 assert!(!exported.contains("ghp_literal_secret"));
534 assert_eq!(exported.matches("${PIPEDASH_TOKEN_PROD:-}").count(), 1);
535 assert!(exported.contains("${PIPEDASH_TOKEN_PROD_2:-}"));
536 }
537 }
538
539 #[test]
540 fn test_export_toml_is_providers_only() {
541 let exported = ProviderSyncService::export_toml(&[]).unwrap();
542
543 assert!(!exported.contains("[general]"));
544 assert!(exported.starts_with("# Pipedash providers export."));
545 }
546}