1use std::collections::{
2 HashMap,
3 HashSet,
4};
5
6use indexmap::IndexMap;
7
8use super::schema::ProviderFileConfig;
9use super::state::ConfigState;
10use super::token_ref::TokenReference;
11use crate::domain::{
12 DomainResult,
13 ProviderConfig,
14};
15use crate::infrastructure::{
16 ConfigBackend,
17 TokenStore,
18};
19
20const EXPORT_HEADER: &str = "# Pipedash providers export.\n# Tokens are not included. Set each token variable in the environment or enter the token in Settings > Providers.\n\n";
21
22const EXPORT_TOKEN_PREFIX: &str = "PIPEDASH_TOKEN_";
23
24#[derive(Debug, Clone, Default)]
25pub struct SyncResult {
26 pub added: Vec<String>,
27 pub updated: Vec<String>,
28 pub removed: Vec<String>,
29}
30
31pub struct ProviderSyncService;
32
33impl ProviderSyncService {
34 pub fn new() -> Self {
35 Self
36 }
37
38 pub async fn sync_toml_to_db(
39 &self, toml_providers: IndexMap<String, ProviderFileConfig>,
40 config_backend: &dyn ConfigBackend, token_store: &dyn TokenStore, delete_orphans: bool,
41 ) -> DomainResult<SyncResult> {
42 let mut result = SyncResult::default();
43
44 let db_providers = config_backend.list_providers().await?;
45
46 let toml_ids: HashSet<String> = toml_providers.keys().cloned().collect();
47
48 for (id, toml_provider) in toml_providers {
49 let (provider_id, created) = match db_providers.iter().find(|p| p.name == id) {
50 None => {
51 let provider_id = self
52 .create_from_toml(&id, &toml_provider, config_backend)
53 .await?;
54 result.added.push(id.clone());
55 (provider_id, true)
56 }
57 Some(db_provider) => {
58 let Some(db_id) = db_provider.id else {
59 continue;
60 };
61 if !self
62 .update_from_toml(&id, &toml_provider, db_provider, config_backend)
63 .await?
64 {
65 continue;
66 }
67 result.updated.push(id.clone());
68 (db_id, false)
69 }
70 };
71
72 if let Err(e) =
73 Self::store_resolved_token(provider_id, &toml_provider.token, created, token_store)
74 .await
75 {
76 tracing::warn!(
77 provider = %id,
78 error = %e,
79 "Failed to store resolved token - provider may not work until token is re-added"
80 );
81 }
82 }
83
84 if delete_orphans {
85 for db_provider in db_providers {
86 if !toml_ids.contains(&db_provider.name) {
87 if let Some(id) = db_provider.id {
88 config_backend.delete_provider(id).await?;
89 result.removed.push(db_provider.name.clone());
90
91 tracing::info!(
92 provider = %db_provider.name,
93 "Removed provider from database (not in TOML)"
94 );
95 }
96 }
97 }
98 }
99
100 Ok(result)
101 }
102
103 pub(crate) async fn create_from_toml(
104 &self, key: &str, toml_provider: &ProviderFileConfig, config_backend: &dyn ConfigBackend,
105 ) -> DomainResult<i64> {
106 let provider_config =
107 Self::provider_config(key, toml_provider, toml_provider.token.clone());
108 let id = config_backend.create_provider(&provider_config).await?;
109
110 tracing::info!(provider = %key, "Synced provider from TOML to database (new)");
111 Ok(id)
112 }
113
114 pub(crate) async fn update_from_toml(
115 &self, key: &str, toml_provider: &ProviderFileConfig, db_provider: &ProviderConfig,
116 config_backend: &dyn ConfigBackend,
117 ) -> DomainResult<bool> {
118 let Some(db_id) = db_provider.id else {
119 return Ok(false);
120 };
121 if !self.needs_update(toml_provider, db_provider) {
122 return Ok(false);
123 }
124
125 let provider_config = ProviderConfig {
126 id: db_provider.id,
127 version: db_provider.version,
128 ..Self::provider_config(key, toml_provider, toml_provider.token.clone())
129 };
130 config_backend
131 .update_provider(db_id, &provider_config)
132 .await?;
133
134 tracing::info!(provider = %key, "Synced provider from TOML to database (updated)");
135 Ok(true)
136 }
137
138 pub(crate) async fn store_resolved_token(
139 id: i64, token_ref: &str, created: bool, token_store: &dyn TokenStore,
140 ) -> DomainResult<()> {
141 match Self::resolve_token_ref(token_ref) {
142 Some(resolved_token) => token_store.store_token(id, &resolved_token).await,
143 None if created => token_store.store_token(id, "").await,
144 None => Ok(()),
145 }
146 }
147
148 pub(crate) fn provider_config(
149 key: &str, file: &ProviderFileConfig, token: String,
150 ) -> ProviderConfig {
151 let mut config = file.config.clone();
152 if let Some(display_name) = &file.name {
153 config.insert("display_name".to_string(), display_name.clone());
154 }
155
156 ProviderConfig {
157 id: None,
158 name: key.to_string(),
159 provider_type: file.provider_type.clone(),
160 token,
161 config,
162 refresh_interval: file.refresh_interval as i64,
163 version: None,
164 }
165 }
166
167 fn needs_update(&self, toml: &ProviderFileConfig, db: &ProviderConfig) -> bool {
168 if toml.provider_type != db.provider_type
169 || toml.token != db.token
170 || toml.refresh_interval as i64 != db.refresh_interval
171 {
172 return true;
173 }
174
175 let db_display_name = db.config.get("display_name").map(|s| s.as_str());
176 let toml_display_name = toml.name.as_deref();
177 if db_display_name != toml_display_name {
178 return true;
179 }
180
181 let db_config_without_display: std::collections::HashMap<_, _> = db
182 .config
183 .iter()
184 .filter(|(k, _)| *k != "display_name")
185 .map(|(k, v)| (k.clone(), v.clone()))
186 .collect();
187
188 toml.config != db_config_without_display
189 }
190
191 fn resolve_token_ref(token_ref: &str) -> Option<String> {
192 if token_ref.is_empty() {
193 return None;
194 }
195
196 match TokenReference::parse(token_ref) {
197 Ok(TokenReference::EnvVar(var_name)) => match std::env::var(&var_name) {
198 Ok(value) => {
199 tracing::debug!(var_name = %var_name, "Resolved token from environment variable");
200 Some(value)
201 }
202 Err(_) => {
203 tracing::warn!(
204 var_name = %var_name,
205 "Environment variable not found for token reference"
206 );
207 None
208 }
209 },
210 Ok(TokenReference::None) => None,
211 Ok(_) => {
212 tracing::warn!(
213 token_ref = %token_ref,
214 "Unsupported token reference format in TOML config"
215 );
216 None
217 }
218 Err(_) => {
219 tracing::debug!("Using token value directly (not a reference)");
220 Some(token_ref.to_string())
221 }
222 }
223 }
224}
225
226impl Default for ProviderSyncService {
227 fn default() -> Self {
228 Self::new()
229 }
230}
231
232impl ProviderSyncService {
233 pub fn export_toml(db_providers: &[ProviderConfig]) -> DomainResult<String> {
234 let mut sorted: Vec<&ProviderConfig> = db_providers.iter().collect();
235 sorted.sort_by_key(|p| p.id);
236
237 let mut used_vars: HashSet<String> = sorted
238 .iter()
239 .filter_map(|db| Self::preserved_token_var(&db.token))
240 .map(str::to_string)
241 .collect();
242 let mut providers = IndexMap::new();
243
244 for db in sorted {
245 let mut config: HashMap<String, String> = db.config.clone();
246 let name = config.remove("display_name");
247
248 let token = match Self::preserved_token_var(&db.token) {
249 Some(var_name) => format!("${{{var_name}:-}}"),
250 None => Self::generated_token_ref(&db.name, &mut used_vars),
251 };
252
253 providers.insert(
254 db.name.clone(),
255 ProviderFileConfig {
256 name,
257 provider_type: db.provider_type.clone(),
258 token,
259 refresh_interval: u32::try_from(db.refresh_interval)
260 .unwrap_or(super::schema::DEFAULT_REFRESH_INTERVAL_SECS),
261 config,
262 },
263 );
264 }
265
266 let mut doc = toml_edit::DocumentMut::new();
267 ConfigState::write_providers(&mut doc, &providers);
268
269 Ok(format!("{EXPORT_HEADER}{doc}"))
270 }
271
272 fn preserved_token_var(stored: &str) -> Option<&str> {
273 let inner = stored.trim().strip_prefix("${")?.strip_suffix('}')?;
274 let var_name = inner.split(":-").next().unwrap_or(inner);
275 let mut chars = var_name.chars();
276 let first = chars.next()?;
277
278 if !(first.is_ascii_alphabetic() || first == '_')
279 || !chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
280 {
281 return None;
282 }
283
284 Some(var_name)
285 }
286
287 fn generated_token_ref(key: &str, used_vars: &mut HashSet<String>) -> String {
288 let mut suffix = String::new();
289 for c in key.chars() {
290 if c.is_ascii_alphanumeric() {
291 suffix.push(c.to_ascii_uppercase());
292 } else if !suffix.ends_with('_') {
293 suffix.push('_');
294 }
295 }
296 let suffix = suffix.trim_matches('_');
297 let base = format!(
298 "{EXPORT_TOKEN_PREFIX}{}",
299 if suffix.is_empty() {
300 "PROVIDER"
301 } else {
302 suffix
303 }
304 );
305
306 let mut var_name = base.clone();
307 let mut n = 2;
308 while !used_vars.insert(var_name.clone()) {
309 var_name = format!("{base}_{n}");
310 n += 1;
311 }
312
313 format!("${{{var_name}:-}}")
314 }
315}
316
317#[cfg(test)]
318mod tests {
319 use std::sync::Arc;
320
321 use tempfile::TempDir;
322
323 use super::*;
324 use crate::infrastructure::database::SqliteConfigBackend;
325 use crate::infrastructure::MemoryTokenStore;
326
327 async fn create_test_backend() -> (SqliteConfigBackend, Arc<MemoryTokenStore>, TempDir) {
328 let temp_dir = TempDir::new().unwrap();
329 let db_path = temp_dir.path().join("test.db");
330 let pool = crate::infrastructure::database::init_database(db_path)
331 .await
332 .unwrap();
333 let backend = SqliteConfigBackend::new(pool);
334 let token_store = Arc::new(MemoryTokenStore::new());
335 (backend, token_store, temp_dir)
336 }
337
338 #[tokio::test]
339 async fn test_sync_toml_to_db_new_provider() {
340 let (backend, token_store, _temp) = create_test_backend().await;
341 let sync_service = ProviderSyncService::new();
342
343 let mut toml_providers = IndexMap::new();
344 toml_providers.insert(
345 "test-github".to_string(),
346 ProviderFileConfig {
347 name: None,
348 provider_type: "github".to_string(),
349 token: "${GITHUB_TOKEN}".to_string(),
350 refresh_interval: 30,
351 config: Default::default(),
352 },
353 );
354
355 let result = sync_service
356 .sync_toml_to_db(toml_providers, &backend, token_store.as_ref(), true)
357 .await
358 .unwrap();
359
360 assert_eq!(result.added.len(), 1);
361 assert_eq!(result.added[0], "test-github");
362 assert!(result.updated.is_empty());
363 assert!(result.removed.is_empty());
364
365 let db_providers = backend.list_providers().await.unwrap();
366 assert_eq!(db_providers.len(), 1);
367 assert_eq!(db_providers[0].name, "test-github");
368 }
369
370 #[tokio::test]
371 async fn test_sync_toml_to_db_update_provider() {
372 let (backend, token_store, _temp) = create_test_backend().await;
373 let sync_service = ProviderSyncService::new();
374
375 let initial_provider = ProviderConfig {
376 id: None,
377 name: "test-github".to_string(),
378 provider_type: "github".to_string(),
379 token: "${OLD_TOKEN}".to_string(),
380 config: Default::default(),
381 refresh_interval: 30,
382 version: None,
383 };
384 backend.create_provider(&initial_provider).await.unwrap();
385
386 let mut updated_toml = IndexMap::new();
387 updated_toml.insert(
388 "test-github".to_string(),
389 ProviderFileConfig {
390 name: None,
391 provider_type: "github".to_string(),
392 token: "${NEW_TOKEN}".to_string(),
393 refresh_interval: 60,
394 config: Default::default(),
395 },
396 );
397
398 let result = sync_service
399 .sync_toml_to_db(updated_toml, &backend, token_store.as_ref(), true)
400 .await
401 .unwrap();
402
403 assert!(result.added.is_empty());
404 assert_eq!(result.updated.len(), 1);
405 assert_eq!(result.updated[0], "test-github");
406 assert!(result.removed.is_empty());
407
408 let db_providers = backend.list_providers().await.unwrap();
409 assert_eq!(db_providers[0].token, "${NEW_TOKEN}");
410 assert_eq!(db_providers[0].refresh_interval, 60);
411 }
412
413 #[tokio::test]
414 async fn test_sync_toml_to_db_remove_provider() {
415 let (backend, token_store, _temp) = create_test_backend().await;
416 let sync_service = ProviderSyncService::new();
417
418 let initial_provider = ProviderConfig {
419 id: None,
420 name: "test-github".to_string(),
421 provider_type: "github".to_string(),
422 token: "${TOKEN}".to_string(),
423 config: Default::default(),
424 refresh_interval: 30,
425 version: None,
426 };
427 backend.create_provider(&initial_provider).await.unwrap();
428
429 let result = sync_service
430 .sync_toml_to_db(IndexMap::new(), &backend, token_store.as_ref(), true)
431 .await
432 .unwrap();
433
434 assert!(result.added.is_empty());
435 assert!(result.updated.is_empty());
436 assert_eq!(result.removed.len(), 1);
437 assert_eq!(result.removed[0], "test-github");
438
439 let db_providers = backend.list_providers().await.unwrap();
440 assert!(db_providers.is_empty());
441 }
442
443 #[tokio::test]
444 async fn test_sync_toml_to_db_keep_orphans_for_postgres() {
445 let (backend, token_store, _temp) = create_test_backend().await;
446 let sync_service = ProviderSyncService::new();
447
448 let db_only_provider = ProviderConfig {
449 id: None,
450 name: "db-only-provider".to_string(),
451 provider_type: "github".to_string(),
452 token: "${DB_TOKEN}".to_string(),
453 config: Default::default(),
454 refresh_interval: 30,
455 version: None,
456 };
457 backend.create_provider(&db_only_provider).await.unwrap();
458
459 let result = sync_service
460 .sync_toml_to_db(IndexMap::new(), &backend, token_store.as_ref(), false)
461 .await
462 .unwrap();
463
464 assert!(result.added.is_empty());
465 assert!(result.updated.is_empty());
466 assert!(result.removed.is_empty());
467
468 let db_providers = backend.list_providers().await.unwrap();
469 assert_eq!(db_providers.len(), 1);
470 assert_eq!(db_providers[0].name, "db-only-provider");
471 }
472
473 #[tokio::test]
474 async fn test_sync_toml_to_db_postgres_mode_adds_from_toml() {
475 let (backend, token_store, _temp) = create_test_backend().await;
476 let sync_service = ProviderSyncService::new();
477
478 let db_only_provider = ProviderConfig {
479 id: None,
480 name: "db-only-provider".to_string(),
481 provider_type: "github".to_string(),
482 token: "${DB_TOKEN}".to_string(),
483 config: Default::default(),
484 refresh_interval: 30,
485 version: None,
486 };
487 backend.create_provider(&db_only_provider).await.unwrap();
488
489 let mut toml_providers = IndexMap::new();
490 toml_providers.insert(
491 "toml-provider".to_string(),
492 ProviderFileConfig {
493 name: None,
494 provider_type: "gitlab".to_string(),
495 token: "${TOML_TOKEN}".to_string(),
496 refresh_interval: 60,
497 config: Default::default(),
498 },
499 );
500
501 let result = sync_service
502 .sync_toml_to_db(toml_providers, &backend, token_store.as_ref(), false)
503 .await
504 .unwrap();
505
506 assert_eq!(result.added.len(), 1);
507 assert_eq!(result.added[0], "toml-provider");
508 assert!(result.updated.is_empty());
509 assert!(result.removed.is_empty());
510
511 let db_providers = backend.list_providers().await.unwrap();
512 assert_eq!(db_providers.len(), 2);
513 }
514
515 #[tokio::test]
516 async fn test_sync_toml_to_db_stores_an_empty_token_when_none_resolves() {
517 let (backend, token_store, _temp) = create_test_backend().await;
518
519 let mut toml_providers = IndexMap::new();
520 toml_providers.insert(
521 "ci-jenkins".to_string(),
522 ProviderFileConfig {
523 name: Some("CI Jenkins".to_string()),
524 provider_type: "jenkins".to_string(),
525 token: String::new(),
526 refresh_interval: 60,
527 config: Default::default(),
528 },
529 );
530
531 ProviderSyncService::new()
532 .sync_toml_to_db(toml_providers, &backend, token_store.as_ref(), true)
533 .await
534 .unwrap();
535
536 let id = backend.list_providers().await.unwrap()[0].id.unwrap();
537 assert_eq!(token_store.get_token(id).await.unwrap(), "");
538 }
539
540 #[test]
541 fn test_export_toml_generated_var_never_reuses_preserved_var() {
542 let provider = |id: i64, name: &str, token: &str| ProviderConfig {
543 id: Some(id),
544 name: name.to_string(),
545 provider_type: "github".to_string(),
546 token: token.to_string(),
547 config: HashMap::new(),
548 refresh_interval: 30,
549 version: None,
550 };
551 let preserved = provider(1, "other", "${PIPEDASH_TOKEN_PROD}");
552 let literal = provider(2, "prod", "ghp_literal_secret");
553
554 for providers in [
555 [preserved.clone(), literal.clone()],
556 [
557 literal.clone(),
558 ProviderConfig {
559 id: Some(3),
560 ..preserved
561 },
562 ],
563 ] {
564 let exported = ProviderSyncService::export_toml(&providers).unwrap();
565
566 assert!(!exported.contains("ghp_literal_secret"));
567 assert_eq!(exported.matches("${PIPEDASH_TOKEN_PROD:-}").count(), 1);
568 assert!(exported.contains("${PIPEDASH_TOKEN_PROD_2:-}"));
569 }
570 }
571
572 #[test]
573 fn test_export_toml_is_providers_only() {
574 let exported = ProviderSyncService::export_toml(&[]).unwrap();
575
576 assert!(!exported.contains("[general]"));
577 assert!(exported.starts_with("# Pipedash providers export."));
578 }
579}