nuwax-sync 0.1.0

Sync GitHub Release assets to Alibaba Cloud OSS
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
use anyhow::{anyhow, Context, Result};
use clap::Parser;
use serde::Deserialize;
use std::env;
use std::path::PathBuf;
use std::time::Instant;
use tracing::{error, info, warn};

use nuwax_common::config::{GitHubConfig, OssConfig, SyncConfig};
use nuwax_common::downloader::{DownloadTask, FileDownloader};
use nuwax_common::github::GitHubClient;
use nuwax_common::oss::AsyncOssClient;

/// 配置文件中的 OSS 部分
#[derive(Debug, Deserialize)]
struct ConfigOss {
    endpoint: Option<String>,
    bucket_name: Option<String>,
    #[serde(default)]
    cdn_domain: Option<String>,
    #[serde(default)]
    path_prefix: Option<String>,
    #[serde(default)]
    upload_timeout_seconds: Option<u64>,
}

/// 配置文件结构
#[derive(Debug, Deserialize)]
struct AppConfig {
    oss: Option<ConfigOss>,
    github: Option<GitHubConfig>,
}

/// 从配置文件加载
fn load_config_file(config_path: &str) -> Result<Option<AppConfig>> {
    let config_file = PathBuf::from(config_path);
    if !config_file.exists() {
        return Ok(None);
    }

    let content = std::fs::read_to_string(&config_file)
        .with_context(|| format!("读取配置文件失败: {}", config_path))?;

    let config: AppConfig = toml::from_str(&content)
        .with_context(|| format!("解析配置文件失败: {}", config_path))?;

    Ok(Some(config))
}

/// GitHub Release 同步工具 - 下载 GitHub Release 并上传到阿里云 OSS
#[derive(Parser, Debug)]
#[command(name = "nuwax-sync")]
#[command(about = "Sync GitHub Release assets to Alibaba Cloud OSS")]
#[command(version)]
struct Args {
    /// 配置文件路径
    #[arg(short = 'C', long, value_name = "FILE", default_value = "config.toml")]
    config: String,

    /// GitHub 仓库 URL,如 https://github.com/owner/repo
    #[arg(short, long, value_name = "URL")]
    repo: String,

    /// 指定 Release tag,不指定则使用 latest
    #[arg(short, long, value_name = "TAG")]
    tag: Option<String>,

    /// OSS 目标目录前缀(默认: 仓库名)
    #[arg(short, long, value_name = "PREFIX")]
    prefix: Option<String>,

    /// 临时下载目录
    #[arg(long, value_name = "DIR", default_value = "./temp_downloads")]
    temp_dir: String,

    /// 并发下载数
    #[arg(short, long, value_name = "N", default_value = "3")]
    concurrent: usize,

    /// 下载重试次数
    #[arg(long, value_name = "N", default_value = "3")]
    retry: u32,

    /// 是否跳过已存在的文件
    #[arg(long, default_value = "true")]
    skip_existing: bool,

    /// 强制重新下载和上传(覆盖已存在文件)
    #[arg(long, default_value = "false")]
    force: bool,
}

/// 从 GitHub URL 解析 owner 和 repo
/// 支持格式:
/// - https://github.com/owner/repo
/// - https://github.com/owner/repo.git
/// - github.com/owner/repo
/// - owner/repo
fn parse_github_url(url: &str) -> Result<(String, String)> {
    let url = url.trim();

    // 移除 .git 后缀
    let url = url.strip_suffix(".git").unwrap_or(url);

    // 尝试解析完整 URL
    if let Some(path_start) = url.find("github.com/") {
        let path = &url[path_start + "github.com/".len()..];
        let parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();

        if parts.len() >= 2 {
            return Ok((parts[0].to_string(), parts[1].to_string()));
        }
    }

    // 尝试解析 owner/repo 格式
    let parts: Vec<&str> = url.split('/').filter(|s| !s.is_empty()).collect();
    if parts.len() == 2 {
        return Ok((parts[0].to_string(), parts[1].to_string()));
    }

    Err(anyhow!(
        "无法解析 GitHub URL: {}。支持的格式: https://github.com/owner/repo 或 owner/repo",
        url
    ))
}

/// 从环境变量构建 GitHub 配置
fn build_github_config(owner: String, repo: String) -> GitHubConfig {
    GitHubConfig {
        api_base_url: env::var("GITHUB_API_URL").unwrap_or_else(|_| "https://api.github.com".to_string()),
        repo_owner: owner,
        repo_name: repo,
        access_token: env::var("GITHUB_TOKEN").unwrap_or_default(),
        monitor_interval_minutes: 10,
        request_timeout_seconds: 30,
    }
}

/// 从配置文件和环境变量构建 OSS 配置
/// 优先级: 环境变量 > 配置文件
fn build_oss_config(config: Option<&AppConfig>) -> Result<OssConfig> {
    let oss_config_file = config.and_then(|c| c.oss.as_ref());

    let access_key_id = env::var("OSS_ACCESS_KEY_ID")
        .map_err(|_| anyhow!("环境变量 OSS_ACCESS_KEY_ID 未设置"))?;
    let access_key_secret = env::var("OSS_ACCESS_KEY_SECRET")
        .map_err(|_| anyhow!("环境变量 OSS_ACCESS_KEY_SECRET 未设置"))?;

    // endpoint: 环境变量 > 配置文件
    let endpoint = env::var("OSS_ENDPOINT")
        .ok()
        .or_else(|| oss_config_file.and_then(|c| c.endpoint.clone()))
        .ok_or_else(|| anyhow!("未找到 OSS_ENDPOINT,请设置环境变量或在配置文件中指定"))?;

    // bucket_name: 环境变量 > 配置文件
    let bucket_name = env::var("OSS_BUCKET_NAME")
        .ok()
        .or_else(|| oss_config_file.and_then(|c| c.bucket_name.clone()))
        .ok_or_else(|| anyhow!("未找到 OSS_BUCKET_NAME,请设置环境变量或在配置文件中指定"))?;

    // cdn_domain: 环境变量 > 配置文件 > 默认空
    let cdn_domain = env::var("OSS_CDN_DOMAIN")
        .ok()
        .or_else(|| oss_config_file.and_then(|c| c.cdn_domain.clone()))
        .filter(|s| !s.is_empty());

    // path_prefix: 环境变量 > 配置文件 > 默认空
    let path_prefix = env::var("OSS_PATH_PREFIX")
        .ok()
        .or_else(|| oss_config_file.and_then(|c| c.path_prefix.clone()))
        .unwrap_or_default();

    // upload_timeout_seconds: 环境变量 > 配置文件 > 默认 3600
    let upload_timeout_seconds = env::var("OSS_UPLOAD_TIMEOUT")
        .ok()
        .and_then(|s| s.parse().ok())
        .or_else(|| oss_config_file.and_then(|c| c.upload_timeout_seconds))
        .unwrap_or(3600);

    Ok(OssConfig {
        endpoint,
        bucket_name,
        access_key_id,
        access_key_secret,
        cdn_domain,
        path_prefix,
        upload_timeout_seconds,
    })
}

/// 构建同步配置
fn build_sync_config(args: &Args) -> SyncConfig {
    SyncConfig {
        temp_download_dir: args.temp_dir.clone(),
        concurrent_downloads: args.concurrent,
        download_retry_count: args.retry,
        auto_sync_enabled: false,
        keep_versions_count: 10,
    }
}

/// 执行同步的主函数
async fn run_sync(args: Args) -> Result<()> {
    let start_time = Instant::now();

    // 1. 解析 GitHub URL
    info!("🔍 解析 GitHub 仓库 URL: {}", args.repo);
    let (owner, repo) = parse_github_url(&args.repo)?;
    info!("✅ 解析成功: owner={}, repo={}", owner, repo);

    // 2. 加载配置文件
    info!("⚙️  加载配置文件: {}...", args.config);
    let config = load_config_file(&args.config)?;
    if config.is_some() {
        info!("✅ 配置文件加载成功");
    } else {
        info!("⚠️  配置文件不存在: {},使用环境变量和默认值", args.config);
    }

    // 3. 从配置和环境变量构建配置
    info!("⚙️  构建客户端配置...");
    let github_config = build_github_config(owner.clone(), repo.clone());
    let oss_config = build_oss_config(config.as_ref())?;
    let sync_config = build_sync_config(&args);

    info!("✅ 配置加载完成");
    info!("   GitHub: {}/{}", github_config.repo_owner, github_config.repo_name);
    info!("   OSS: {}/{}", oss_config.endpoint, oss_config.bucket_name);
    if let Some(cdn) = &oss_config.cdn_domain {
        info!("   CDN: {}", cdn);
    }
    if !oss_config.path_prefix.is_empty() {
        info!("   路径前缀: {}", oss_config.path_prefix);
    }

    // 3. 初始化 GitHub 客户端
    let github_client = GitHubClient::from_config(&github_config)
        .context("初始化 GitHub 客户端失败")?;

    // 4. 获取 Release 信息
    let release = if let Some(tag) = &args.tag {
        info!("📦 获取指定 Release: {}", tag);
        github_client.get_release_by_tag(tag).await?
    } else {
        info!("📦 获取最新 Release...");
        github_client.get_latest_release().await?
    };

    info!("✅ 找到 Release: {} ({})", release.name, release.tag_name);
    info!("   发布时间: {}", release.published_at.map(|d| d.to_string()).unwrap_or_default());
    info!("   Assets 数量: {}", release.assets.len());

    if release.assets.is_empty() {
        warn!("⚠️  该 Release 没有可下载的文件");
        return Ok(());
    }

    // 5. 确定 OSS 上传路径前缀
    let oss_prefix = args.prefix.unwrap_or_else(|| repo.clone());
    let version_tag = release.tag_name.clone();

    info!("☁️  OSS 上传路径: {}/{}/", oss_prefix, version_tag);

    // 6. 初始化下载器
    let downloader = FileDownloader::from_config(&sync_config)
        .context("初始化文件下载器失败")?;

    // 7. 初始化 OSS 客户端
    let oss_client = AsyncOssClient::from_config(&oss_config).await
        .context("初始化 OSS 客户端失败")?;

    // 8. 准备下载任务
    let temp_dir = PathBuf::from(&sync_config.temp_download_dir);
    std::fs::create_dir_all(&temp_dir)?;

    let mut download_tasks = Vec::new();
    let mut skipped_files = Vec::new();

    for asset in &release.assets {
        // 构建 OSS 远程路径
        let remote_path = format!("{}/{}/{}", oss_prefix, version_tag, asset.name);

        // 检查文件是否已存在于 OSS
        if !args.force {
            match oss_client.file_exists(&remote_path).await {
                Ok(true) => {
                    info!("⏭️  文件已存在于 OSS,跳过: {}", asset.name);
                    skipped_files.push(asset.name.clone());
                    continue;
                }
                Ok(false) => {}
                Err(e) => {
                    warn!("⚠️  检查文件存在性失败 {}: {}", asset.name, e);
                }
            }
        }

        // 从 digest 提取 SHA256
        let expected_sha256 = asset.digest.as_ref().map(|d| {
            if d.starts_with("sha256:") {
                d.strip_prefix("sha256:").unwrap_or(d).to_string()
            } else {
                d.clone()
            }
        });

        let task = DownloadTask::from_github_asset(asset, &temp_dir, expected_sha256);
        download_tasks.push(task);
    }

    if download_tasks.is_empty() && !skipped_files.is_empty() {
        info!("✅ 所有文件都已存在于 OSS,无需同步");
        return Ok(());
    }

    info!("📥 需要下载 {} 个文件", download_tasks.len());

    // 9. 下载文件
    let download_results = if !download_tasks.is_empty() {
        info!("🚀 开始下载文件...");
        downloader.download_files(download_tasks).await?
    } else {
        Vec::new()
    };

    // 检查下载结果
    let mut failed_downloads = Vec::new();
    let mut successful_downloads = Vec::new();

    for result in &download_results {
        if result.success {
            successful_downloads.push(result);
            info!("✅ 下载成功: {} ({} bytes)", result.task.filename, result.actual_size);
        } else {
            failed_downloads.push(result);
            error!("❌ 下载失败: {} - {:?}", result.task.filename, result.error_message);
        }
    }

    if !failed_downloads.is_empty() {
        return Err(anyhow!(
            "{} 个文件下载失败",
            failed_downloads.len()
        ));
    }

    // 10. 上传文件到 OSS
    info!("☁️  开始上传文件到 OSS...");
    let mut uploaded_count = 0;
    let mut failed_uploads = Vec::new();

    for result in &successful_downloads {
        let local_path = &result.task.local_path;
        let filename = &result.task.filename;
        let remote_path = format!("{}/{}/{}", oss_prefix, version_tag, filename);
        let content_type = result.task.content_type.as_deref()
            .unwrap_or("application/octet-stream");

        info!("📤 上传: {} -> {}", filename, remote_path);

        match oss_client.upload_file(local_path, &remote_path, content_type).await {
            Ok(url) => {
                info!("✅ 上传成功: {}", url);
                uploaded_count += 1;
            }
            Err(e) => {
                error!("❌ 上传失败 {}: {}", filename, e);
                failed_uploads.push((filename.clone(), e.to_string()));
            }
        }
    }

    // 11. 清理临时文件
    info!("🧹 清理临时文件...");
    if let Err(e) = downloader.cleanup_temp_files().await {
        warn!("⚠️  清理临时文件失败: {}", e);
    }

    // 12. 输出统计信息
    let duration = start_time.elapsed();
    info!("");
    info!("📊 同步完成统计:");
    info!("   总耗时: {:.2} 秒", duration.as_secs_f64());
    info!("   Release: {} ({})", release.name, release.tag_name);
    info!("   跳过文件: {}", skipped_files.len());
    info!("   下载成功: {}", successful_downloads.len());
    info!("   上传成功: {}", uploaded_count);

    if !failed_uploads.is_empty() {
        error!("   上传失败: {}", failed_uploads.len());
        for (file, error) in &failed_uploads {
            error!("      - {}: {}", file, error);
        }
        return Err(anyhow!("{} 个文件上传失败", failed_uploads.len()));
    }

    if !skipped_files.is_empty() {
        info!("   跳过的文件:");
        for file in &skipped_files {
            info!("      - {}", file);
        }
    }

    info!("");
    info!("🎉 同步完成! 文件已上传到: {}/{}/", oss_prefix, version_tag);

    Ok(())
}

#[tokio::main]
async fn main() {
    // 初始化日志
    tracing_subscriber::fmt()
        .with_env_filter(
            env::var("RUST_LOG")
                .unwrap_or_else(|_| "info".to_string()),
        )
        .init();

    let args = Args::parse();

    info!("🚀 GitHub Release Sync Tool");
    info!("   仓库: {}", args.repo);
    if let Some(tag) = &args.tag {
        info!("   版本: {}", tag);
    } else {
        info!("   版本: latest");
    }
    info!("");

    if let Err(e) = run_sync(args).await {
        error!("❌ 同步失败: {}", e);
        std::process::exit(1);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_github_url() {
        // 测试完整 URL
        assert_eq!(
            parse_github_url("https://github.com/owner/repo").unwrap(),
            ("owner".to_string(), "repo".to_string())
        );

        // 测试带 .git 后缀
        assert_eq!(
            parse_github_url("https://github.com/owner/repo.git").unwrap(),
            ("owner".to_string(), "repo".to_string())
        );

        // 测试不带协议
        assert_eq!(
            parse_github_url("github.com/owner/repo").unwrap(),
            ("owner".to_string(), "repo".to_string())
        );

        // 测试 owner/repo 格式
        assert_eq!(
            parse_github_url("owner/repo").unwrap(),
            ("owner".to_string(), "repo".to_string())
        );

        // 测试带尾部斜杠
        assert_eq!(
            parse_github_url("https://github.com/owner/repo/").unwrap(),
            ("owner".to_string(), "repo".to_string())
        );

        // 测试无效 URL
        assert!(parse_github_url("invalid").is_err());
        assert!(parse_github_url("https://github.com/owner").is_err());
    }
}