crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! 应用核心配置
//!
//! 包含数据库、缓存(oxcache)、服务器、速率限制和并发控制等核心配置项

use serde::{Deserialize, Serialize};

/// 数据库配置设置
///
/// 配置数据库连接参数和连接池设置
///
/// # 字段说明
///
/// * `url` - PostgreSQL 数据库连接字符串(敏感信息,仅 crate 可见)
/// * `max_connections` - 连接池中最大连接数
/// * `min_connections` - 连接池中最小连接数
/// * `connect_timeout` - 连接超时时间(秒)
/// * `idle_timeout` - 空闲连接超时时间(秒)
/// * `max_lifetime` - 连接最大存活时间(秒)
/// * `connection_keepalive` - 连接存活检查间隔(秒)
/// * `health_check_interval` - 空闲连接健康检查间隔(秒)
///
/// # 默认值口径
///
/// 上述各键的生效值以 `config/default.toml` `[database]` 节为准;
/// `#[config(default)]` 仅在对应配置键缺失时生效,两者不一致时 toml 优先。
///
/// # 安全提示
///
/// `url` 字段包含数据库连接字符串,可能包含敏感信息(密码等)。
/// 该字段仅对 crate 可见,外部模块应使用 `url()` 方法访问。
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__DATABASE__")]
pub struct DatabaseSettings {
    /// 数据库连接URL (敏感信息)
    /// 注意:此字段包含敏感信息,仅 crate 内部可访问
    pub(crate) url: String,

    /// 最大连接数
    #[config(default = Some(150))]
    pub max_connections: Option<u32>,

    /// 最小连接数
    #[config(default = Some(20))]
    pub min_connections: Option<u32>,

    /// 连接超时时间(秒)
    #[config(default = Some(15))]
    pub connect_timeout: Option<u64>,

    /// 空闲连接超时时间(秒)
    #[config(default = Some(300))]
    pub idle_timeout: Option<u64>,

    /// 连接最大存活时间(秒)
    #[config(default = Some(1800))]
    pub max_lifetime: Option<u64>,

    /// 连接存活检查间隔(秒)
    #[config(default = Some(30))]
    pub connection_keepalive: Option<u64>,

    /// 健康检查间隔(秒)
    #[config(default = Some(60))]
    pub health_check_interval: Option<u64>,

    /// 预编译语句缓存容量(吸收自研 dbnexus `prepare-cache`)
    ///
    /// 池级 prepared statement LRU,命中后跳过 prepare/parse 往返,降低热路径延迟。
    /// `None` 关闭;`Some(n)` 为缓存容量(默认 128)。
    #[config(default = Some(128))]
    pub prepare_cache_capacity: Option<usize>,

    /// 启动期自动执行迁移(按编译 dialect 选择 `migrations/<dialect>/`)。
    ///
    /// 缺省 false:不执行迁移也不报错(既有语义)。true 时 `create_pool`
    /// 装配 `migrations_dir` + `auto_migrate`;未编译任何 db dialect
    /// feature 时启动显性报错(R-multidb-schema-002/003)。
    ///
    /// `serde(default)` 兼容旧 config 文件缺键(同 `fail_closed` 字段先例)。
    #[serde(default)]
    #[config(default = false)]
    pub auto_migrate: bool,
}

impl std::fmt::Debug for DatabaseSettings {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("DatabaseSettings")
            .field("url", &"***REDACTED***")
            .field("max_connections", &self.max_connections)
            .field("min_connections", &self.min_connections)
            .field("connect_timeout", &self.connect_timeout)
            .field("idle_timeout", &self.idle_timeout)
            .field("max_lifetime", &self.max_lifetime)
            .field("connection_keepalive", &self.connection_keepalive)
            .field("health_check_interval", &self.health_check_interval)
            .field("prepare_cache_capacity", &self.prepare_cache_capacity)
            .field("auto_migrate", &self.auto_migrate)
            .finish()
    }
}

impl DatabaseSettings {
    /// 获取数据库连接URL
    ///
    /// # 安全提示
    ///
    /// 此方法返回包含敏感信息的连接字符串,调用者应谨慎处理,
    /// 不要记录到日志或暴露给用户。
    pub fn url(&self) -> &str {
        &self.url
    }
}

/// 服务器配置设置
///
/// 配置 HTTP 服务器的监听参数
///
/// # 字段说明
///
/// * `host` - 服务器监听的主机地址,通常为 "0.0.0.0" 或 "127.0.0.1"
/// * `port` - 服务器监听的端口号,默认 3000
/// * `enable_port_detection` - 是否开启端口嗅探功能
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SERVER__")]
pub struct ServerSettings {
    /// 服务器监听主机地址
    #[config(default = "0.0.0.0".to_string())]
    pub host: String,

    /// 服务器监听端口
    #[config(default = 8899)]
    pub port: u16,

    /// 是否开启端口嗅探功能
    #[config(default = true)]
    pub enable_port_detection: bool,
}

/// WebSocket 连接治理配置(R-ws-task-status-003)
///
/// `/v1/tasks/{id}/ws` 的连接资源上限:超限新连接被拒(429),
/// 空闲超时(无事件可推送)后服务端主动关闭连接。
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__WEBSOCKET__")]
pub struct WebSocketSettings {
    /// 最大并发 ws 连接数
    #[config(default = 100)]
    pub max_connections: usize,

    /// 空闲超时(秒):窗口内无事件可推送即关闭连接
    #[config(default = 300)]
    pub idle_timeout_secs: u64,
}

/// 速率限制配置设置
///
/// 控制 API 请求的速率限制参数
///
/// # 字段说明
///
/// * `enabled` - 是否启用速率限制,默认 true
/// * `default_rpm` - 默认每分钟请求数限制,默认 100
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__RATE_LIMITING__")]
pub struct RateLimitingSettings {
    /// 是否启用速率限制
    #[config(default = true)]
    pub enabled: bool,

    /// 默认每分钟请求数限制
    #[config(default = 100)]
    pub default_rpm: u32,

    /// 突发请求数大小
    #[config(default = 20)]
    pub burst_size: u32,

    /// 匿名请求每 IP 每分钟上限(IP 限流中间件)
    #[config(default = 10)]
    pub anon_ip_rpm: u32,

    /// 携带 Bearer 凭证请求每 IP 每分钟上限(IP 限流中间件;
    /// per-API-key 配额仍由 handler 在认证后执行)
    #[config(default = 600)]
    pub auth_ip_rpm: u32,

    /// 限流计数/封禁记录的存储后端
    ///
    /// - `memory`(默认):进程内存储,单进程语义,重启即清零(历史行为)
    /// - `database`:Postgres 共享存储(仅 db-postgres 构建生效),
    ///   api/worker 双进程共享计数与封禁,重启不丢失
    ///
    /// 非法值在装配时告警并回退 `memory`(限流链路 fail-open 语义)。
    #[config(default = "memory".to_string())]
    pub storage_backend: String,
}

/// 并发控制配置设置
///
/// 控制系统并发度和资源使用的参数
///
/// # 字段说明
///
/// * `default_team_limit` - 每个团队的最大并发任务数,默认 10
/// * `task_lock_duration_seconds` - 任务锁持续时间,防止重复处理,默认 300 秒(5 分钟)
/// * `mem_pressure_threshold` - 内存使用率进入 Pressure 状态的阈值,默认 0.8
/// * `mem_critical_threshold` - 内存使用率进入 Critical 状态的阈值,默认 0.9
/// * `critical_timeout_seconds` - Critical 状态持续多久后触发优雅关闭信号,默认 30 秒
/// * `adaptive_enabled` - 是否启用 AIMD 自适应并发,默认 false
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CONCURRENCY__")]
pub struct ConcurrencySettings {
    /// 默认团队并发限制
    #[config(default = 10)]
    pub default_team_limit: u64,

    /// 任务锁持续时间(秒)
    #[config(default = 300)]
    pub task_lock_duration_seconds: u64,

    /// 内存使用率进入 Pressure 状态的阈值(0.0 - 1.0)
    ///
    /// 调度器在该阈值之上会对新任务执行 `Admission::Defer`。
    /// 必须严格小于 `mem_critical_threshold`。
    #[config(default = 0.8)]
    pub mem_pressure_threshold: f64,

    /// 内存使用率进入 Critical 状态的阈值(0.0 - 1.0)
    ///
    /// 调度器在该阈值之上会对新任务执行 `Admission::Reschedule`,
    /// 持续超过 `critical_timeout_seconds` 后发出优雅关闭信号。
    #[config(default = 0.9)]
    pub mem_critical_threshold: f64,

    /// Critical 状态持续超时(秒)
    ///
    /// 持续处于 Critical 状态超过该时长,调度器会触发优雅关闭信号。
    #[config(default = 30)]
    pub critical_timeout_seconds: u64,

    /// AIMD 自适应并发开关
    ///
    /// - `false`(默认):使用固定并发(`default_team_limit`),行为等同之前
    /// - `true`:每队信号量由 `AdaptiveSemaphore` 承载,`scrape_worker` 成功/失败
    ///   回填 `AIMDController::record_*`,动态 target 推入 `AdaptiveSemaphore::set_target`
    ///
    /// 默认关闭以保留向后兼容;开启后增强固定并发为动态带宽利用。
    #[config(default = false)]
    pub adaptive_enabled: bool,
}

impl ConcurrencySettings {
    /// Validate cross-field invariants.
    ///
    /// Must be called after config loading to catch environment variable misconfiguration.
    /// Panics with a clear message if invariants are violated.
    pub fn validate(&self) {
        assert!(
            self.mem_pressure_threshold > 0.0 && self.mem_pressure_threshold < 1.0,
            "concurrency.mem_pressure_threshold must be in (0.0, 1.0), got {}",
            self.mem_pressure_threshold
        );
        assert!(
            self.mem_critical_threshold > 0.0 && self.mem_critical_threshold < 1.0,
            "concurrency.mem_critical_threshold must be in (0.0, 1.0), got {}",
            self.mem_critical_threshold
        );
        assert!(
            self.mem_pressure_threshold < self.mem_critical_threshold,
            "concurrency.mem_pressure_threshold ({}) must be strictly less than \
             mem_critical_threshold ({})",
            self.mem_pressure_threshold,
            self.mem_critical_threshold
        );
    }
}

/// 数据保留配置设置
///
/// 控制各类数据的保留周期,超期数据由 expiration-worker 定期清理
///
/// # 字段说明
///
/// * `scrape_results_days` - scrape_results 表保留天数,0 表示禁用清理(旧行为:永不删除)
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__RETENTION__")]
pub struct RetentionSettings {
    /// 抓取结果保留天数(0 = 永不清理)
    #[config(default = 30)]
    pub scrape_results_days: i64,
}

#[cfg(test)]
mod tests {
    use crate::config::WorkerCount;

    #[test]
    fn test_worker_count_fixed() {
        let count = WorkerCount::Fixed(10);
        assert_eq!(count.resolve(), 10);
    }

    #[test]
    fn test_worker_count_auto_returns_positive() {
        let count = WorkerCount::Auto("auto".to_string());
        let resolved = count.resolve();
        // Auto 模式应该返回大于 0 的值(基于 CPU 核心数)
        assert!(
            resolved > 0,
            "Auto mode should return positive value, got {}",
            resolved
        );
    }

    #[test]
    fn test_worker_count_default_is_auto() {
        let count = WorkerCount::default();
        // 默认应该是 Auto 模式
        match count {
            WorkerCount::Auto(_) => {} // 正确
            WorkerCount::Fixed(n) => panic!("Expected Auto, got Fixed({})", n),
        }
    }

    // ========== DatabaseSettings ==========

    #[test]
    fn test_database_settings_url() {
        let settings = super::DatabaseSettings {
            url: "postgresql://user:pass@localhost/db".to_string(),
            max_connections: Some(100),
            min_connections: Some(10),
            connect_timeout: Some(10),
            idle_timeout: Some(300),
            max_lifetime: Some(1800),
            connection_keepalive: Some(30),
            health_check_interval: Some(60),
            prepare_cache_capacity: None,
            auto_migrate: false,
        };
        assert_eq!(settings.url(), "postgresql://user:pass@localhost/db");
    }

    #[test]
    fn test_database_settings_debug_redacts_url() {
        let settings = super::DatabaseSettings {
            url: "postgresql://secret:password@localhost/db".to_string(),
            max_connections: Some(100),
            min_connections: Some(10),
            connect_timeout: Some(10),
            idle_timeout: Some(300),
            max_lifetime: Some(1800),
            connection_keepalive: Some(30),
            health_check_interval: Some(60),
            prepare_cache_capacity: None,
            auto_migrate: false,
        };
        let debug_str = format!("{:?}", settings);
        assert!(debug_str.contains("***REDACTED***"));
        assert!(!debug_str.contains("secret:password"));
    }

    // ========== ServerSettings ==========

    #[test]
    fn test_server_settings_construction() {
        let settings = super::ServerSettings {
            host: "127.0.0.1".to_string(),
            port: 8080,
            enable_port_detection: false,
        };
        assert_eq!(settings.host, "127.0.0.1");
        assert_eq!(settings.port, 8080);
        assert!(!settings.enable_port_detection);
    }

    // ========== RateLimitingSettings ==========

    #[test]
    fn test_rate_limiting_settings_construction() {
        let settings = super::RateLimitingSettings {
            enabled: false,
            default_rpm: 200,
            burst_size: 50,
            anon_ip_rpm: 10,
            auth_ip_rpm: 600,
            storage_backend: "memory".to_string(),
        };
        assert!(!settings.enabled);
        assert_eq!(settings.default_rpm, 200);
        assert_eq!(settings.burst_size, 50);
        assert_eq!(settings.storage_backend, "memory");
    }

    // ========== ConcurrencySettings ==========

    #[test]
    fn test_concurrency_settings_construction() {
        let settings = super::ConcurrencySettings {
            default_team_limit: 20,
            task_lock_duration_seconds: 600,
            mem_pressure_threshold: 0.75,
            mem_critical_threshold: 0.85,
            critical_timeout_seconds: 45,
            adaptive_enabled: true,
        };
        assert_eq!(settings.default_team_limit, 20);
        assert_eq!(settings.task_lock_duration_seconds, 600);
        assert!((settings.mem_pressure_threshold - 0.75).abs() < f64::EPSILON);
        assert!((settings.mem_critical_threshold - 0.85).abs() < f64::EPSILON);
        assert_eq!(settings.critical_timeout_seconds, 45);
        assert!(settings.adaptive_enabled);
    }

    /// 内存感知调度器默认阈值符合(pressure=0.8, critical=0.9, timeout=30s)
    #[test]
    fn test_concurrency_settings_memory_defaults() {
        let settings = super::ConcurrencySettings::default();
        assert!((settings.mem_pressure_threshold - 0.8).abs() < f64::EPSILON);
        assert!((settings.mem_critical_threshold - 0.9).abs() < f64::EPSILON);
        assert_eq!(settings.critical_timeout_seconds, 30);
        // pressure 必须严格小于 critical,否则状态机永不进入 Pressure
        assert!(settings.mem_pressure_threshold < settings.mem_critical_threshold);
    }

    /// adaptive_enabled 默认 false(向后兼容)
    ///
    /// 默认关闭 AIMD 自适应并发,行为等同之前的固定并发模式。
    /// 开启后增强为动态带宽利用(每队 AdaptiveSemaphore + AIMDController)。
    #[test]
    fn test_concurrency_settings_adaptive_enabled_default_false() {
        let settings = super::ConcurrencySettings::default();
        assert!(
            !settings.adaptive_enabled,
            "adaptive_enabled must default to false for backward compatibility"
        );
    }
}