crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! Tests for the sdforge-based SDK interface layer.
//!
//! Extracted from `mod.rs` as a file-based `mod tests` so that `#[path]`
//! resolves relative to this file's directory (`src/presentation/sdk/`),
//! which physically exists — inline `mod tests` would create a virtual
//! `src/presentation/sdk/tests/` directory that breaks `..` resolution.

use super::*;
use crate::common::test_helpers::create_test_db_pool;
use crate::domain::auth::ApiKeyScope;
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::services::search_service::SearchServiceTrait;
use crate::i18n::{I18nBundle, Locale};
use crate::presentation::middleware::auth_middleware::AuthState;
use crate::queue::task_queue::TaskQueue;
use axum::Extension;
use axum_test::TestServer;
use serde_json::{json, Value};
use std::sync::Arc;

// `mocks` 模块定义在 `src/presentation/sdk/mocks.rs`,由 `mod.rs` 通过
// `#[cfg(any(test, feature = "test-mocks"))] pub mod mocks;` 暴露。
// unit test 自动通过 `cfg(test)` 启用;integration tests
// (`tests/sdk_api_test.rs`) 通过 `cargo test --features test-mocks` 启用;
// production binary 默认不启用 `test-mocks`,mocks 完全不参与编译。

// Test environment helpers

/// 检查 TEST_DATABASE_URL 是否设置;未设置时测试应早期返回。
/// sdk 测试需要 AuthState,而 AuthState 需要 DbPool,DbPool 构造需要真实 DB URL。
fn skip_if_no_db() -> bool {
    crate::common::test_helpers::skip_if_no_test_db()
}

// `make_db_pool` helper 已集中到 `src/common/test_helpers.rs::create_test_db_pool`,
// 通过 `use crate::common::test_helpers::create_test_db_pool;` 引入。

/// 构造测试用 AuthState,使用随机 team_id 和 api_key_id。
fn make_auth_state() -> AuthState {
    AuthState::new(
        create_test_db_pool(),
        crate::domain::id::Id::generate(),
        crate::domain::id::Id::generate(),
        ApiKeyScope::default(),
    )
}

// Shared mock implementations
//
// Mock structs now live in `src/presentation/sdk/mocks.rs` (compiled only
// under `#[cfg(test)]`). Both this unit-test module and the integration
// tests in `tests/sdk_api_test.rs` import them via the canonical path
// `crate::presentation::sdk::mocks::*` / `crawlrs::presentation::sdk::mocks::*`,
// eliminating the previous `include!` + `use crate as crawlrs;` hack.

use super::mocks::{
    MockCrawlRepository, MockCrawlRepositoryError, MockSearchService, MockSearchServiceError,
    MockTaskQueue, MockTaskQueueError,
};

// TestServer builders

/// 构造测试用 i18n bundle(en-US 固定,断言不受宿主系统语言影响)
fn make_test_i18n() -> (Locale, Arc<I18nBundle>) {
    let locale: Locale = "en-US".parse().unwrap();
    let bundle = Arc::new(I18nBundle::load("en-US", &["en-US", "zh-CN"]).unwrap());
    (locale, bundle)
}

/// 构造 TestServer,注入成功版 mock 服务。
fn make_server_success() -> TestServer {
    let auth_state = make_auth_state();
    let (locale, i18n) = make_test_i18n();
    let app = build_sdk_router()
        .layer(Extension(
            Arc::new(MockSearchService) as Arc<dyn SearchServiceTrait>
        ))
        .layer(Extension(Arc::new(MockTaskQueue) as Arc<dyn TaskQueue>))
        .layer(Extension(
            Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>
        ))
        // SDK crawl 的可选依赖:单测不启用团队黑名单/限流(None 走跳过分支)。
        .layer(Extension(None::<
            Arc<dyn crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository>,
        >))
        .layer(Extension(None::<
            Arc<dyn crate::domain::services::rate_limiting_service::RateLimitingService>,
        >))
        .layer(Extension(auth_state))
        .layer(Extension(locale))
        .layer(Extension(i18n));
    TestServer::new(app)
}

/// 构造 TestServer,注入搜索服务失败版 mock(其他服务用成功版)。
fn make_server_search_error() -> TestServer {
    let auth_state = make_auth_state();
    let (locale, i18n) = make_test_i18n();
    let app = build_sdk_router()
        .layer(Extension(
            Arc::new(MockSearchServiceError) as Arc<dyn SearchServiceTrait>
        ))
        .layer(Extension(Arc::new(MockTaskQueue) as Arc<dyn TaskQueue>))
        .layer(Extension(
            Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>
        ))
        // SDK crawl 的可选依赖:单测不启用团队黑名单/限流(None 走跳过分支)。
        .layer(Extension(None::<
            Arc<dyn crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository>,
        >))
        .layer(Extension(None::<
            Arc<dyn crate::domain::services::rate_limiting_service::RateLimitingService>,
        >))
        .layer(Extension(auth_state))
        .layer(Extension(locale))
        .layer(Extension(i18n));
    TestServer::new(app)
}

/// 构造 TestServer,注入任务队列失败版 mock(其他服务用成功版)。
fn make_server_queue_error() -> TestServer {
    let auth_state = make_auth_state();
    let (locale, i18n) = make_test_i18n();
    let app = build_sdk_router()
        .layer(Extension(
            Arc::new(MockSearchService) as Arc<dyn SearchServiceTrait>
        ))
        .layer(Extension(Arc::new(MockTaskQueueError) as Arc<dyn TaskQueue>))
        .layer(Extension(
            Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>
        ))
        // SDK crawl 的可选依赖:单测不启用团队黑名单/限流(None 走跳过分支)。
        .layer(Extension(None::<
            Arc<dyn crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository>,
        >))
        .layer(Extension(None::<
            Arc<dyn crate::domain::services::rate_limiting_service::RateLimitingService>,
        >))
        .layer(Extension(auth_state))
        .layer(Extension(locale))
        .layer(Extension(i18n));
    TestServer::new(app)
}

/// 构造 TestServer,注入 crawl 仓库失败版 mock(其他服务用成功版)。
fn make_server_crawl_repo_error() -> TestServer {
    let auth_state = make_auth_state();
    let (locale, i18n) = make_test_i18n();
    let app = build_sdk_router()
        .layer(Extension(
            Arc::new(MockSearchService) as Arc<dyn SearchServiceTrait>
        ))
        .layer(Extension(Arc::new(MockTaskQueue) as Arc<dyn TaskQueue>))
        .layer(Extension(
            Arc::new(MockCrawlRepositoryError) as Arc<dyn CrawlRepository>
        ))
        // SDK crawl 的可选依赖:单测不启用团队黑名单/限流(None 走跳过分支)。
        .layer(Extension(None::<
            Arc<dyn crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository>,
        >))
        .layer(Extension(None::<
            Arc<dyn crate::domain::services::rate_limiting_service::RateLimitingService>,
        >))
        .layer(Extension(auth_state))
        .layer(Extension(locale))
        .layer(Extension(i18n));
    TestServer::new(app)
}

// sdk_search tests

/// sdk_search happy path:合法 query 应返回 200 + 搜索结果。
/// 覆盖 SearchQuery 构造、search_service.search 调用、响应映射。
#[tokio::test]
async fn test_sdk_search_happy_path() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "query": "rust web framework",
        "limit": 5
    });
    let res = server.post("/api/v1/sdk/search").json(&body).await;
    res.assert_status_ok();
    let payload: Value = res.json();
    assert_eq!(payload["query"], "rust web framework");
    assert!(payload["results"].is_array());
    assert_eq!(payload["results"][0]["title"], "Mock Title");
    assert_eq!(payload["results"][0]["url"], "https://example.com");
    assert_eq!(payload["results"][0]["engine"], "mock");
    assert_eq!(payload["credits_used"], 1);
}

/// sdk_search 空 query(trim 后为空)应返回 400 InvalidInput。
/// 覆盖 query.trim.is_empty 分支。
#[tokio::test]
async fn test_sdk_search_empty_query_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({ "query": "   " });
    let res = server.post("/api/v1/sdk/search").json(&body).await;
    res.assert_status_bad_request();
}

/// sdk_search 无 query 字段应返回 400(反序列化失败,serde 拒绝缺失必填字段)。
#[tokio::test]
async fn test_sdk_search_missing_query_field_returns_422() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({ "limit": 5 });
    let res = server.post("/api/v1/sdk/search").json(&body).await;
    // axum 对 JSON 反序列化失败(missing field)默认返回 422 Unprocessable Entity
    assert_eq!(
        res.status_code().as_u16(),
        422,
        "Expected status code 422 (Unprocessable Entity) for missing field, got {}",
        res.status_code()
    );
}

/// sdk_search 搜索服务返回错误时应返回 500 Internal。
/// 覆盖 map_err 分支,ApiError::Internal 构造。
#[tokio::test]
async fn test_sdk_search_service_error_returns_500() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_search_error();
    let body = json!({ "query": "test" });
    let res = server.post("/api/v1/sdk/search").json(&body).await;
    assert_eq!(res.status_code().as_u16(), 500);
}

// sdk_create_task tests

/// sdk_create_task happy path(task_type=scrape):应返回 200 + status=Queued。
/// 覆盖 task_type 匹配、Task 构造、enqueue、响应映射。
#[tokio::test]
async fn test_sdk_create_task_scrape_happy_path() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "url": "https://example.com",
        "task_type": "scrape"
    });
    let res = server.post("/api/v1/sdk/tasks").json(&body).await;
    res.assert_status_ok();
    let payload: Value = res.json();
    assert!(payload["id"].is_string());
    assert_eq!(payload["url"], "https://example.com");
    assert_eq!(payload["status"], "Queued");
}

/// sdk_create_task happy path(task_type=crawl):应返回 200 + status=Queued。
/// 覆盖 task_type="crawl" 分支。
#[tokio::test]
async fn test_sdk_create_task_crawl_happy_path() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "url": "https://example.com/crawl",
        "task_type": "crawl"
    });
    let res = server.post("/api/v1/sdk/tasks").json(&body).await;
    res.assert_status_ok();
    let payload: Value = res.json();
    assert!(payload["id"].is_string());
    assert_eq!(payload["url"], "https://example.com/crawl");
    assert_eq!(payload["status"], "Queued");
}

/// sdk_create_task 空 url 应返回 400 InvalidInput。
/// 覆盖 url.trim.is_empty 分支。
#[tokio::test]
async fn test_sdk_create_task_empty_url_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({ "url": "  ", "task_type": "scrape" });
    let res = server.post("/api/v1/sdk/tasks").json(&body).await;
    res.assert_status_bad_request();
}

/// sdk_create_task 无效 task_type 应返回 400 InvalidInput。
/// 覆盖 _ => InvalidInput 分支。
#[tokio::test]
async fn test_sdk_create_task_invalid_type_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "url": "https://example.com",
        "task_type": "invalid_type"
    });
    let res = server.post("/api/v1/sdk/tasks").json(&body).await;
    res.assert_status_bad_request();
}

/// sdk_create_task 队列入队失败应返回 500 Internal。
/// 覆盖 map_err 分支,ApiError::Internal 构造。
#[tokio::test]
async fn test_sdk_create_task_queue_error_returns_500() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_queue_error();
    let body = json!({
        "url": "https://example.com",
        "task_type": "scrape"
    });
    let res = server.post("/api/v1/sdk/tasks").json(&body).await;
    assert_eq!(res.status_code().as_u16(), 500);
}

// sdk_scrape tests

/// sdk_scrape happy path:应返回 200 + id + url。
/// 覆盖 Task 构造、enqueue、响应映射。
#[tokio::test]
async fn test_sdk_scrape_happy_path() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({ "url": "https://example.com/page" });
    let res = server.post("/api/v1/sdk/scrape").json(&body).await;
    res.assert_status_ok();
    let payload: Value = res.json();
    assert!(payload["id"].is_string());
    assert_eq!(payload["url"], "https://example.com/page");
}

/// sdk_scrape 空 url 应返回 400 InvalidInput。
/// 覆盖 url.trim.is_empty 分支。
#[tokio::test]
async fn test_sdk_scrape_empty_url_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({ "url": "" });
    let res = server.post("/api/v1/sdk/scrape").json(&body).await;
    res.assert_status_bad_request();
}

/// sdk_scrape 队列入队失败应返回 500 Internal。
/// 覆盖 map_err 分支,ApiError::Internal 构造。
#[tokio::test]
async fn test_sdk_scrape_queue_error_returns_500() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_queue_error();
    let body = json!({ "url": "https://example.com/page" });
    let res = server.post("/api/v1/sdk/scrape").json(&body).await;
    assert_eq!(res.status_code().as_u16(), 500);
}

// sdk_create_crawl tests

/// sdk_create_crawl happy path:应返回 200 + status=Queued + url。
/// 覆盖 Crawl::new、crawl_repo.create、响应映射。
#[tokio::test]
async fn test_sdk_create_crawl_happy_path() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "name": "test crawl",
        "url": "https://example.com",
        "seed_url": "https://example.com/start"
    });
    let res = server.post("/api/v1/sdk/crawl").json(&body).await;
    res.assert_status_ok();
    let payload: Value = res.json();
    assert!(payload["id"].is_string());
    assert_eq!(payload["status"], "Queued");
    assert_eq!(payload["url"], "https://example.com");
}

/// sdk_create_crawl 空 url 应返回 400 InvalidInput。
/// 覆盖 url.trim.is_empty 分支。
#[tokio::test]
async fn test_sdk_create_crawl_empty_url_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "name": "test crawl",
        "url": "  ",
        "seed_url": "https://example.com/start"
    });
    let res = server.post("/api/v1/sdk/crawl").json(&body).await;
    res.assert_status_bad_request();
}

/// sdk_create_crawl 仓库创建失败应返回 500 Internal。
/// 覆盖 map_err 分支,ApiError::Internal 构造。
#[tokio::test]
async fn test_sdk_create_crawl_repo_error_returns_500() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_crawl_repo_error();
    let body = json!({
        "name": "test crawl",
        "url": "https://example.com",
        "seed_url": "https://example.com/start"
    });
    let res = server.post("/api/v1/sdk/crawl").json(&body).await;
    assert_eq!(res.status_code().as_u16(), 500);
}

/// seed_url 与 url 同口径校验:空值必须 400,而不是带着空 URL 落库。
#[tokio::test]
async fn test_sdk_create_crawl_empty_seed_url_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "name": "test crawl",
        "url": "https://example.com",
        "seed_url": "   "
    });
    let res = server.post("/api/v1/sdk/crawl").json(&body).await;
    res.assert_status_bad_request();
}

/// max_depth 上限与 HTTP 入口同源(MAX_CRAWL_DEPTH=5),6 应 400。
#[tokio::test]
async fn test_sdk_create_crawl_depth_out_of_range_returns_400() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_success();
    let body = json!({
        "name": "test crawl",
        "url": "https://example.com",
        "seed_url": "https://example.com/start",
        "max_depth": 6
    });
    let res = server.post("/api/v1/sdk/crawl").json(&body).await;
    res.assert_status_bad_request();
}

/// 首任务入队失败必须 500,且不得留下 total=1/0 任务的孤儿 crawl。
/// 覆盖 enqueue 失败分支(成功路径的 enqueue 由 happy_path 覆盖)。
#[tokio::test]
async fn test_sdk_create_crawl_queue_error_returns_500() {
    if skip_if_no_db() {
        return;
    }
    let server = make_server_queue_error();
    let body = json!({
        "name": "test crawl",
        "url": "https://example.com",
        "seed_url": "https://example.com/start"
    });
    let res = server.post("/api/v1/sdk/crawl").json(&body).await;
    assert_eq!(res.status_code().as_u16(), 500);
}