enum ServiceType {
Api,
Worker,
Bootstrap,
Migrate,
}
fn parse_service_type(arg: Option<&str>) -> Result<ServiceType, String> {
let service_type = arg.unwrap_or("api");
match service_type {
"api" => Ok(ServiceType::Api),
"worker" => Ok(ServiceType::Worker),
"bootstrap" => Ok(ServiceType::Bootstrap),
"migrate" => Ok(ServiceType::Migrate),
other => Err(format!(
"Invalid service type: '{}'. Use 'api', 'worker', 'bootstrap', or 'migrate'.",
other
)),
}
}
impl ServiceType {
fn from_args() -> Self {
use log::error;
use std::{env, process};
let args: Vec<String> = env::args_os()
.enumerate()
.map(|(i, os)| match os.into_string() {
Ok(s) => s,
Err(utf8_err) => {
eprintln!(
"crawlrs: argument {} contains invalid UTF-8: {:?}",
i, utf8_err
);
process::exit(2);
}
})
.collect();
let arg = args.get(1).map(String::as_str);
match parse_service_type(arg) {
Ok(st) => st,
Err(msg) => {
error!("{}", msg);
process::exit(1);
}
}
}
}
async fn shutdown_signal() {
#[cfg(unix)]
{
use tokio::signal::unix::{signal, SignalKind};
let mut sigterm = signal(SignalKind::terminate()).expect("install SIGTERM handler");
let mut sigint = signal(SignalKind::interrupt()).expect("install SIGINT handler");
tokio::select! {
_ = sigterm.recv() => {
log::info!("SIGTERM received, draining API server");
}
_ = sigint.recv() => {
log::info!("SIGINT received, draining API server");
}
}
}
#[cfg(not(unix))]
{
tokio::signal::ctrl_c()
.await
.expect("install Ctrl+C handler");
log::info!("Ctrl+C received, draining API server");
}
}
mod app {
use super::ServiceType;
use crawlrs::bootstrap::routes::build_api_app_with_state;
use crawlrs::bootstrap::workers::spawn_common_workers;
use crawlrs::di::modules::{
CacheModule, DatabaseModule, EngineModule, HttpModule, IdModule, InfrastructureModule,
RepositoryModule, ServiceModule, SettingsModule,
};
use crawlrs::di::{CrawlRsState, CrawlRsStateExt};
use crawlrs::workers::manager::{WorkerManager, WorkerManagerConfig, WorkerManagerDeps};
use crawlrs::workers::shutdown::ShutdownCoordinator;
use std::env;
use std::sync::Arc;
use tokio::net::TcpListener;
use trait_kit::AsyncKit;
async fn start_api_service(
app_state: &CrawlRsState,
settings: Arc<crawlrs::config::settings::Settings>,
) -> anyhow::Result<()> {
log::info!("Starting API service...");
if env::var("CRAWLRS_BOOTSTRAP_ADMIN")
.map(|v| v.eq_ignore_ascii_case("true") || v.eq_ignore_ascii_case("1"))
.unwrap_or(false)
{
log::info!("CRAWLRS_BOOTSTRAP_ADMIN=true, running auto-bootstrap...");
run_bootstrap(app_state, &settings).await?;
}
let _common_worker_handles = spawn_common_workers(app_state, &settings).await;
let app = build_api_app_with_state(app_state, settings.clone());
let addr = format!("{}:{}", settings.server.host, settings.server.port);
let listener = TcpListener::bind(&addr).await?;
log::info!("Server listening on {}", addr);
let gateway_for_shutdown = app_state.fluxweir_gateway();
axum::serve(
listener,
app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.with_graceful_shutdown(async move {
crate::shutdown_signal().await;
gateway_for_shutdown.shutdown().await;
})
.await?;
Ok(())
}
async fn start_worker_service(
app_state: &CrawlRsState,
settings: Arc<crawlrs::config::settings::Settings>,
http_client: Arc<reqwest::Client>,
) -> anyhow::Result<()> {
log::info!("Starting Worker service...");
let coordinator = Arc::new(ShutdownCoordinator::new(std::time::Duration::from_secs(
settings.workers.graceful_shutdown_seconds,
)));
spawn_shutdown_signal_listener(coordinator.clone());
let _common_worker_handles = spawn_common_workers(app_state, &settings).await;
let deps = build_worker_manager_deps(app_state, http_client, coordinator.clone());
let config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: settings.concurrency.default_team_limit as usize,
};
let mut worker_manager = WorkerManager::new(deps, config);
let worker_count = settings.workers.count.resolve();
log::info!("Starting {} worker(s)", worker_count);
worker_manager.start_workers(worker_count).await;
let worker_ids = worker_manager.worker_ids();
worker_manager.wait_for_shutdown().await;
log::info!("Shutting down worker service...");
register_worker_shutdown_hooks(&coordinator, app_state, &settings, worker_ids);
coordinator.run_phased_shutdown().await;
Ok(())
}
fn spawn_shutdown_signal_listener(coordinator: Arc<ShutdownCoordinator>) {
tokio::spawn(async move {
if let Err(e) = crawlrs::workers::shutdown::listen_unix_signals(coordinator).await {
log::error!("Failed to listen for shutdown signals: {}", e);
}
});
}
fn build_worker_manager_deps(
app_state: &CrawlRsState,
http_client: Arc<reqwest::Client>,
coordinator: Arc<ShutdownCoordinator>,
) -> WorkerManagerDeps {
WorkerManagerDeps {
queue: app_state.task_queue(),
repository: app_state.task_repo(),
result_repository: app_state.result_repo(),
crawl_repository: app_state.crawl_repo(),
webhook_service: app_state.webhook_service(),
credits_repository: app_state.credits_repo(),
engine_client: app_state.engine_client(),
create_scrape_use_case: app_state.create_scrape_use_case(),
team_semaphore: app_state.team_semaphore.clone(),
request_coalescer: app_state.request_coalescer.clone(),
robots_checker: app_state.robots_checker.clone(),
http_client,
extraction_service: app_state.extraction_service(),
content_extractor: app_state.content_extractor(),
cache_service: app_state.cache_service(),
shutdown_coordinator: coordinator,
searxng_instances: app_state.searxng_instances(),
}
}
fn register_worker_shutdown_hooks(
coordinator: &ShutdownCoordinator,
app_state: &CrawlRsState,
settings: &crawlrs::config::settings::Settings,
worker_ids: Vec<crawlrs::domain::id::Id>,
) {
let repo_for_rollback = app_state.task_repo();
let rollback_graceful =
std::time::Duration::from_secs(settings.workers.graceful_shutdown_seconds);
coordinator.register_shutdown_hook(
crawlrs::workers::shutdown::ShutdownPhase::DrainQueue,
move || {
Box::pin(async move {
crawlrs::workers::shutdown::rollback_pending_tasks(
&repo_for_rollback,
rollback_graceful,
&worker_ids,
)
.await;
})
},
);
let gateway_for_worker = app_state.fluxweir_gateway();
coordinator.register_shutdown_hook(
crawlrs::workers::shutdown::ShutdownPhase::CloseConnections,
move || {
Box::pin(async move {
gateway_for_worker.shutdown().await;
})
},
);
}
async fn run_bootstrap(
app_state: &CrawlRsState,
_settings: &Arc<crawlrs::config::settings::Settings>,
) -> anyhow::Result<()> {
use crawlrs::common::constants::default_identity::DEFAULT_TEAM_ID;
let pool = app_state.db_pool();
let session = pool
.get_session("admin")
.await
.map_err(|e| anyhow::anyhow!("db session: {}", e))?;
let conn = session
.connection()
.map_err(|e| anyhow::anyhow!("db conn: {}", e))?;
let team_id = DEFAULT_TEAM_ID;
ensure_default_team(conn, team_id).await?;
#[cfg(feature = "auth")]
{
create_bootstrap_admin_key(conn, team_id).await?;
}
#[cfg(not(feature = "auth"))]
{
println!("Auth feature is not enabled in this build.");
println!("Rebuild with default features to enable authentication.");
}
Ok(())
}
async fn ensure_default_team(
conn: &dbnexus::sea_orm::DatabaseConnection,
team_id: crawlrs::domain::id::Id,
) -> anyhow::Result<()> {
use crawlrs::infrastructure::database::entities::team;
use dbnexus::sea_orm::{EntityTrait, Set};
let existing = team::Entity::find_by_id(team_id)
.one(conn)
.await
.map_err(|e| anyhow::anyhow!("find team: {}", e))?;
if existing.is_none() {
let now = chrono::Utc::now();
let model = team::ActiveModel {
id: Set(team_id.into()),
name: Set("default-team".to_string()),
allowed_countries: Set(None),
blocked_countries: Set(None),
ip_whitelist: Set(None),
domain_blacklist: Set(None),
enable_geo_restrictions: Set(false),
created_at: Set(now.into()),
updated_at: Set(now.into()),
};
team::Entity::insert(model)
.exec(conn)
.await
.map_err(|e| anyhow::anyhow!("insert team: {}", e))?;
println!("Created team: {}", team_id);
} else {
println!("Team already exists: {}", team_id);
}
Ok(())
}
#[cfg(feature = "auth")]
async fn create_bootstrap_admin_key(
conn: &dbnexus::sea_orm::DatabaseConnection,
team_id: crawlrs::domain::id::Id,
) -> anyhow::Result<()> {
let (api_key_id, plaintext_key) = provision_admin_key_row(conn, team_id).await?;
print_bootstrap_admin_key(api_key_id, team_id, &plaintext_key);
Ok(())
}
#[cfg(feature = "auth")]
async fn provision_admin_key_row(
conn: &dbnexus::sea_orm::DatabaseConnection,
team_id: crawlrs::domain::id::Id,
) -> anyhow::Result<(crawlrs::domain::id::Id, String)> {
use crawlrs::infrastructure::database::entities::api_key;
use dbnexus::sea_orm::{EntityTrait, Set};
use garrison::protocol::apikey::ApiKeyHandler;
let dao = crawlrs::infrastructure::auth::garrison_dao::get_garrison_dao()
.ok_or_else(|| anyhow::anyhow!("garrison DAO not initialized"))?;
let handler = ApiKeyHandler::new(dao);
let api_key_id = crawlrs::domain::id::Id::generate();
let login_id = api_key_id.to_string();
let scopes = vec![
"crawlrs:read".to_string(),
"crawlrs:write".to_string(),
"crawlrs:admin".to_string(),
];
let expires_in_secs: i64 = 365 * 24 * 3600;
let plaintext_key = handler
.generate_with_namespace(login_id, "crawlrs", scopes, expires_in_secs)
.await
.map_err(|e| anyhow::anyhow!("garrison generate key: {}", e))?;
let garrison_key_id = match plaintext_key.split_once('.') {
Some((k_id, _)) => k_id.to_string(),
None => return Err(anyhow::anyhow!("garrison returned malformed key")),
};
let now_ts = chrono::Utc::now();
let key_model = api_key::ActiveModel {
id: Set(api_key_id.into()),
team_id: Set(team_id.into()),
key: Set(garrison_key_id),
created_at: Set(now_ts.into()),
updated_at: Set(None),
};
api_key::Entity::insert(key_model)
.exec(conn)
.await
.map_err(|e| anyhow::anyhow!("insert api_key: {}", e))?;
Ok((api_key_id, plaintext_key))
}
#[cfg(feature = "auth")]
fn print_bootstrap_admin_key(
api_key_id: crawlrs::domain::id::Id,
team_id: crawlrs::domain::id::Id,
plaintext_key: &str,
) {
use std::io::IsTerminal;
if std::io::stdout().is_terminal() {
println!("Created admin API key:");
println!(" api_key_id: {}", api_key_id);
println!(" team_id: {}", team_id);
println!(" api_key: {}", plaintext_key);
println!(" scopes: read, write, admin");
println!();
println!("Use this key as Bearer token:");
println!(
" curl -H 'Authorization: Bearer {}' http://localhost:8899/v1/teams/me",
plaintext_key
);
} else {
println!("Created admin API key:");
println!(" api_key_id: {}", api_key_id);
println!(" team_id: {}", team_id);
println!(
" api_key: {} (masked; rerun in a terminal to display the full key)",
crawlrs::common::mask_secret(plaintext_key)
);
println!(" scopes: read, write, admin");
}
}
async fn start_migrate_service(app_state: &CrawlRsState) -> anyhow::Result<()> {
use crawlrs::infrastructure::database::dbnexus_connection::{
dialect_migrations_dir, MIGRATIONS_DIR_ENV,
};
let base = std::path::PathBuf::from(
env::var(MIGRATIONS_DIR_ENV).unwrap_or_else(|_| "migrations".to_string()),
);
let dir = dialect_migrations_dir(&base).ok_or_else(|| {
anyhow::anyhow!(
"no db dialect feature (db-postgres/db-sqlite/db-mysql) compiled; \
cannot select migrations/<dialect>/ under {}",
base.display()
)
})?;
if !dir.is_dir() {
anyhow::bail!(
"migrations directory not found: {} (set {MIGRATIONS_DIR_ENV} to override the base dir)",
dir.display()
);
}
log::info!("Running database migrations from {}...", dir.display());
let applied =
crawlrs::infrastructure::database::dbnexus_connection::run_crawlrs_migrations(
&app_state.db_pool(),
true,
Some(&base),
)
.await
.map_err(|e| anyhow::anyhow!("migration failed: {e}"))?;
log::info!("Database migrations complete: {applied} applied");
println!("Migrations applied: {applied}");
Ok(())
}
pub(crate) async fn run() -> anyhow::Result<()> {
let settings = load_runtime_settings()?;
let _logger_manager = crawlrs::bootstrap::telemetry::init_all(&settings.logging)
.await
.map_err(|e| anyhow::anyhow!("Failed to initialize inklog logger: {}", e))?;
log::info!("Initializing application dependencies...");
let mut kit = AsyncKit::new();
kit.set_config(settings.clone());
register_core_modules(&mut kit)?;
let kit = kit
.build()
.await
.map_err(|e| anyhow::anyhow!("build AsyncKit: {e}"))?;
let app_state = CrawlRsState::from_kit(&kit)?;
let http_client = kit
.require::<HttpModule>()
.map_err(|e| anyhow::anyhow!("require HttpModule: {e}"))?;
log::info!("Application dependencies initialized successfully");
dispatch_service(&app_state, settings, http_client).await
}
fn load_runtime_settings() -> anyhow::Result<Arc<crawlrs::config::settings::Settings>> {
let is_production = env::var("CRAWLRS_ENV")
.map(|v| v.eq_ignore_ascii_case("production") || v.eq_ignore_ascii_case("prod"))
.unwrap_or(false);
let (settings, _port) = crawlrs::bootstrap::config::load_and_configure(is_production)?;
let settings = Arc::new(settings);
if let Some(bundle) = crawlrs::i18n::build_startup_bundle(&settings.i18n) {
crawlrs::i18n::init_startup_i18n(bundle);
}
Ok(settings)
}
fn register_core_modules(kit: &mut AsyncKit) -> anyhow::Result<()> {
kit.register::<SettingsModule>()
.map_err(|e| anyhow::anyhow!("register SettingsModule: {e}"))?;
kit.register::<DatabaseModule>()
.map_err(|e| anyhow::anyhow!("register DatabaseModule: {e}"))?;
kit.register::<HttpModule>()
.map_err(|e| anyhow::anyhow!("register HttpModule: {e}"))?;
kit.register::<CacheModule>()
.map_err(|e| anyhow::anyhow!("register CacheModule: {e}"))?;
kit.register::<RepositoryModule>()
.map_err(|e| anyhow::anyhow!("register RepositoryModule: {e}"))?;
kit.register::<EngineModule>()
.map_err(|e| anyhow::anyhow!("register EngineModule: {e}"))?;
kit.register::<InfrastructureModule>()
.map_err(|e| anyhow::anyhow!("register InfrastructureModule: {e}"))?;
kit.register::<ServiceModule>()
.map_err(|e| anyhow::anyhow!("register ServiceModule: {e}"))?;
kit.register::<IdModule>().map_err(|e| {
anyhow::anyhow!(
"{}: {e}",
crawlrs::i18n::tr_log("boot-register-id-module-failed")
)
})?;
Ok(())
}
async fn dispatch_service(
app_state: &CrawlRsState,
settings: Arc<crawlrs::config::settings::Settings>,
http_client: Arc<reqwest::Client>,
) -> anyhow::Result<()> {
match ServiceType::from_args() {
ServiceType::Api => {
start_api_service(app_state, settings).await?;
}
ServiceType::Worker => {
start_worker_service(app_state, settings, http_client).await?;
}
ServiceType::Bootstrap => {
run_bootstrap(app_state, &settings).await?;
}
ServiceType::Migrate => {
start_migrate_service(app_state).await?;
}
}
Ok(())
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
app::run().await
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tc_parse_service_type_api() {
let result = parse_service_type(Some("api"));
assert!(matches!(result, Ok(ServiceType::Api)));
}
#[test]
fn tc_parse_service_type_worker() {
let result = parse_service_type(Some("worker"));
assert!(matches!(result, Ok(ServiceType::Worker)));
}
#[test]
fn tc_parse_service_type_none_defaults_to_api() {
let result = parse_service_type(None);
assert!(matches!(result, Ok(ServiceType::Api)));
}
#[test]
fn tc_parse_service_type_migrate() {
let result = parse_service_type(Some("migrate"));
assert!(matches!(result, Ok(ServiceType::Migrate)));
}
#[test]
fn tc_parse_service_type_bootstrap() {
let result = parse_service_type(Some("bootstrap"));
assert!(matches!(result, Ok(ServiceType::Bootstrap)));
}
#[test]
fn tc_parse_service_type_invalid_returns_error() {
let result = parse_service_type(Some("invalid"));
match result {
Err(msg) => {
assert!(msg.contains("Invalid service type"), "got: {}", msg);
assert!(msg.contains("invalid"), "got: {}", msg);
assert!(
msg.contains("api") || msg.contains("worker"),
"got: {}",
msg
);
}
Ok(_) => panic!("expected error for invalid service type, got Ok"),
}
}
#[test]
fn tc_parse_service_type_empty_string_returns_error() {
let result = parse_service_type(Some(""));
match result {
Err(msg) => {
assert!(msg.contains("Invalid service type"), "got: {}", msg);
}
Ok(_) => panic!("expected error for empty string, got Ok"),
}
}
#[test]
fn tc_parse_service_type_case_sensitive() {
let result = parse_service_type(Some("API"));
assert!(result.is_err(), "uppercase API should be invalid");
let result = parse_service_type(Some("Worker"));
assert!(result.is_err(), "mixed-case Worker should be invalid");
}
}