crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
# Copyright (c) 2025-2026 Kirky.X🌠
# SPDX-License-Identifier: Apache-2.0

# =============================================================================
# Crawlrs 统一 Docker Compose 配置
# =============================================================================
# 支持一键启动完整项目环境,包括:
# - 应用服务 (API + 引擎)
# - 基础设施 (PostgreSQL)
# - 浏览器服务 (Chrome, FlareSolverr)
# - 监控服务 (Prometheus, Grafana)
#
# Feature set 通过 BUILD_FEATURES 构建参数控制(默认: standard)。
# 完整构建:  docker compose build --build-arg BUILD_FEATURES=full
#
# 使用方法:
#   全部启动:     docker compose --profile all up -d
#   仅基础设施:   docker compose --profile infrastructure up -d
#   仅应用:       docker compose --profile app up -d
#   启动监控:     docker compose --profile monitoring up -d
#   独立 Worker:  docker compose --profile worker up -d
#   关闭全部:     docker compose --profile all down
# =============================================================================

services:
  # ==========================================================================
  # 应用服务层
  # ==========================================================================

  # Crawlrs 主应用服务
  crawlrs:
    build:
      context: ..
      dockerfile: docker/Dockerfile
      args:
        BUILD_FEATURES: ${BUILD_FEATURES:-standard}
    image: crawlrs:latest
    container_name: crawlrs
    command: ["api"]
    ports:
      - "${APP_PORT:-8899}:8899"
    environment:
      # 基础配置
      - CRAWLRS__RUST_LOG=${RUST_LOG:-info}
      - CRAWLRS__ENV_MODE=${ENV_MODE:-docker}

      # 代理配置
      - HTTP_PROXY=${HTTP_PROXY:-}
      - HTTPS_PROXY=${HTTPS_PROXY:-}
      - NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,.local,.corp}

      # 数据库配置
      - CRAWLRS__DATABASE__URL=postgres://${DB_USER:-crawlrs}:${DB_PASSWORD:?DB_PASSWORD is required in .env}@${DB_HOST:-db}:5432/${DB_NAME:-crawlrs}
      - CRAWLRS__DATABASE__MAX_CONNECTIONS=${DATABASE_MAX_CONNECTIONS:-20}
      - CRAWLRS__DATABASE__CONNECT_TIMEOUT=60

      # 服务器配置
      - CRAWLRS__SERVER__HOST=0.0.0.0
      - CRAWLRS__SERVER__PORT=8899

      # 服务类型
      - CRAWLRS__SERVICE_TYPE=${SERVICE_TYPE:-api}

      # 搜索引擎配置(SearchEnginesSettings 仅含 bing/baidu/sogou 开关)
      - CRAWLRS__SEARCH__ENGINES__BING_ENABLED=${SEARCH_ENGINE_BING_ENABLED:-true}
      - CRAWLRS__SEARCH__ENGINES__BAIDU_ENABLED=${SEARCH_ENGINE_BAIDU_ENABLED:-true}
      - CRAWLRS__SEARCH__ENGINES__SOGOU_ENABLED=${SEARCH_ENGINE_SOGOU_ENABLED:-true}
      - CRAWLRS__SEARCH__DEFAULT_ENGINE=${SEARCH_ENGINE_DEFAULT:-bing}

      # Google搜索配置
      - CRAWLRS__GOOGLE_SEARCH__API_KEY=${GOOGLE_SEARCH_API_KEY:-}
      - CRAWLRS__GOOGLE_SEARCH__CX=${GOOGLE_SEARCH_CX:-}

      # FlareSolverr配置(引擎侧 [engines.flaresolverr],非 search 侧)
      - CRAWLRS__ENGINES__FLARESOLVERR__URL=http://${FLARESOLVERR_HOST:-flaresolverr}:8191/v1
      - CRAWLRS__ENGINES__FLARESOLVERR__ENABLED=${FLARESOLVERR_ENABLED:-true}

      # 远程浏览器端点:crawlrs 直读无前缀环境变量 CHROMIUM_REMOTE_DEBUGGING_URL
      # (src/infrastructure/services/config_service.rs,供 Playwright 引擎与 BrowserPool 消费)。
      # 此处不注入,需接入 browserless/chrome 容器时显式设置该变量。

      # 速率限制配置
      - CRAWLRS__RATE_LIMITING__ENABLED=${RATE_LIMITING_ENABLED:-false}
      - CRAWLRS__RATE_LIMITING__DEFAULT_RPM=${RATE_LIMITING_DEFAULT_RPM:-60}

      # 认证配置(auth feature 必需,JWT 密钥 ≥32 字节)
      # 不提供默认值:缺 AUTH_JWT_SECRET 时 compose 直接拒绝启动,避免弱默认密钥
      # 被用于任何部署(CWE-1188)。本地开发在 .env 中设置一个 ≥32 字节的随机值。
      - CRAWLRS__AUTH__JWT_SECRET=${AUTH_JWT_SECRET:?AUTH_JWT_SECRET is required in .env}

      # 首次启动自动创建 admin API Key(同进程内 garrison 内存持久化)
      - CRAWLRS_BOOTSTRAP_ADMIN=${BOOTSTRAP_ADMIN:-true}

      # 并发配置
      - CRAWLRS__CONCURRENCY__DEFAULT_TEAM_LIMIT=${CONCURRENCY_DEFAULT_TEAM_LIMIT:-10}
      - CRAWLRS__CONCURRENCY__TASK_LOCK_DURATION=${CONCURRENCY_TASK_LOCK_DURATION:-300}

      # 监控配置
      - CRAWLRS__METRICS__ENABLED=${METRICS_ENABLED:-true}
      - CRAWLRS__METRICS__PORT=8900

    depends_on:
      db:
        condition: service_healthy
    volumes:
      - ../storage:/home/crawlrs/storage
      - ../logs:/home/crawlrs/logs
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:8899/health"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 15s
    networks:
      - crawlrs-network
    profiles:
      - app
      - all

  # ==========================================================================
  # 基础设施层
  # ==========================================================================

  # PostgreSQL 数据库
  db:
    image: postgres:16
    container_name: crawlrs-db
    environment:
      - POSTGRES_USER=${DB_USER:-crawlrs}
      - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required in .env}
      - POSTGRES_DB=${DB_NAME:-crawlrs}
    ports:
      - "${DB_PORT:-5432}:5432"
    volumes:
      - db_data:/var/lib/postgresql/data
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-crawlrs} -d ${DB_NAME:-crawlrs}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    networks:
      - crawlrs-network
    profiles:
      - infrastructure
      - all

  # ==========================================================================
  # 浏览器服务层 (用于搜索引擎爬取)
  # ==========================================================================

  # FlareSolverr (反爬虫解决方案,用于Google搜索)
  flaresolverr:
    image: flaresolverr/flaresolverr:latest
    container_name: crawlrs-flaresolverr
    environment:
      - LOG_LEVEL=${FLARESOLVERR_LOG_LEVEL:-info}
      - CAPTCHA_SOLVER=${FLARESOLVERR_CAPTCHA_SOLVER:-none}
      - TWOCAPTCHA_API_KEY=${TWOCAPTCHA_API_KEY:-}
    ports:
      - "${FLARESOLVERR_PORT:-8191}:8191"
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:8191/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    networks:
      - crawlrs-network
    profiles:
      - browser
      - all

  # Headless Chrome (用于渲染动态网页)
  chrome:
    image: browserless/chrome:latest
    container_name: crawlrs-chrome
    environment:
      - MAX_CONCURRENT_SESSIONS=10
      - CONNECTION_TIMEOUT=30000
    ports:
      - "${CHROME_PORT:-9222}:3000"
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:3000/json/version"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 20s
    networks:
      - crawlrs-network
    profiles:
      - browser
      - all

  # ==========================================================================
  # 监控层
  # ==========================================================================

  # Prometheus 监控
  prometheus:
    image: prom/prometheus:latest
    container_name: crawlrs-prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.enable-lifecycle'
    ports:
      - "${PROMETHEUS_PORT:-9090}:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - crawlrs-network
    profiles:
      - monitoring
      - all

  # Grafana 可视化 (可选)
  grafana:
    image: grafana/grafana:latest
    container_name: crawlrs-grafana
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:?GRAFANA_PASSWORD is required in .env}
    ports:
      - "${GRAFANA_PORT:-3001}:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    restart: unless-stopped
    depends_on:
      - prometheus
    networks:
      - crawlrs-network
    profiles:
      - monitoring
      - all

  # ==========================================================================
  # 工具服务
  # ==========================================================================

  # 数据库管理工具 (pgAdmin - 可选)
  pgadmin:
    image: dpage/pgadmin8:latest
    container_name: crawlrs-pgadmin
    environment:
      - PGADMIN_DEFAULT_EMAIL=admin@crawlrs.local
      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD:?PGADMIN_PASSWORD is required in .env}
    ports:
      - "${PGADMIN_PORT:-5050}:80"
    volumes:
      - pgadmin_data:/var/lib/pgadmin
    restart: unless-stopped
    networks:
      - crawlrs-network
    profiles:
      - tools

  # ==========================================================================
  # Worker 进程(可选,独立于 API 的后台任务处理)
  # ==========================================================================
  worker:
    build:
      context: ..
      dockerfile: docker/Dockerfile
      args:
        BUILD_FEATURES: ${BUILD_FEATURES:-standard}
    image: crawlrs:latest
    container_name: crawlrs-worker
    command: ["worker"]
    environment:
      - CRAWLRS__RUST_LOG=${RUST_LOG:-info}
      - CRAWLRS__ENV_MODE=${ENV_MODE:-docker}
      - CRAWLRS__DATABASE__URL=postgres://${DB_USER:-crawlrs}:${DB_PASSWORD:?DB_PASSWORD is required in .env}@${DB_HOST:-db}:5432/${DB_NAME:-crawlrs}
      - HTTP_PROXY=${HTTP_PROXY:-}
      - HTTPS_PROXY=${HTTPS_PROXY:-}
      - NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,.local,.corp}
    depends_on:
      db:
        condition: service_healthy
    volumes:
      - ../storage:/home/crawlrs/storage
      - ../logs:/home/crawlrs/logs
    restart: unless-stopped
    networks:
      - crawlrs-network
    profiles:
      - worker

# =============================================================================
# 网络配置
# =============================================================================
networks:
  crawlrs-network:
    driver: bridge

# =============================================================================
# 数据卷配置
# =============================================================================
volumes:
  db_data:
  prometheus_data:
  grafana_data:
  pgadmin_data:

# =============================================================================
# 端口分配
# =============================================================================
# - Crawlrs API:  8899 (APP_PORT)
# - PostgreSQL:   5432 (DB_PORT)
# - FlareSolverr: 8191 (FLARESOLVERR_PORT)
# - Chrome:       9222 (CHROME_PORT, 容器内 3000)
# - Prometheus:   9090 (PROMETHEUS_PORT)
# - Grafana:      3001 (GRAFANA_PORT)
# - pgAdmin:      5050 (PGADMIN_PORT)
# =============================================================================