services:
crawlrs:
build:
context: ..
dockerfile: docker/Dockerfile
args:
BUILD_FEATURES: ${BUILD_FEATURES:-standard}
image: crawlrs:latest
container_name: crawlrs
command: ["api"]
ports:
- "${APP_PORT:-8899}:8899"
environment:
- CRAWLRS__RUST_LOG=${RUST_LOG:-info}
- CRAWLRS__ENV_MODE=${ENV_MODE:-docker}
- HTTP_PROXY=${HTTP_PROXY:-}
- HTTPS_PROXY=${HTTPS_PROXY:-}
- NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,.local,.corp}
- CRAWLRS__DATABASE__URL=postgres://${DB_USER:-crawlrs}:${DB_PASSWORD:?DB_PASSWORD is required in .env}@${DB_HOST:-db}:5432/${DB_NAME:-crawlrs}
- CRAWLRS__DATABASE__MAX_CONNECTIONS=${DATABASE_MAX_CONNECTIONS:-20}
- CRAWLRS__DATABASE__CONNECT_TIMEOUT=60
- CRAWLRS__SERVER__HOST=0.0.0.0
- CRAWLRS__SERVER__PORT=8899
- CRAWLRS__SERVICE_TYPE=${SERVICE_TYPE:-api}
- CRAWLRS__SEARCH__ENGINES__BING_ENABLED=${SEARCH_ENGINE_BING_ENABLED:-true}
- CRAWLRS__SEARCH__ENGINES__BAIDU_ENABLED=${SEARCH_ENGINE_BAIDU_ENABLED:-true}
- CRAWLRS__SEARCH__ENGINES__SOGOU_ENABLED=${SEARCH_ENGINE_SOGOU_ENABLED:-true}
- CRAWLRS__SEARCH__DEFAULT_ENGINE=${SEARCH_ENGINE_DEFAULT:-bing}
- CRAWLRS__GOOGLE_SEARCH__API_KEY=${GOOGLE_SEARCH_API_KEY:-}
- CRAWLRS__GOOGLE_SEARCH__CX=${GOOGLE_SEARCH_CX:-}
- CRAWLRS__ENGINES__FLARESOLVERR__URL=http://${FLARESOLVERR_HOST:-flaresolverr}:8191/v1
- CRAWLRS__ENGINES__FLARESOLVERR__ENABLED=${FLARESOLVERR_ENABLED:-true}
- CRAWLRS__RATE_LIMITING__ENABLED=${RATE_LIMITING_ENABLED:-false}
- CRAWLRS__RATE_LIMITING__DEFAULT_RPM=${RATE_LIMITING_DEFAULT_RPM:-60}
- CRAWLRS__AUTH__JWT_SECRET=${AUTH_JWT_SECRET:?AUTH_JWT_SECRET is required in .env}
- CRAWLRS_BOOTSTRAP_ADMIN=${BOOTSTRAP_ADMIN:-true}
- CRAWLRS__CONCURRENCY__DEFAULT_TEAM_LIMIT=${CONCURRENCY_DEFAULT_TEAM_LIMIT:-10}
- CRAWLRS__CONCURRENCY__TASK_LOCK_DURATION=${CONCURRENCY_TASK_LOCK_DURATION:-300}
- CRAWLRS__METRICS__ENABLED=${METRICS_ENABLED:-true}
- CRAWLRS__METRICS__PORT=8900
depends_on:
db:
condition: service_healthy
volumes:
- ../storage:/home/crawlrs/storage
- ../logs:/home/crawlrs/logs
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:8899/health"]
interval: 30s
timeout: 5s
retries: 3
start_period: 15s
networks:
- crawlrs-network
profiles:
- app
- all
db:
image: postgres:16
container_name: crawlrs-db
environment:
- POSTGRES_USER=${DB_USER:-crawlrs}
- POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required in .env}
- POSTGRES_DB=${DB_NAME:-crawlrs}
ports:
- "${DB_PORT:-5432}:5432"
volumes:
- db_data:/var/lib/postgresql/data
restart: unless-stopped
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-crawlrs} -d ${DB_NAME:-crawlrs}"]
interval: 10s
timeout: 5s
retries: 5
start_period: 10s
networks:
- crawlrs-network
profiles:
- infrastructure
- all
flaresolverr:
image: flaresolverr/flaresolverr:latest
container_name: crawlrs-flaresolverr
environment:
- LOG_LEVEL=${FLARESOLVERR_LOG_LEVEL:-info}
- CAPTCHA_SOLVER=${FLARESOLVERR_CAPTCHA_SOLVER:-none}
- TWOCAPTCHA_API_KEY=${TWOCAPTCHA_API_KEY:-}
ports:
- "${FLARESOLVERR_PORT:-8191}:8191"
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:8191/health"]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
networks:
- crawlrs-network
profiles:
- browser
- all
chrome:
image: browserless/chrome:latest
container_name: crawlrs-chrome
environment:
- MAX_CONCURRENT_SESSIONS=10
- CONNECTION_TIMEOUT=30000
ports:
- "${CHROME_PORT:-9222}:3000"
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:3000/json/version"]
interval: 30s
timeout: 10s
retries: 5
start_period: 20s
networks:
- crawlrs-network
profiles:
- browser
- all
prometheus:
image: prom/prometheus:latest
container_name: crawlrs-prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.enable-lifecycle'
ports:
- "${PROMETHEUS_PORT:-9090}:9090"
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
interval: 30s
timeout: 10s
retries: 5
networks:
- crawlrs-network
profiles:
- monitoring
- all
grafana:
image: grafana/grafana:latest
container_name: crawlrs-grafana
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:?GRAFANA_PASSWORD is required in .env}
ports:
- "${GRAFANA_PORT:-3001}:3000"
volumes:
- grafana_data:/var/lib/grafana
restart: unless-stopped
depends_on:
- prometheus
networks:
- crawlrs-network
profiles:
- monitoring
- all
pgadmin:
image: dpage/pgadmin8:latest
container_name: crawlrs-pgadmin
environment:
- PGADMIN_DEFAULT_EMAIL=admin@crawlrs.local
- PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD:?PGADMIN_PASSWORD is required in .env}
ports:
- "${PGADMIN_PORT:-5050}:80"
volumes:
- pgadmin_data:/var/lib/pgadmin
restart: unless-stopped
networks:
- crawlrs-network
profiles:
- tools
worker:
build:
context: ..
dockerfile: docker/Dockerfile
args:
BUILD_FEATURES: ${BUILD_FEATURES:-standard}
image: crawlrs:latest
container_name: crawlrs-worker
command: ["worker"]
environment:
- CRAWLRS__RUST_LOG=${RUST_LOG:-info}
- CRAWLRS__ENV_MODE=${ENV_MODE:-docker}
- CRAWLRS__DATABASE__URL=postgres://${DB_USER:-crawlrs}:${DB_PASSWORD:?DB_PASSWORD is required in .env}@${DB_HOST:-db}:5432/${DB_NAME:-crawlrs}
- HTTP_PROXY=${HTTP_PROXY:-}
- HTTPS_PROXY=${HTTPS_PROXY:-}
- NO_PROXY=${NO_PROXY:-localhost,127.0.0.1,.local,.corp}
depends_on:
db:
condition: service_healthy
volumes:
- ../storage:/home/crawlrs/storage
- ../logs:/home/crawlrs/logs
restart: unless-stopped
networks:
- crawlrs-network
profiles:
- worker
networks:
crawlrs-network:
driver: bridge
volumes:
db_data:
prometheus_data:
grafana_data:
pgadmin_data: