crusty 0.4.0

Fast && scalable Broad Web Crawler developed on top of crusty-core
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

version: '3.8'

services:

  clickhouse:
    container_name: crusty-clickhouse
    build:
      context: ./infra/clickhouse
    ulimits:
      nproc: 65535
      nofile:
        soft: 1048576
        hard: 1048576
    ports:
      - "8123:8123"
      - "9000:9000"
      - "9009:9009"
    volumes:
    - clickhouse-volume:/var/lib/clickhouse
    - /etc/localtime:/etc/localtime:ro

  grafana:
    container_name: crusty-grafana
    build:
      context: ./infra/grafana
    volumes:
      - grafana-volume:/var/lib/grafana
      - grafana-plugins:/grafana-plugins
      - /etc/localtime:/etc/localtime:ro
    ports:
      - "3000:3000"
    links:
      - clickhouse
    environment:
      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource
      - GF_PLUGIN_DIR=/grafana-plugins

  crusty:
    container_name: crusty
    build:
      context: .
      dockerfile: ./infra/crusty/Dockerfile
    ulimits:
      nofile:
        soft: 1048576
        hard: 1048576
    links:
      - clickhouse
    volumes:
      - /etc/localtime:/etc/localtime:ro
    environment:
      - CRUSTY_SEEDS=${CRUSTY_SEEDS}

volumes:
  clickhouse-volume:
  grafana-volume:
  grafana-plugins: