bgustscraper 0.2.0

Advanced semantic scraping engine with AI-driven compliance checks and legal terms validation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

# ==========================================
# 🐳 Dockerfile para bgustscraper
# Soporte completo para Playwright, Modelos locales Hugging Face (ONNX) y API
# ==========================================

FROM rust:1.75-bookworm as builder

# Instalar Node.js y dependencias del sistema para Playwright y WGPU
RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
    apt-get update && apt-get install -y \
    nodejs \
    build-essential \
    pkg-config \
    libssl-dev \
    libudev-dev \
    libegl1-mesa-dev \
    libwayland-dev \
    libxkbcommon-dev \
    libasound2-dev \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /usr/src/app
COPY . .

# Compilar binario de Rust en modo Release
RUN cargo build --release

# ==========================================
# Imagen final de ejecución asilada
# ==========================================
FROM debian:bookworm-slim

# Instalar dependencias del sistema operativo requeridas para Playwright Chromium Headless y Node.js
RUN apt-get update && apt-get install -y \
    curl \
    gnupg \
    && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - && \
    apt-get install -y \
    nodejs \
    libssl3 \
    ca-certificates \
    # Dependencias requeridas de sistema por Chromium (Playwright)
    libglib2.0-0 \
    libnss3 \
    libatk1.0-0 \
    libatk-bridge2.0-0 \
    libcups2 \
    libdrm2 \
    libdbus-1-3 \
    libxcb1 \
    libxkbcommon0 \
    libx11-6 \
    libxcomposite1 \
    libxdamage1 \
    libxext6 \
    libxfixes3 \
    libxrandr2 \
    libgbm1 \
    libpango-1.0-0 \
    libcairo2 \
    libasound2 \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /app

# Copiar dependencias y binarios del constructor
COPY --from=builder /usr/src/app/target/release/bgustscraper /usr/local/bin/bgustscraper
COPY --from=builder /usr/src/app/worker ./worker

# Instalar dependencias npm del worker
WORKDIR /app/worker
RUN npm install

# Instalar el navegador Chromium específico para Playwright de forma headless
RUN npx playwright install chromium

# ==========================================
# Pre-caché de modelos Hugging Face para el Benchmark
# Esto evita que el benchmark mida la descarga por red en la primera corrida.
# ==========================================
COPY --from=builder /usr/src/app/worker/test_transformers.js .
COPY --from=builder /usr/src/app/worker/test_qwen.js .
RUN node test_transformers.js && node test_qwen.js && rm test_transformers.js test_qwen.js

WORKDIR /app

# Crear directorios para descargas y bases de datos locales
RUN mkdir -p downloads config models

# Exponer el puerto por defecto para el Servidor Web Axum
EXPOSE 8080

# Variable de entorno para indicarle a los modelos de Hugging Face dónde escribir la caché local
ENV HF_HOME=/app/models/.cache

# Por defecto arranca en modo daemon/servidor
CMD ["bgustscraper", "--daemon"]