bgustcrawler 0.2.0

const { chromium } = require('playwright-extra');
const stealth = require('puppeteer-extra-plugin-stealth')();
const readline = require('readline');
const TurndownService = require('turndown');
const { gfm } = require('turndown-plugin-gfm');

const turndownService = new TurndownService({
    headingStyle: 'atx',
    codeBlockStyle: 'fenced'
});
turndownService.use(gfm);

// Intentar cargar el plugin de stealth específico si está disponible
try {
    const playwrightStealth = require('playwright-extra-plugin-stealth')();
    chromium.use(playwrightStealth);
} catch (e) {
    chromium.use(stealth);
}

const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout,
    terminal: false
});

const isHeadless = process.argv[2] !== 'false';

if (isHeadless) {
    process.stderr.write('Worker iniciando en modo HEADLESS (sin ventana)\n');
} else {
    process.stderr.write('Worker iniciando en modo INTERACTIVO (con ventana)\n');
}

async function run() {
    const browser = await chromium.launch({ 
        headless: isHeadless,
        args: [
            '--start-maximized', 
            '--no-sandbox',
            '--disable-blink-features=AutomationControlled', // Evasión Activa Técnica C
            '--disable-infobars',
        ] 
    });
    
    rl.on('line', async (line) => {
        try {
            const { action, url, options, proxy } = JSON.parse(line);
            
            if (action === 'scrape') {
                const isDeepScan = options && options.deepScan;
                const contextOptions = {
                    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
                    viewport: isHeadless ? { width: 1920, height: 1080 } : null,
                    deviceScaleFactor: 1,
                    javaScriptEnabled: true,
                };

                if (proxy) {
                    contextOptions.proxy = { server: proxy };
                }

                const context = await browser.newContext(contextOptions);
                
                // Técnica C: Evasión Activa (Overrides de Navegador)
                await context.addInitScript(() => {
                    Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
                    window.chrome = { runtime: {} };
                    Object.defineProperty(navigator, 'languages', { get: () => ['es-ES', 'es', 'en'] });
                    Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
                });

                const page = await context.newPage();
                
                await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 90000 });
                
                if (isDeepScan) {
                    process.stderr.write('Iniciando Deep Scan: Buscando botones de descarga dinámicos...\n');
                    const downloadSelectors = [
                        'text="Download PDF"', 'text="Descargar PDF"', 'text="Full-text PDF"',
                        'text="Download"', 'text="Descargar"', '.download-button', '[title*="Download"]'
                    ];

                    for (const selector of downloadSelectors) {
                        try {
                            const btn = page.locator(selector).first();
                            if (await btn.isVisible()) {
                                process.stderr.write(`Haciendo clic en: ${selector}\n`);
                                await btn.click({ timeout: 5000 });
                                await page.waitForTimeout(3000);
                            }
                        } catch (e) {}
                    }
                }
                
                if (!isHeadless) {
                    process.stderr.write('Esperando intervención humana o carga completa...\n');
                    await page.waitForTimeout(15000); 
                } else {
                    await page.waitForTimeout(2000 + Math.random() * 3000);
                }

                let content = await page.content();
                if (content.includes('cf-challenge') || content.includes('ray-id') || content.includes('Checking your browser')) {
                    process.stderr.write('Cloudflare detectado, aplicando espera extra...\n');
                    await page.waitForTimeout(10000);
                    content = await page.content();
                }

                // Scroll suave optimizado para evitar bucles infinitos
                process.stderr.write('Ejecutando scroll para cargar contenidos dinámicos...\n');
                await page.evaluate(async () => {
                    await new Promise((resolve) => {
                        let totalHeight = 0;
                        let distance = 200;
                        let maxScroll = 5000; // Limitar el scroll máximo para evitar colgarse
                        let timer = setInterval(() => {
                            let scrollHeight = document.body.scrollHeight;
                            window.scrollBy(0, distance);
                            totalHeight += distance;
                            if(totalHeight >= scrollHeight || totalHeight >= maxScroll){
                                clearInterval(timer);
                                resolve();
                            }
                        }, 100);
                    });
                });

                const finalContent = await page.content();
                const title = await page.title();
                
                process.stderr.write('Generando Markdown y finalizando...\n');
                // Técnica B: Markdown Conversion (Firecrawl style)
                const markdown = turndownService.turndown(finalContent);
                
                process.stdout.write(JSON.stringify({ 
                    status: 'success', 
                    html: finalContent, 
                    markdown: markdown,
                    title, 
                    url 
                }) + '\n');
                process.stderr.write('Scrape finalizado con éxito.\n');
                
                await context.close();
            } else if (action === 'exit') {
                await browser.close();
                process.exit(0);
            }
        } catch (err) {
            process.stdout.write(JSON.stringify({ status: 'error', message: err.message }) + '\n');
        }
    });
}

run().catch(err => {
    process.stderr.write(`Fatal Error: ${err.message}\n`);
    process.exit(1);
});