bgustscraper 0.2.0

Advanced semantic scraping engine with AI-driven compliance checks and legal terms validation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

const cloudscraper = require('cloudscraper');
const readline = require('readline');

const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout,
    terminal: false
});

process.stderr.write('Cloudscraper Worker iniciado (Engine: TLS Bypass)\n');

rl.on('line', async (line) => {
    try {
        const { action, url } = JSON.parse(line);
        
        if (action === 'scrape') {
            cloudscraper.get(url, (error, response, body) => {
                if (error) {
                    process.stdout.write(JSON.stringify({ status: 'error', message: error.message }) + '\n');
                } else {
                    process.stdout.write(JSON.stringify({ 
                        status: 'success', 
                        html: body, 
                        title: '', // Cloudscraper no da el título fácilmente sin parsear
                        url: url 
                    }) + '\n');
                }
            });
        } else if (action === 'exit') {
            process.exit(0);
        }
    } catch (err) {
        process.stdout.write(JSON.stringify({ status: 'error', message: err.message }) + '\n');
    }
});