const { chromium } = require('playwright-extra');
const stealth = require('puppeteer-extra-plugin-stealth')();
const readline = require('readline');
const TurndownService = require('turndown');
const { gfm } = require('turndown-plugin-gfm');
const turndownService = new TurndownService({
headingStyle: 'atx',
codeBlockStyle: 'fenced'
});
turndownService.use(gfm);
try {
const playwrightStealth = require('playwright-extra-plugin-stealth')();
chromium.use(playwrightStealth);
} catch (e) {
chromium.use(stealth);
}
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
terminal: false
});
const isHeadless = process.argv[2] !== 'false';
if (isHeadless) {
process.stderr.write('Worker iniciando en modo HEADLESS (sin ventana)\n');
} else {
process.stderr.write('Worker iniciando en modo INTERACTIVO (con ventana)\n');
}
let pipeline = null;
let sentenceExtractor = null;
let qwenGenerator = null;
let browser = null;
async function getBrowser() {
if (!browser) {
process.stderr.write('Lanzando navegador Chromium (Playwright Stealth)...\n');
browser = await chromium.launch({
headless: isHeadless,
args: [
'--start-maximized',
'--no-sandbox',
'--disable-blink-features=AutomationControlled',
'--disable-infobars',
]
});
}
return browser;
}
async function getSentenceExtractor() {
if (!sentenceExtractor) {
process.stderr.write('Cargando all-MiniLM-L6-v2 ONNX local de Hugging Face...\n');
if (!pipeline) {
const transformers = await import('@huggingface/transformers');
transformers.env.allowRemoteModels = false;
transformers.env.localModelPath = require('path').join(__dirname, '..', 'models');
pipeline = transformers.pipeline;
}
sentenceExtractor = await pipeline('feature-extraction', 'MiniLM-L6');
}
return sentenceExtractor;
}
async function getQwenGenerator() {
if (!qwenGenerator) {
process.stderr.write('Cargando Qwen2.5-0.5B-Instruct ONNX (Q4) local...\n');
if (!pipeline) {
const transformers = await import('@huggingface/transformers');
transformers.env.allowRemoteModels = false;
transformers.env.localModelPath = require('path').join(__dirname, '..', 'models');
pipeline = transformers.pipeline;
}
qwenGenerator = await pipeline('text-generation', 'Qwen2.5-0.5B', {
quantized: true,
});
}
return qwenGenerator;
}
function splitIntoSentences(text) {
if (!text) return [];
return text
.split(/(?<=[.!?])\s+|\n+/)
.map(s => s.trim())
.filter(s => s.length > 15 && s.length < 400 && !s.startsWith('#') && !s.startsWith('-') && !s.startsWith('*'));
}
async function generateEmbeddings(sentences) {
const extractor = await getSentenceExtractor();
const embeddings = [];
process.stderr.write(`Generando embeddings locales para ${sentences.length} oraciones...\n`);
const batchSize = 8;
for (let i = 0; i < sentences.length; i += batchSize) {
const batch = sentences.slice(i, i + batchSize);
const batchResults = await Promise.all(
batch.map(sentence =>
extractor(sentence, { pooling: 'mean', normalize: true })
)
);
for (const res of batchResults) {
embeddings.push(Array.from(res.data));
}
}
return embeddings;
}
async function run() {
rl.on('line', async (line) => {
try {
const { action, url, options, proxy } = JSON.parse(line);
if (action === 'scrape') {
const isDeepScan = options && options.deepScan;
const isSemantic = options && options.semantic_embeddings;
const activeBrowser = await getBrowser();
const contextOptions = {
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
viewport: isHeadless ? { width: 1920, height: 1080 } : null,
deviceScaleFactor: 1,
javaScriptEnabled: true,
};
if (proxy) {
contextOptions.proxy = { server: proxy };
}
const context = await activeBrowser.newContext(contextOptions);
await context.addInitScript(() => {
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
window.chrome = { runtime: {} };
Object.defineProperty(navigator, 'languages', { get: () => ['es-ES', 'es', 'en'] });
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
});
const page = await context.newPage();
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 90000 });
if (isDeepScan) {
process.stderr.write('Iniciando Deep Scan: Buscando botones de descarga dinámicos...\n');
const downloadSelectors = [
'text="Download PDF"', 'text="Descargar PDF"', 'text="Full-text PDF"',
'text="Download"', 'text="Descargar"', '.download-button', '[title*="Download"]'
];
for (const selector of downloadSelectors) {
try {
const btn = page.locator(selector).first();
if (await btn.isVisible()) {
process.stderr.write(`Haciendo clic en: ${selector}\n`);
await btn.click({ timeout: 5000 });
await page.waitForTimeout(3000);
}
} catch (e) {}
}
}
if (!isHeadless) {
process.stderr.write('Esperando intervención humana o carga completa...\n');
await page.waitForTimeout(15000);
} else {
await page.waitForTimeout(2000 + Math.random() * 3000);
}
let content = await page.content();
if (content.includes('cf-challenge') || content.includes('ray-id') || content.includes('Checking your browser')) {
process.stderr.write('Cloudflare detectado, aplicando espera extra...\n');
await page.waitForTimeout(10000);
content = await page.content();
}
process.stderr.write('Ejecutando scroll para cargar contenidos dinámicos...\n');
await page.evaluate(async () => {
await new Promise((resolve) => {
let totalHeight = 0;
let distance = 200;
let maxScroll = 5000;
let timer = setInterval(() => {
let scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight || totalHeight >= maxScroll){
clearInterval(timer);
resolve();
}
}, 100);
});
});
const finalContent = await page.content();
const title = await page.title();
process.stderr.write('Generando Markdown y finalizando...\n');
const markdown = turndownService.turndown(finalContent);
let finalResponse = {
status: 'success',
html: finalContent,
markdown: markdown,
title,
url
};
if (isSemantic) {
try {
const sentences = splitIntoSentences(markdown);
if (sentences.length > 0) {
const embeddings = await generateEmbeddings(sentences);
finalResponse.sentences = sentences;
finalResponse.embeddings = embeddings;
}
} catch (e) {
process.stderr.write(`Error generando embeddings semánticos: ${e.message}\n`);
}
}
process.stdout.write(JSON.stringify(finalResponse) + '\n');
process.stderr.write('Scrape finalizado con éxito.\n');
await context.close();
} else if (action === 'extract') {
const { text, prompt } = options;
process.stderr.write('Iniciando extracción semántica estructurada con Qwen...\n');
const generator = await getQwenGenerator();
const systemPrompt = `You are a precise semantic data extractor.
Your task is to analyze the provided text and extract the requested information.
You MUST output ONLY a valid JSON object.
DO NOT wrap the output in markdown code blocks like \`\`\`json.
DO NOT write any prose or explanation. Just raw JSON.`;
const userPrompt = `Text to analyze:
${text}
Instruction:
${prompt}
Respond with the extracted JSON structure directly.`;
const messages = [
{ role: "system", content: systemPrompt },
{ role: "user", content: userPrompt }
];
const output = await generator(messages, {
max_new_tokens: 512,
temperature: 0.1,
return_full_text: false
});
const rawResponse = output[0].generated_text[2].content.trim();
let cleanResponse = rawResponse;
if (cleanResponse.startsWith('```')) {
cleanResponse = cleanResponse.replace(/^```[a-zA-Z]*\n/, '').replace(/\n```$/, '');
}
cleanResponse = cleanResponse.trim();
try {
const parsedData = JSON.parse(cleanResponse);
process.stdout.write(JSON.stringify({
status: 'success',
data: parsedData
}) + '\n');
} catch (e) {
process.stderr.write(`Error al parsear JSON del modelo: ${e.message}\nRespuesta cruda: ${cleanResponse}\n`);
process.stdout.write(JSON.stringify({
status: 'error',
message: `El modelo no generó un JSON válido: ${e.message}`,
raw: cleanResponse
}) + '\n');
}
} else if (action === 'exit') {
if (browser) {
await browser.close();
}
process.exit(0);
}
} catch (err) {
process.stdout.write(JSON.stringify({ status: 'error', message: err.message }) + '\n');
}
});
}
run().catch(err => {
process.stderr.write(`Fatal Error: ${err.message}\n`);
process.exit(1);
});