hf2q 0.1.27

/**
 * web-search-fetch — local SearXNG search + Crawl4AI fetch tools for opencode.
 *
 * Backends (autostart via launchd, loopback only, $0 / no API keys):
 *   SearXNG   http://127.0.0.1:8888  (metasearch over ~70 engines, JSON API)
 *   Crawl4AI  http://127.0.0.1:11235 (headless-Chromium markdown fetch, static fast-path)
 *
 * Tools:
 *   web_search  — canonical search front door; searches, then reads top pages
 *   web_fetch   — fetches one user-supplied or search-discovered URL
 *   web_crawl   — bounded multi-page crawl with domain/relevance filters
 *   web_extract — structured CSS or semantic extraction from one URL
 *
 * OpenCode 2.x plugin API: `export default {id, setup}` with tools registered
 * through `ctx.tool.transform`. The built-in fetch-only `webfetch` is removed
 * from the catalog so a search request cannot degrade into guessed URLs; the
 * replacement `web_fetch` retains and expands that capability. OpenCode 2.x
 * allows tool use by default, so no permission entries are needed. Bash,
 * files, tasks, skills, and MCP tools remain untouched.
 *
 * Executors thread OpenCode's session cancellation `signal` into every
 * backend request alongside the per-call timeout.
 */

const SEARXNG = process.env.OPENCODE_SEARXNG_URL || "http://127.0.0.1:8888";
const FETCH = process.env.OPENCODE_FETCH_URL || "http://127.0.0.1:11235";

const FETCH_TIMEOUT_MS = 150_000;
const SEARCH_TIMEOUT_MS = 20_000;

// Known junk/spam/clickbait domains filtered from search results.
const JUNK_DOMAINS = new Set([
  "99designs.com",
  "anyrgb.com",
  "boredpanda.com",
  "buzzfeed.com",
  "clickbait.com",
  "dailymail.co.uk",
  "digg.com",
  "eonline.com",
  "eskipaper.com",
  "fanpop.com",
  "fotofacil.com.br",
  "grabcad.com",
  "hobbylark.com",
  "pinterest.com",
  "pinterest.co.uk",
  "promopanda.com",
  "quotefancy.com",
  "redbubble.com",
  "shutterstock.com",
  "slideshare.net",
  "spongebob.com",
  "taringa.net",
  "teepublic.com",
  "thefamouspeople.com",
  "themogh.org",
  "wallpaperaccess.com",
  "wallpaperflare.com",
  "wattpad.com",
]);

const QUERY_STOP_WORDS = new Set([
  "about",
  "company",
  "current",
  "does",
  "for",
  "from",
  "how",
  "into",
  "latest",
  "please",
  "tell",
  "that",
  "the",
  "their",
  "this",
  "today",
  "what",
  "when",
  "where",
  "which",
  "who",
  "why",
  "with",
  "wrote",
]);

const ENGINE_MAP = {
  general: "bing,google,duckduckgo web,mojeek",
  academic: "arxiv,google scholar,semantic scholar,pubmed,crossref,openalex",
  tech: "github,stackoverflow,mdn,docker hub,arch linux wiki,gentoo",
  news: "google,duckduckgo,mastodon hashtags",
  wikidata: "wikipedia,wikidata,wiktionary",
};

function isJunkUrl(url) {
  try {
    return JUNK_DOMAINS.has(new URL(url).hostname.replace(/^www\./, ""));
  } catch {
    return false;
  }
}

function dedupByUrl(results) {
  const seen = new Set();
  const out = [];
  for (const r of results) {
    let key;
    try {
      key = new URL(r.url).hostname + new URL(r.url).pathname;
    } catch {
      key = r.url;
    }
    if (!seen.has(key)) {
      seen.add(key);
      out.push(r);
    }
  }
  return out;
}

function significantQueryTerms(query) {
  return [...new Set(query.toLowerCase().match(/[a-z0-9][a-z0-9.+#-]*/g) || [])].filter(
    (term) => term.length >= 3 && !QUERY_STOP_WORDS.has(term),
  );
}

export function resultsLookRelevant(query, results) {
  return filterRelevantResults(query, results).length > 0;
}

function evidenceContainsTerm(evidence, term) {
  return termVariants(term).some((variant) => {
    const escaped = variant.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
    return new RegExp(`(^|[^a-z0-9])${escaped}(?=$|[^a-z0-9])`, "i").test(evidence);
  });
}

function termVariants(term) {
  const variants = new Set([term]);
  if (term.length > 4 && term.endsWith("ies")) {
    variants.add(`${term.slice(0, -3)}y`);
  } else if (["ches", "shes", "sses", "xes", "zes"].some((suffix) => term.endsWith(suffix))) {
    variants.add(term.slice(0, -2));
  } else if (term.length > 3 && term.endsWith("es")) {
    variants.add(term.slice(0, -1));
  } else if (
    term.length > 3 &&
    term.endsWith("s") &&
    !["ss", "is", "us"].some((suffix) => term.endsWith(suffix)) &&
    !["news", "series", "species"].includes(term)
  ) {
    variants.add(term.slice(0, -1));
  } else if (term.endsWith("y") && term.length > 1 && !"aeiou".includes(term[term.length - 2])) {
    variants.add(`${term.slice(0, -1)}ies`);
  } else if (["s", "x", "z", "ch", "sh"].some((suffix) => term.endsWith(suffix))) {
    variants.add(`${term}es`);
  } else if (!["is", "us"].some((suffix) => term.endsWith(suffix))) {
    variants.add(`${term}s`);
  }
  return [...variants];
}

export function filterRelevantResults(query, results) {
  const terms = significantQueryTerms(query);
  if (!terms.length) return [];
  return results.filter((result) => {
    const evidence = `${result.title || ""} ${result.url || ""} ${result.content || ""}`.toLowerCase();
    const matched = terms.filter((term) => evidenceContainsTerm(evidence, term)).length;
    return matched === terms.length || (terms.length >= 3 && matched * 3 >= terms.length * 2);
  });
}

// Combine a per-call timeout with the session's cancellation signal.
function withTimeout(timeoutMs, signal) {
  const timeout = AbortSignal.timeout(timeoutMs);
  return signal ? AbortSignal.any([timeout, signal]) : timeout;
}

async function postJSON(url, body, timeoutMs, signal) {
  const res = await fetch(url, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(body),
    signal: withTimeout(timeoutMs, signal),
  });
  if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
  return res.json();
}

function clip(s, n) {
  if (!s) return "";
  return s.length > n ? s.slice(0, n) + " …" : s;
}

function assertHttpUrl(url) {
  const parsed = new URL(url);
  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
    throw new Error(`URL must use http or https: ${url}`);
  }
}

function commaList(value) {
  if (!value) return undefined;
  const values = value
    .split(",")
    .map((item) => item.trim())
    .filter(Boolean);
  return values.length ? values : undefined;
}

function primaryFailures(data) {
  return (data?.unresponsive_engines || []).map((failure) =>
    Array.isArray(failure) ? `${failure[0]}: ${failure[1]}` : String(failure),
  );
}

function mayUseDiscoveryFallback(args) {
  return !args.engines && (!args.category || args.category === "general") && !args.time_range;
}

export async function searchExecute(args, signal) {
  const pages = Math.min(Math.max(Math.trunc(args.pages ?? 2), 1), 3);
  const excerpt = Math.min(Math.max(Math.trunc(args.excerpt_chars ?? 4000), 500), 8000);
  try {
    const engines = args.engines || ENGINE_MAP[args.category] || ENGINE_MAP.general;
    let data;
    let primaryError;
    try {
      data = await searx(args.query, {
        time_range: args.time_range,
        engines,
        language: args.language,
      }, signal);
    } catch (error) {
      primaryError = error;
      data = { results: [], unresponsive_engines: [] };
    }

    const primaryCandidates = dedupByUrl(data.results || []).filter((r) => !isJunkUrl(r.url));
    let results = filterRelevantResults(args.query, primaryCandidates).slice(0, pages);
    let route = "SearXNG primary";
    let fallbackError;
    let qualityError = primaryCandidates.length && !results.length
      ? "SearXNG results failed the query-term relevance gate"
      : null;
    if ((!results.length || qualityError) && mayUseDiscoveryFallback(args)) {
      try {
        const fallback = await postJSON(
          `${FETCH}/search-fallback`,
          {
            query: args.query,
            language: args.language,
            max_results: pages,
          },
          FETCH_TIMEOUT_MS,
          signal,
        );
        if (fallback.ok) {
          const fallbackCandidates = dedupByUrl(fallback.results || []).filter(
            (result) => !isJunkUrl(result.url),
          );
          const fallbackResults = filterRelevantResults(args.query, fallbackCandidates).slice(0, pages);
          if (fallbackResults.length) {
            results = fallbackResults;
            qualityError = null;
            route = `${fallback.provider || "browser fallback"} via ${fallback.via || "unknown"}`;
          } else {
            fallbackError = "returned no query-relevant organic results";
          }
        } else {
          fallbackError = fallback.error || "no validated organic results";
        }
      } catch (error) {
        fallbackError = error.message;
      }
    }

    if (qualityError) results = [];

    if (!results.length) {
      const failures = primaryFailures(data);
      if (primaryError || failures.length || fallbackError || qualityError) {
        const details = [
          primaryError ? `SearXNG: ${primaryError.message}` : null,
          failures.length ? `engines: ${failures.join(", ")}` : null,
          qualityError,
          fallbackError ? `fallback: ${fallbackError}` : null,
          !mayUseDiscoveryFallback(args) ? "fallback skipped to preserve explicit constraints" : null,
        ].filter(Boolean);
        return `WEB_SEARCH_FAILED for "${args.query}": ${details.join("; ")}. Do not guess URLs.`;
      }
      return `No search results for: ${args.query} (SearXNG reported no engine failures)`;
    }
    const sections = await Promise.all(
      results.map(async (result, index) => {
        const engines = (result.engines || [result.engine]).filter(Boolean).join(",");
        const date = result.publishedDate ? ` (${result.publishedDate.slice(0, 10)})` : "";
        const searchEvidence = clip(result.content, 500);
        try {
          const page = await fetchPage(result.url, {
            mode: "static",
            max_chars: excerpt,
            public_only: true,
          }, signal);
          const body = page.ok ? clip(page.markdown, excerpt) : `(page read failed: ${page.error})`;
          return `## Source ${index + 1}: ${result.title}${date}\n${result.url} [${engines}]\n\nSearch excerpt: ${searchEvidence}\n\n${body}`;
        } catch (error) {
          return `## Source ${index + 1}: ${result.title}${date}\n${result.url} [${engines}]\n\nSearch excerpt: ${searchEvidence}\n\n(page read failed: ${error.message})`;
        }
      }),
    );
    return `# Web research: "${args.query}"\nSearch route: ${route}\n\n${sections.join("\n\n---\n\n")}`;
  } catch (error) {
    return `WEB_SEARCH_FAILED: ${error.message}. Do not guess URLs or retry with web_fetch.`;
  }
}

async function fetchExecute(args, signal) {
  try {
    assertHttpUrl(args.url);
    const maxChars = Math.min(Math.max(Math.trunc(args.max_chars ?? 12000), 500), 40000);
    const page = await fetchPage(args.url, { ...args, max_chars: maxChars }, signal);
    if (!page.ok) return `WEB_FETCH_FAILED for ${args.url}: ${page.error || "empty content"}`;
    const head = `# ${page.title || page.url}\n${page.url}  (via ${page.via}${page.truncated ? ", truncated" : ""})\n\n`;
    return head + (page.markdown || "(no text content extracted)");
  } catch (error) {
    return `WEB_FETCH_FAILED: ${error.message}`;
  }
}

async function crawlExecute(args, signal) {
  try {
    assertHttpUrl(args.url);
    const maxDepth = Math.min(Math.max(Math.trunc(args.max_depth ?? 2), 1), 5);
    const maxPages = Math.min(Math.max(Math.trunc(args.max_pages ?? 6), 1), 20);
    const maxChars = Math.min(Math.max(Math.trunc(args.max_chars ?? 6000), 500), 20000);
    const result = await postJSON(
      `${FETCH}/crawl`,
      {
        url: args.url,
        max_depth: maxDepth,
        max_pages: maxPages,
        allowed_domains: commaList(args.allowed_domains),
        blocked_domains: commaList(args.blocked_domains),
        include_external: args.include_external ?? false,
        query: args.query,
        max_chars: maxChars,
      },
      FETCH_TIMEOUT_MS,
      signal,
    );
    if (!result.ok) return `WEB_CRAWL_FAILED for ${args.url}: ${result.error || "crawl failed"}`;
    const pages = (result.pages || []).map(
      (page, index) =>
        `## Page ${index + 1}: ${page.title || page.url}\n${page.url} (depth ${page.depth ?? "?"})\n\n${page.markdown || "(no text extracted)"}`,
    );
    return `# Web crawl: ${args.url}\nCrawled ${result.crawled ?? pages.length} page(s).\n\n${pages.join("\n\n---\n\n")}`;
  } catch (error) {
    return `WEB_CRAWL_FAILED: ${error.message}`;
  }
}

export function normalizeJsonCssSchema(schema) {
  if (!schema || typeof schema !== "object" || Array.isArray(schema)) return schema;

  const normalizeField = (name, value) => {
    if (typeof value === "string") return { name, selector: value, type: "text" };
    if (value && typeof value === "object" && !Array.isArray(value)) {
      return { name, type: "text", ...value };
    }
    return value;
  };

  if (schema.fields && !Array.isArray(schema.fields) && typeof schema.fields === "object") {
    return {
      ...schema,
      baseSelector: schema.baseSelector || "body",
      fields: Object.entries(schema.fields).map(([name, value]) => normalizeField(name, value)),
    };
  }
  if (Array.isArray(schema.fields)) {
    return { ...schema, baseSelector: schema.baseSelector || "body" };
  }

  // Accept the compact shape models naturally emit, e.g. {"heading":"h1"}.
  // Reserved JSON-CSS metadata remains metadata; all other keys become fields.
  const reserved = new Set(["name", "baseSelector", "baseFields"]);
  const entries = Object.entries(schema).filter(([name]) => !reserved.has(name));
  if (entries.length && entries.every(([, value]) => typeof value === "string" || (value && typeof value === "object"))) {
    return {
      name: schema.name,
      baseSelector: schema.baseSelector || "body",
      baseFields: schema.baseFields,
      fields: entries.map(([name, value]) => normalizeField(name, value)),
    };
  }
  return schema;
}

async function extractExecute(args, signal) {
  try {
    assertHttpUrl(args.url);
    let schema;
    if (args.schema_json) schema = JSON.parse(args.schema_json);
    // Qwen-family tool callers occasionally place a JSON-CSS schema in the
    // adjacent `query` field. Accept that unambiguous shape instead of burning
    // a failed tool round-trip; ordinary semantic queries remain untouched.
    if (!schema && (args.strategy ?? "json_css") === "json_css" && args.query?.trim().startsWith("{")) {
      schema = JSON.parse(args.query);
    }
    schema = normalizeJsonCssSchema(schema);
    if ((args.strategy ?? "json_css") === "json_css" && !schema) {
      return "WEB_EXTRACT_REJECTED: json_css requires schema_json";
    }
    const result = await postJSON(
      `${FETCH}/extract`,
      {
        url: args.url,
        strategy: args.strategy ?? "json_css",
        schema,
        query: args.query,
        max_chars: Math.min(Math.max(Math.trunc(args.max_chars ?? 40000), 500), 100000),
      },
      FETCH_TIMEOUT_MS,
      signal,
    );
    if (!result.ok) return `WEB_EXTRACT_FAILED for ${args.url}: ${result.error || "extraction failed"}`;
    return `# Web extraction: ${args.url}\n\n${
      typeof result.data === "string" ? result.data : JSON.stringify(result.data, null, 2)
    }`;
  } catch (error) {
    return `WEB_EXTRACT_FAILED: ${error.message}`;
  }
}

export async function searx(query, opts = {}, signal) {
  const params = new URLSearchParams({ q: query, format: "json" });
  if (opts.categories) params.set("categories", opts.categories);
  if (opts.engines) params.set("engines", opts.engines);
  if (opts.time_range) params.set("time_range", opts.time_range);
  if (opts.language) params.set("language", opts.language);
  const res = await fetch(`${SEARXNG}/search?${params}`, {
    signal: withTimeout(SEARCH_TIMEOUT_MS, signal),
  });
  if (!res.ok) throw new Error(`HTTP ${res.status} from SearXNG`);
  return res.json();
}

async function fetchPage(url, opts = {}, signal) {
  return postJSON(
    `${FETCH}/fetch`,
    {
      url,
      mode: opts.mode || "auto",
      css_selector: opts.css_selector,
      max_chars: opts.max_chars,
      public_only: opts.public_only ?? false,
    },
    FETCH_TIMEOUT_MS,
    signal,
  );
}

const SEARCH_INPUT = {
  type: "object",
  properties: {
    query: { type: "string", description: "The search or research query" },
    category: {
      type: "string",
      enum: Object.keys(ENGINE_MAP),
      description: "Engine set: general, academic, tech, news, or wikidata",
    },
    engines: {
      type: "string",
      description: "Explicit comma-separated engine list (overrides category)",
    },
    language: { type: "string", description: "Search language code, e.g. 'en'" },
    pages: { type: "number", description: "Top pages to read in parallel (default 2, max 3)" },
    time_range: {
      type: "string",
      enum: ["day", "week", "month", "year"],
      description: "Optional recency window",
    },
    excerpt_chars: {
      type: "number",
      description: "Content characters per page (default 4000, range 500-8000)",
    },
  },
  required: ["query"],
  additionalProperties: false,
};

const FETCH_INPUT = {
  type: "object",
  properties: {
    url: { type: "string", description: "Exact URL from the user or web_search" },
    mode: {
      type: "string",
      enum: ["auto", "static", "browser", "stealth"],
      description: "Fetch strategy (default: auto)",
    },
    css_selector: {
      type: "string",
      description: "Optional CSS selector, such as 'article' or '.docs-content'",
    },
    max_chars: {
      type: "number",
      description: "Maximum markdown characters (default 12000, range 500-40000)",
    },
  },
  required: ["url"],
  additionalProperties: false,
};

const CRAWL_INPUT = {
  type: "object",
  properties: {
    url: { type: "string", description: "Exact starting URL from the user or web_search" },
    max_depth: { type: "number", description: "Link depth (default 2, max 5)" },
    max_pages: { type: "number", description: "Page limit (default 6, max 20)" },
    allowed_domains: { type: "string", description: "Optional comma-separated domain allowlist" },
    blocked_domains: { type: "string", description: "Optional comma-separated domain blocklist" },
    include_external: { type: "boolean", description: "Follow external links (default false)" },
    query: { type: "string", description: "Optional semantic relevance query" },
    max_chars: { type: "number", description: "Characters retained per page (default 6000)" },
  },
  required: ["url"],
  additionalProperties: false,
};

const EXTRACT_INPUT = {
  type: "object",
  properties: {
    url: { type: "string", description: "Exact URL from the user or web_search" },
    strategy: {
      type: "string",
      enum: ["json_css", "cosine"],
      description: "Extraction strategy; defaults to json_css",
    },
    schema_json: {
      type: "string",
      description: "Required for json_css: put the JSON-CSS schema string here, never in query",
    },
    query: {
      type: "string",
      description: "Only for cosine: semantic filter text; never put JSON schema here",
    },
    max_chars: { type: "number", description: "Maximum extracted characters" },
  },
  required: ["url"],
  additionalProperties: false,
};

export default {
  id: "web-search-fetch",
  async setup(ctx) {
    await ctx.tool.transform((editor) => {
      // Remove the built-in fetch-only `webfetch` so a search request cannot
      // degrade into guessed URLs; `web_fetch` replaces and expands it.
      editor.remove("webfetch");

      editor.add({
        name: "web_search",
        description:
          "Canonical front door for every request to search, research, find, look up, or get current web information. " +
          "Searches via local SearXNG, rejects irrelevant result sets, uses one bounded fixed-provider discovery cascade when needed, " +
          "then reads the top public pages through the guarded static path with route and engine provenance. " +
          "Always call this before web_fetch; never guess a URL. One call should normally complete the research.",
        input: SEARCH_INPUT,
        execute: async (input, context) => ({ content: await searchExecute(input, context.signal) }),
      });

      editor.add({
        name: "web_fetch",
        description:
          "Fetch one URL supplied by the user or returned by web_search. Never invent or guess a URL. " +
          "Returns clean LLM-ready markdown via local Crawl4AI. " +
          "mode='auto' (default) uses static, then Chromium, then a Cloudflare-aware stealth browser only when needed.",
        input: FETCH_INPUT,
        execute: async (input, context) => ({ content: await fetchExecute(input, context.signal) }),
      });

      editor.add({
        name: "web_crawl",
        description:
          "Crawl multiple pages starting at one exact URL. Use after web_search when a task needs a bounded section of a site, " +
          "not for one-page reading. Supports domain and relevance filters and returns clean markdown from each page.",
        input: CRAWL_INPUT,
        execute: async (input, context) => ({ content: await crawlExecute(input, context.signal) }),
      });

      editor.add({
        name: "web_extract",
        description:
          "Extract structured data from one exact URL. Use json_css with schema_json, for example " +
          "{\"baseSelector\":\"body\",\"fields\":[{\"name\":\"heading\",\"selector\":\"h1\",\"type\":\"text\"}]}; " +
          "or use cosine with a semantic query.",
        input: EXTRACT_INPUT,
        execute: async (input, context) => ({ content: await extractExecute(input, context.signal) }),
      });
    });
  },
};