rust-web-server 18.16.0

{
"ai_robots": {
"agents": [
[
"AGI Agent",
"AGI, Inc",
"AI Agents"
],
[
"AI2Bot",
"[Ai2](https://allenai.org/crawler)",
"Content is used to train open language models."
],
[
"AI2Bot-DeepResearchEval",
"Ai2, a non-profit AI research institute",
"AI Assistants"
],
[
"AIWebIndex",
"[Lyrenth](https://lyrenth.com)",
"AI Search Crawlers"
],
[
"AddSearchBot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"AgentDataBot",
"Unclear at this time.",
"AI Data Providers"
],
[
"AgentTimes",
"[The Agent Times](https://theagenttimes.com/about)",
"Data Scraper from RSS Feeds."
],
[
"Ai2Bot-Dolma",
"[Ai2](https://allenai.org/crawler)",
"Content is used to train open language models."
],
[
"AmazonBuyForMe",
"[Amazon](https://amazon.com)",
"AI Agents"
],
[
"Amazonbot",
"Amazon",
"Service improvement and enabling answers for Alexa users."
],
[
"Amzn-SearchBot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"Amzn-User",
"Amazon, used for fetching web content to answer user queries through Alexa and other Amazon AI services",
"AI Assistants"
],
[
"Andibot",
"[Andi](https://andisearch.com/)",
"Search engine using generative AI, AI Search Assistant"
],
[
"Anomura",
"[Direqt](https://direqt.ai)",
"Collects data for AI search"
],
[
"ApifyBot",
"Unclear at this time.",
"AI Data Providers"
],
[
"ApifyWebsiteContentCrawler",
"Unclear at this time.",
"AI Data Providers"
],
[
"Applebot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"Applebot-Extended",
"[Apple](https://support.apple.com/en-us/119829#datausage)",
"Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others."
],
[
"Aranet-SearchBot",
"Unclear at this time.",
"Undocumented AI Agents"
],
[
"Awario",
"Awario",
"AI Data Scrapers"
],
[
"AzureAI-SearchBot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"BixelBot",
"Unclear at this time.",
"AI Data Providers"
],
[
"Bravebot",
"https://safe.search.brave.com/help/brave-search-crawler",
"AI Data Providers"
],
[
"Brightbot",
"Unclear at this time.",
"AI Data Providers"
],
[
"Brightbot 1.0",
"https://brightdata.com/brightbot",
"LLM/AI training."
],
[
"BuddyBot",
"[BuddyBotLearning](https://www.buddybotlearning.com)",
"AI Learning Companion"
],
[
"Bytespider",
"ByteDance",
"LLM training."
],
[
"CCBot",
"[Common Crawl Foundation](https://commoncrawl.org)",
"Provides open crawl dataset, used for many purposes, including Machine Learning/AI."
],
[
"CartAI Agentic Commerce",
"CartAI that accesses online stores to support user-directed shopping and checkout workflows",
"AI Agents"
],
[
"Channel3Bot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"ChatGLM-Spider",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"ChatGPT Agent",
"[OpenAI](https://openai.com)",
"AI Agents"
],
[
"ChatGPT-User",
"[OpenAI](https://openai.com)",
"AI Assistants"
],
[
"Claude-Code",
"Unclear at this time.",
"AI Coding Agents"
],
[
"Claude-SearchBot",
"[Anthropic](https://www.anthropic.com)",
"Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses."
],
[
"Claude-User",
"[Anthropic](https://www.anthropic.com)",
"AI Assistants"
],
[
"Claude-Web",
"Anthropic",
"Undocumented AI Agents"
],
[
"ClaudeBot",
"[Anthropic](https://www.anthropic.com)",
"Scrapes data to train Anthropic's AI products."
],
[
"CledaraBot",
"Cledara that visits SaaS providers to complete customer-approved administrative tasks, including invoice collection and user managem\u2026",
"AI Agents"
],
[
"CloudVertexBot",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"Cloudflare-AutoRAG",
"[Cloudflare](https://developers.cloudflare.com/autorag)",
"Collects data for AI search"
],
[
"CloudflareBrowserRenderingCrawler",
"Cloudflare that returns rendered website content for research, monitoring, and AI data workflows",
"AI Data Providers"
],
[
"Code",
"Unclear at this time.",
"AI Coding Agents"
],
[
"Codex",
"Unclear at this time.",
"AI Coding Agents"
],
[
"Cotoyogi",
"[ROIS](https://ds.rois.ac.jp/en_center8/en_crawler/)",
"AI LLM Scraper."
],
[
"CragCrawler",
"CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services",
"AI Data Providers"
],
[
"Crawl4AI",
"Unclear at this time.",
"Undocumented AI Agents"
],
[
"Crawlspace",
"[Crawlspace](https://crawlspace.dev)",
"AI Data Providers"
],
[
"Cursor",
"Unclear at this time.",
"AI Coding Agents"
],
[
"Datenbank Crawler",
"Datenbank",
"AI Data Scrapers"
],
[
"DeepSeekBot",
"DeepSeek",
"Training language models and improving AI products"
],
[
"Devin",
"Devin AI",
"AI Coding Agents"
],
[
"Diffbot",
"[Diffbot](https://www.diffbot.com/)",
"AI Data Providers"
],
[
"Diffbot-User",
"[Diffbot](https://www.diffbot.com/)",
"AI Assistants"
],
[
"Dot",
"Unclear at this time.",
"AI Agents"
],
[
"DoubaoBot",
"ByteDance",
"AI crawler for ByteDance's Doubao AI assistant."
],
[
"DuckAssistBot",
"Unclear at this time.",
"AI Assistants"
],
[
"ERNIEBot",
"Baidu",
"Collects public web content for Baidu's ERNIE large language models."
],
[
"Echobot Bot",
"Echobox",
"AI Data Scrapers"
],
[
"EchoboxBot",
"[Echobox](https://echobox.com)",
"Data collection to support AI-powered products."
],
[
"ElevenlabsBot",
"[ElevenLabs](https://elevenlabs.io)",
"AI Assistants"
],
[
"ElevenlabsCrawler",
"[ElevenLabs](https://elevenlabs.io)",
"AI Data Scraper"
],
[
"ExaBot",
"Unclear at this time.",
"AI Data Providers"
],
[
"ExaSearchBot",
"[Exa](https://exa.ai)",
"AI Search Crawlers"
],
[
"ExaUser",
"Unclear at this time.",
"AI Assistants"
],
[
"FacebookBot",
"Meta/Facebook",
"Training language models"
],
[
"Factset_spyderbot",
"[Factset](https://www.factset.com/ai)",
"AI model training."
],
[
"FirecrawlAgent",
"Firecrawl that extracts web content and converts it into structured data for use in LLM and AI applications",
"AI Data Providers"
],
[
"FriendlyCrawler",
"Unknown",
"We are using the data from the crawler to build datasets for machine learning experiments."
],
[
"GPTBot",
"[OpenAI](https://openai.com)",
"Scrapes data to train OpenAI's products."
],
[
"GeistHaus-PageFetcher",
"GeistHaus, a company developing AI systems for therapy and psychological assessment",
"AI Assistants"
],
[
"Gemini-Deep-Research",
"Unclear at this time.",
"AI Assistants"
],
[
"Google-Agent",
"Unclear at this time.",
"AI Agents"
],
[
"Google-CloudVertexBot",
"Google",
"Build and manage AI models for businesses employing Vertex AI"
],
[
"Google-Extended",
"Google",
"LLM training."
],
[
"Google-Firebase",
"Google",
"Used as part of AI apps developed by users of Google's Firebase AI products."
],
[
"Google-Gemini-CLI",
"Unclear at this time.",
"AI Coding Agents"
],
[
"Google-NotebookLM",
"Unclear at this time.",
"AI Assistants"
],
[
"GoogleAgent-Mariner",
"Google",
"AI Agents"
],
[
"GoogleAgent-URLContext",
"Google that retrieves web content on behalf of Gemini API users",
"AI Assistants"
],
[
"GoogleOther",
"Google",
"Scrapes data."
],
[
"GoogleOther-Image",
"Google",
"Scrapes data."
],
[
"GoogleOther-Video",
"Google",
"Scrapes data."
],
[
"Grok Bot",
"Unclear at this time.",
"AI Agents"
],
[
"GrokAppAndroid",
"[xAI](https://x.ai)",
"AI Search Crawlers"
],
[
"GrokBot",
"[xAI](https://x.ai)",
"AI Search Crawlers"
],
[
"HarkBot",
"Unclear at this time.",
"AI Agents"
],
[
"HenkBot",
"Unclear at this time.",
"AI Data Providers"
],
[
"Hunyuan",
"[Tencent](https://hunyuan.tencent.com)",
"Hunyuan is a training crawler operated by Tencent."
],
[
"ICC-Crawler",
"[NICT](https://nict.go.jp)",
"Scrapes data to train and support AI technologies."
],
[
"ISSCyberRiskCrawler",
"[ISS-Corporate](https://iss-cyber.com)",
"Scrapes data to train machine learning models."
],
[
"ImagesiftBot",
"[ImageSift](https://imagesift.com)",
"ImageSiftBot is a web crawler that scrapes the internet for publicly available images to support their suite of web intelligence products"
],
[
"Instinct",
"Unclear at this time.",
"AI Agents"
],
[
"Kangaroo Bot",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"KeenableBot",
"[Keenable](https://keenable.ai/)",
"AI Search Crawlers"
],
[
"Kimi-Agent",
"Moonshot AI that browses websites while carrying out user-directed research and other tasks",
"AI Agents"
],
[
"Kimi-SearchBot",
"[Moonshot AI](https://www.moonshot.ai)",
"AI Search Crawlers"
],
[
"Kimi-User",
"Moonshot AI that fetches web content on behalf of users interacting with Kimi",
"AI Assistants"
],
[
"KimiBot",
"[Moonshot AI](https://www.moonshot.ai)",
"AI Data Scrapers"
],
[
"KlaviyoAIBot",
"[Klaviyo](https://www.klaviyo.com)",
"AI Assistants"
],
[
"KunatoCrawler",
"Unclear at this time.",
"Undocumented AI Agents"
],
[
"LAIONDownloader",
"[Large-scale Artificial Intelligence Open Network](https://laion.ai/)",
"AI tools and models for machine learning research."
],
[
"LCC",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"Lightpanda",
"Anyone who downloads the Lightpanda client. Possibly being used by a [Grok-adjacent](https://github.com/lightpanda-io/browser/issues/3156#issuecomment-5217843616) organization's botnet.",
"AI Data Scrapers"
],
[
"LinerBot",
"Unclear at this time.",
"AI Assistants"
],
[
"Linguee Bot",
"[Linguee](https://www.linguee.com)",
"AI powered translation service"
],
[
"LinkupBot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"Manus-User",
"Butterfly Effect, a company based in China",
"AI Agents"
],
[
"Meta-ExternalAgent",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"Meta-ExternalFetcher",
"Unclear at this time.",
"AI Assistants"
],
[
"MistralAI-Index",
"Mistral AI",
"Indexes web content for Mistral AI's search, used to answer questions in Le Chat. Per Mistral, not used for model training."
],
[
"MistralAI-Training",
"[Mistral AI](https://mistral.ai)",
"AI Data Scrapers"
],
[
"MistralAI-User",
"Mistral",
"AI Assistants"
],
[
"MistralAI-User/1.0",
"Mistral AI",
"Takes action based on user prompts."
],
[
"MoonshotBot",
"[Kimi](https://www.kimi.ai/)",
"MoonshotBot is a training crawler."
],
[
"Mozilla-Tabstack",
"Mozilla that performs programmatic, AI-driven interactions with web content through Tabstack",
"AI Data Providers"
],
[
"Muse",
"Unclear at this time.",
"AI Agents"
],
[
"MyCentralAIScraperBot",
"Unclear at this time.",
"AI data scraper"
],
[
"NagetBot",
"Naget Inc (founded by Chris Samarinas, headquarter in Amherst, Massachusetts)",
"AI data scraper"
],
[
"NotebookLM",
"Unclear at this time.",
"AI Assistants"
],
[
"NovaAct",
"Unclear at this time.",
"AI Agents"
],
[
"OAI-AdsBot",
"[OpenAI](https://openai.com)",
"Validates and targets ads on ChatGPT."
],
[
"OAI-SearchBot",
"[OpenAI](https://openai.com)",
"Search result generation."
],
[
"OpenAI",
"[OpenAI](https://openai.com)",
"Unclear at this time."
],
[
"Operator",
"Unclear at this time.",
"AI Agents"
],
[
"PanguBot",
"the Chinese company Huawei",
"AI Data Scrapers"
],
[
"Panscient",
"[Panscient](https://panscient.com)",
"Data collection and analysis using machine learning and AI."
],
[
"Payhawk Invoice Fetching Agent",
"Payhawk that retrieves invoices from supplier websites for users and attaches them to expense records",
"AI Agents"
],
[
"Perplexity-User",
"[Perplexity](https://www.perplexity.ai/)",
"AI Assistants"
],
[
"PerplexityBot",
"[Perplexity](https://www.perplexity.ai/)",
"Search result generation."
],
[
"PetalBot",
"[Huawei](https://huawei.com/)",
"Used to provide recommendations in Hauwei assistant and AI search services."
],
[
"PhindBot",
"[phind](https://www.phind.com/)",
"AI Assistants"
],
[
"Poggio-Citations",
"Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan\u2026",
"AI Assistants"
],
[
"Poseidon Research Crawler",
"[Poseidon Research](https://www.poseidonresearch.com)",
"AI research crawler"
],
[
"QualifiedBot",
"[Qualified](https://www.qualified.com)",
"AI Assistants"
],
[
"Querit-SearchBot",
"Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg\u2026",
"AI Data Providers"
],
[
"QueritBot",
"Querit, a company providing a search API for large language model integration",
"AI Data Providers"
],
[
"QuillBot",
"[Quillbot](https://quillbot.com)",
"Company offers AI detection, writing tools and other services."
],
[
"QwenBot",
"Alibaba",
"Collects public web content for Alibaba's Qwen (Tongyi Qianwen) large language models."
],
[
"Reflectionbot",
"[Reflection](https://reflection.ai/)",
"Undocumented AI Agents"
],
[
"SBIntuitionsBot",
"[SB Intuitions](https://www.sbintuitions.co.jp/en/)",
"Uses data gathered in AI development and information analysis."
],
[
"Scrapy",
"[Zyte](https://www.zyte.com)",
"Scrapes data for a variety of uses including training AI."
],
[
"ScryBot",
"[Scry](https://scry.io/)",
"Used with AI agents to scrape data for answers to queries"
],
[
"SemrushBot-OCOB",
"[Semrush](https://www.semrush.com/)",
"Crawls your site for ContentShake AI tool."
],
[
"SemrushBot-SWA",
"[Semrush](https://www.semrush.com/)",
"Checks URLs on your site for SEO Writing Assistant."
],
[
"Shap-User",
"Unclear at this time.",
"AI Assistants"
],
[
"ShapBot",
"[Parallel](https://parallel.ai)",
"AI Data Providers"
],
[
"Sidetrade indexer bot",
"[Sidetrade](https://www.sidetrade.com)",
"Extracts data for a variety of uses including training AI."
],
[
"Spider",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"TavilyBot",
"Unclear at this time.",
"AI Data Providers"
],
[
"Terra Cotta",
"Unclear at this time.",
"AI Data Providers"
],
[
"TerraCotta",
"[Ceramic AI](https://ceramic.ai/)",
"AI Data Providers"
],
[
"Thinkbot",
"[Thinkbot](https://www.thinkbot.agency)",
"Insights on AI integration and automation."
],
[
"TikTokSpider",
"ByteDance",
"LLM training."
],
[
"Timpibot",
"[Timpi](https://timpi.io)",
"Scrapes data for use in training LLMs."
],
[
"TongyiBot",
"Alibaba that fetches web content for the Tongyi Qianwen assistant and related Qwen-generated answers",
"AI Assistants"
],
[
"Trae",
"Unclear at this time.",
"AI Coding Agents"
],
[
"TwinAgent",
"Twin, a platform that creates automated workers to perform tasks by integrating with APIs and controlling web applications through browser automa\u2026",
"AI Agents"
],
[
"UseAI",
"Unclear at this time.",
"AI Assistants"
],
[
"VelenPublicWebCrawler",
"[Velen Crawler](https://velen.io)",
"Scrapes data for business data sets and machine learning models."
],
[
"WARDBot",
"WEBSPARK",
"AI Data Scrapers"
],
[
"WRTNBot",
"Unclear at this time.",
"Undocumented AI Agents"
],
[
"Webzio-Extended",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"YaK",
"[Meltwater](https://www.meltwater.com/en/suite/consumer-intelligence)",
"According to the [Meltwater Consumer Intelligence page](https://www.meltwater.com/en/suite/consumer-intelligence) 'By applying AI, data science, and market research expertise to a live feed of global data sources, we transform unstructured data into actionable insights allowing better decision-making'."
],
[
"YandexAdditional",
"[Yandex](https://yandex.ru)",
"Scrapes/analyzes data for the YandexGPT LLM."
],
[
"YandexAdditionalBot",
"[Yandex](https://yandex.ru)",
"Scrapes/analyzes data for the YandexGPT LLM."
],
[
"YiBot",
"[01.ai](https://01.ai/), an AI company founded by Kai-Fu Lee.",
"This bot crawls the web to gather data and content for indexation and AI model training."
],
[
"YiyanBot",
"Baidu that fetches web content for the yiyan",
"AI Assistants"
],
[
"YouBot",
"[You](https://about.you.com/youchat/)",
"Scrapes data for search engine and LLMs."
],
[
"ZanistaBot",
"Unclear at this time.",
"AI Search Crawlers"
],
[
"aiHitBot",
"[aiHit](https://www.aihitdata.com/about)",
"A massive, artificial intelligence/machine learning, automated system."
],
[
"amazon-QBusiness",
"Unclear at this time.",
"AI Assistants"
],
[
"amazon-kendra",
"Amazon",
"Collects data for AI natural language search"
],
[
"anthropic-ai",
"[Anthropic](https://www.anthropic.com)",
"Scrapes data to train Anthropic's AI products."
],
[
"atlassian-bot",
"[Atlassian](https://www.atlassian.com)",
"AI search, assistants and agents"
],
[
"bedrockbot",
"[Amazon](https://amazon.com)",
"Data scraping for custom AI applications."
],
[
"bigsur.ai",
"Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi\u2026",
"AI Assistants"
],
[
"cohere-ai",
"[Cohere](https://cohere.com)",
"Retrieves data to provide responses to user-initiated prompts."
],
[
"cohere-training-data-crawler",
"Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products",
"AI Data Scrapers"
],
[
"facebookexternalhit",
"Meta/Facebook",
"Ostensibly only for sharing, but likely used as an AI crawler as well"
],
[
"iAskBot",
"Unclear at this time.",
"Undocumented AI Agents"
],
[
"iaskspider",
"Unclear at this time.",
"Undocumented AI Agents"
],
[
"iaskspider/2.0",
"iAsk",
"Crawls sites to provide answers to user queries."
],
[
"imageSpider",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"img2dataset",
"[img2dataset](https://github.com/rom1504/img2dataset)",
"Scrapes images for use in LLMs."
],
[
"kagi-fetcher",
"Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res\u2026",
"AI Assistants"
],
[
"laion-huggingface-processor",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"meta-externalagent",
"[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)",
"Used to train models and improve products."
],
[
"meta-externalfetcher",
"Unclear at this time.",
"AI Assistants"
],
[
"meta-webindexer",
"[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)",
"AI Assistants"
],
[
"nbot",
"[NBot](https://nbot.ai)",
"AI Search Crawlers"
],
[
"netEstate Imprint Crawler",
"netEstate",
"AI Data Scrapers"
],
[
"newsai",
"Unclear at this time.",
"AI data scraper"
],
[
"omgili",
"[Webz.io](https://webz.io/)",
"Data is sold."
],
[
"omgilibot",
"[Webz.io](https://webz.io/)",
"Data is sold."
],
[
"opencode",
"Unclear at this time.",
"AI Coding Agents"
],
[
"panscient.com",
"[Panscient](https://panscient.com)",
"Data collection and analysis using machine learning and AI."
],
[
"qodercli",
"Qoder that helps developers build, debug, and modify software from the terminal",
"AI Coding Agents"
],
[
"quillbot.com",
"[Quillbot](https://quillbot.com)",
"Company offers AI detection, writing tools and other services."
],
[
"webzio-extended",
"Unclear at this time.",
"AI Data Scrapers"
],
[
"wpbot",
"[QuantumCloud](https://www.quantumcloud.com)",
"Live chat support and lead generation."
],
[
"xAI-Grok",
"[xAI](https://x.ai/)",
"xAI-Grok is an xAI crawler token associated with Grok content collection."
]
],
"license": "MIT",
"source": "https://github.com/ai-robots-txt/ai.robots.txt @ d8e8b77ca6f04ee132fd5dec0ce842b5d45e85da"
},
"crawlers": {
"license": "MIT",
"patterns": [
[
"Googlebot\\/",
"search-engine"
],
[
"Googlebot-Mobile",
"search-engine"
],
[
"Googlebot-Image",
"search-engine"
],
[
"Googlebot-News",
"search-engine"
],
[
"Googlebot-Video",
"search-engine"
],
[
"AdsBot-Google([^-]|$)",
"advertising"
],
[
"AdsBot-Google-Mobile",
"advertising"
],
[
"Feedfetcher-Google",
"feed-reader"
],
[
"Mediapartners-Google",
"advertising"
],
[
"Mediapartners \\(Googlebot\\)",
"advertising"
],
[
"APIs-Google",
"search-engine"
],
[
"Google-InspectionTool",
"search-engine"
],
[
"Storebot-Google",
"search-engine"
],
[
"GoogleOther",
"search-engine"
],
[
"bingbot",
"search-engine"
],
[
"Slurp",
"search-engine"
],
[
"[wW]get",
"http-library"
],
[
"LinkedInBot",
"social-preview"
],
[
"Python-urllib",
"http-library"
],
[
"python-requests",
"http-library"
],
[
"aiohttp",
"http-library"
],
[
"httpx",
"http-library"
],
[
"libwww-perl",
"http-library"
],
[
"httpunit",
"http-library"
],
[
"Nutch",
"search-engine"
],
[
"Go-http-client",
"http-library"
],
[
"phpcrawl",
"http-library"
],
[
"msnbot",
"search-engine"
],
[
"jyxobot",
"search-engine"
],
[
"FAST-WebCrawler",
"search-engine"
],
[
"FAST Enterprise Crawler",
"search-engine"
],
[
"BIGLOTRON",
"search-engine"
],
[
"Teoma",
"search-engine"
],
[
"convera",
"search-engine"
],
[
"^Seekbot",
"search-engine"
],
[
"Gigabot",
"search-engine"
],
[
"Gigablast",
"search-engine"
],
[
"exabot",
"search-engine"
],
[
"ia_archiver",
"archiver"
],
[
"GingerCrawler",
"seo"
],
[
"webmon ",
"monitoring"
],
[
"HTTrack",
"http-library"
],
[
"grub\\.org",
"search-engine"
],
[
"UsineNouvelleCrawler",
"search-engine"
],
[
"antibot",
"scanner"
],
[
"netresearchserver",
"seo"
],
[
"speedy",
"search-engine"
],
[
"fluffy",
"search-engine"
],
[
"findlink",
"seo"
],
[
"msrbot",
"search-engine"
],
[
"panscient",
"seo"
],
[
"yacybot",
"search-engine"
],
[
"AISearchBot",
"search-engine,ai-crawler"
],
[
"ips-agent",
"seo"
],
[
"tagoobot",
"search-engine"
],
[
"MJ12bot",
"seo"
],
[
"woriobot",
"search-engine"
],
[
"yanga",
"search-engine"
],
[
"buzzbot",
"seo"
],
[
"mlbot",
"seo"
],
[
"yandex\\.com\\/bots",
"search-engine"
],
[
"purebot",
"search-engine"
],
[
"Linguee Bot",
"search-engine"
],
[
"CyberPatrol",
"scanner"
],
[
"voilabot",
"search-engine"
],
[
"Baiduspider",
"search-engine"
],
[
"citeseerxbot",
"academic"
],
[
"spbot",
"seo"
],
[
"twengabot",
"seo"
],
[
"postrank",
"seo"
],
[
"Turnitin",
"academic"
],
[
"scribdbot",
"search-engine"
],
[
"page2rss",
"feed-reader"
],
[
"sitebot",
"seo"
],
[
"linkdex",
"seo"
],
[
"Adidxbot",
"advertising,search-engine"
],
[
"ezooms",
"seo"
],
[
"dotbot",
"seo"
],
[
"Mail\\.RU_Bot",
"search-engine"
],
[
"discobot",
"search-engine"
],
[
"heritrix",
"archiver"
],
[
"findthatfile",
"search-engine"
],
[
"europarchive\\.org",
"archiver"
],
[
"NerdByNature\\.Bot",
"seo"
],
[
"(sistrix|SISTRIX) [cC]rawler",
"seo"
],
[
"Ahrefs(Bot|SiteAudit)",
"seo"
],
[
"fuelbot",
"seo"
],
[
"^CrunchBot",
"seo"
],
[
"IndeedBot",
"search-engine"
],
[
"mappydata",
"seo"
],
[
"woobot",
"seo"
],
[
"ZoominfoBot",
"seo"
],
[
"PrivacyAwareBot",
"seo"
],
[
"Multiviewbot",
"seo"
],
[
"SWIMGBot",
"seo"
],
[
"Grobbot",
"seo"
],
[
"eright",
"seo"
],
[
"Apercite",
"seo"
],
[
"semanticbot",
"seo"
],
[
"Aboundex",
"search-engine"
],
[
"domaincrawler",
"seo"
],
[
"wbsearchbot",
"search-engine"
],
[
"summify",
"seo"
],
[
"CCBot",
"ai-crawler"
],
[
"edisterbot",
"search-engine"
],
[
"SeznamBot",
"search-engine"
],
[
"ec2linkfinder",
"scanner"
],
[
"gslfbot",
"seo"
],
[
"aiHitBot",
"search-engine"
],
[
"intelium_bot",
"seo"
],
[
"facebookexternalhit",
"social-preview"
],
[
"Yeti",
"search-engine"
],
[
"RetrevoPageAnalyzer",
"seo"
],
[
"lb-spider",
"search-engine"
],
[
"Sogou",
"search-engine"
],
[
"lssbot",
"seo"
],
[
"careerbot",
"seo"
],
[
"wotbox",
"search-engine"
],
[
"wocbot",
"search-engine"
],
[
"ichiro",
"search-engine"
],
[
"DuckDuckBot",
"search-engine"
],
[
"lssrocketcrawler",
"seo"
],
[
"drupact",
"seo"
],
[
"webcompanycrawler",
"seo"
],
[
"acoonbot",
"search-engine"
],
[
"openindexspider",
"search-engine"
],
[
"gnam gnam spider",
"seo"
],
[
"web-archive-net\\.com\\.bot",
"archiver"
],
[
"backlinkcrawler",
"seo"
],
[
"coccoc",
"search-engine"
],
[
"integromedb",
"academic"
],
[
"content crawler spider",
"seo"
],
[
"toplistbot",
"seo"
],
[
"it2media-domain-crawler",
"seo"
],
[
"ip-web-crawler\\.com",
"scanner"
],
[
"siteexplorer\\.info",
"seo"
],
[
"elisabot",
"seo"
],
[
"proximic",
"advertising"
],
[
"changedetection",
"monitoring"
],
[
"arabot",
"search-engine"
],
[
"WeSEE:Search",
"search-engine"
],
[
"niki-bot",
"seo"
],
[
"CrystalSemanticsBot",
"seo"
],
[
"rogerbot",
"seo"
],
[
"360Spider",
"search-engine"
],
[
"psbot",
"search-engine"
],
[
"InterfaxScanBot",
"seo"
],
[
"CC Metadata Scaper",
"archiver"
],
[
"g00g1e\\.net",
"seo"
],
[
"GrapeshotCrawler",
"advertising"
],
[
"urlappendbot",
"seo"
],
[
"brainobot",
"seo"
],
[
"fr-crawler",
"search-engine"
],
[
"binlar",
"scanner"
],
[
"SimpleCrawler",
"http-library"
],
[
"Twitterbot",
"social-preview"
],
[
"cXensebot",
"advertising"
],
[
"smtbot",
"seo"
],
[
"bnf\\.fr_bot",
"archiver"
],
[
"A6-Indexer",
"search-engine"
],
[
"ADmantX",
"advertising"
],
[
"Facebot",
"social-preview"
],
[
"OrangeBot\\/",
"search-engine"
],
[
"memorybot",
"seo"
],
[
"AdvBot",
"advertising"
],
[
"MegaIndex",
"seo"
],
[
"SemanticScholarBot",
"academic"
],
[
"ltx71",
"scanner"
],
[
"nerdybot",
"seo"
],
[
"xovibot",
"seo"
],
[
"BUbiNG",
"academic"
],
[
"Qwantify",
"search-engine"
],
[
"archive\\.org_bot",
"archiver"
],
[
"Applebot",
"search-engine"
],
[
"TweetmemeBot",
"social-preview"
],
[
"crawler4j",
"http-library"
],
[
"findxbot",
"search-engine"
],
[
"S[eE][mM]rushBot",
"seo"
],
[
"yoozBot",
"seo"
],
[
"lipperhey",
"seo"
],
[
"Y!J",
"search-engine"
],
[
"Domain Re-Animator Bot",
"seo"
],
[
"AddThis",
"social-preview"
],
[
"Screaming Frog SEO Spider",
"seo"
],
[
"MetaURI",
"seo"
],
[
"Scrapy",
"http-library"
],
[
"Livelap[bB]ot",
"seo"
],
[
"OpenHoseBot",
"search-engine"
],
[
"CapsuleChecker",
"monitoring"
],
[
"collection@infegy\\.com",
"seo"
],
[
"IstellaBot",
"search-engine"
],
[
"DeuSu\\/",
"search-engine"
],
[
"betaBot",
"seo"
],
[
"Cliqzbot\\/",
"search-engine"
],
[
"MojeekBot\\/",
"search-engine"
],
[
"netEstate NE Crawler",
"seo"
],
[
"SafeSearch microdata crawler",
"search-engine"
],
[
"Gluten Free Crawler\\/",
"search-engine"
],
[
"Sonic",
"seo"
],
[
"Sysomos",
"seo"
],
[
"Trove",
"archiver"
],
[
"deadlinkchecker",
"seo"
],
[
"Slack-ImgProxy",
"social-preview"
],
[
"Embedly",
"social-preview"
],
[
"RankActiveLinkBot",
"seo"
],
[
"iskanie",
"seo"
],
[
"SafeDNSBot",
"scanner"
],
[
"SkypeUriPreview",
"social-preview"
],
[
"Veoozbot",
"seo"
],
[
"Slackbot",
"social-preview"
],
[
"redditbot",
"social-preview"
],
[
"datagnionbot",
"seo"
],
[
"Google-Adwords-Instant",
"advertising"
],
[
"adbeat_bot",
"advertising"
],
[
"WhatsApp",
"social-preview"
],
[
"contxbot",
"seo"
],
[
"pinterest\\.com\\/bot",
"social-preview"
],
[
"electricmonk",
"seo"
],
[
"GarlikCrawler",
"seo"
],
[
"BingPreview\\/",
"social-preview"
],
[
"vebidoobot",
"seo"
],
[
"FemtosearchBot",
"search-engine"
],
[
"Yahoo Link Preview",
"social-preview"
],
[
"MetaJobBot",
"seo"
],
[
"DomainStatsBot",
"seo"
],
[
"mindUpBot",
"seo"
],
[
"Daum\\/",
"search-engine"
],
[
"Jugendschutzprogramm-Crawler",
"scanner"
],
[
"Xenu Link Sleuth",
"seo"
],
[
"Pcore-HTTP",
"http-library"
],
[
"moatbot",
"advertising"
],
[
"KosmioBot",
"seo"
],
[
"[pP]ingdom",
"monitoring"
],
[
"AppInsights",
"monitoring"
],
[
"PhantomJS",
"browser-automation"
],
[
"Gowikibot",
"seo"
],
[
"PiplBot",
"seo"
],
[
"Discordbot",
"social-preview"
],
[
"TelegramBot",
"social-preview"
],
[
"Jetslide",
"social-preview"
],
[
"newsharecounts",
"social-preview"
],
[
"James BOT",
"seo"
],
[
"Bark[rR]owler",
"seo"
],
[
"TinEye",
"seo"
],
[
"SocialRankIOBot",
"seo"
],
[
"trendictionbot",
"seo"
],
[
"Ocarinabot",
"seo"
],
[
"epicbot",
"seo"
],
[
"Primalbot",
"seo"
],
[
"DuckDuckGo-Favicons-Bot",
"search-engine"
],
[
"GnowitNewsbot",
"seo"
],
[
"Leikibot",
"seo"
],
[
"LinkArchiver",
"archiver"
],
[
"YaK\\/",
"seo"
],
[
"PaperLiBot",
"seo"
],
[
"Digg Deeper",
"seo"
],
[
"^dcrawl",
"seo"
],
[
"Snacktory",
"seo"
],
[
"AndersPinkBot",
"seo"
],
[
"Fyrebot",
"seo"
],
[
"EveryoneSocialBot",
"social-preview"
],
[
"Mediatoolkitbot",
"monitoring"
],
[
"Luminator-robots",
"seo"
],
[
"ExtLinksBot",
"seo"
],
[
"SurveyBot",
"seo"
],
[
"NING\\/",
"social-preview"
],
[
"okhttp",
"http-library"
],
[
"Nuzzel",
"seo"
],
[
"omgili",
"seo"
],
[
"PocketParser",
"seo"
],
[
"YisouSpider",
"search-engine"
],
[
"um-LN",
"seo"
],
[
"ToutiaoSpider",
"search-engine"
],
[
"MuckRack",
"seo"
],
[
"Jamie's Spider",
"seo"
],
[
"AHC\\/",
"http-library"
],
[
"NetcraftSurveyAgent",
"scanner"
],
[
"Laserlikebot",
"seo"
],
[
"^Apache-HttpClient",
"http-library"
],
[
"AppEngine-Google",
"search-engine"
],
[
"Jetty",
"http-library"
],
[
"Upflow",
"seo"
],
[
"Thinklab",
"academic"
],
[
"Traackr\\.com",
"seo"
],
[
"Twurly",
"seo"
],
[
"Mastodon",
"social-preview"
],
[
"http_get",
"http-library"
],
[
"DnyzBot",
"seo"
],
[
"botify",
"seo"
],
[
"007ac9 Crawler",
"seo"
],
[
"BehloolBot",
"seo"
],
[
"BrandVerity",
"seo"
],
[
"check_http",
"monitoring"
],
[
"BDCbot",
"seo"
],
[
"ZumBot",
"search-engine"
],
[
"EZID",
"seo"
],
[
"ICC-Crawler",
"academic"
],
[
"ArchiveBot",
"archiver"
],
[
"^LCC ",
"academic"
],
[
"filterdb\\.iss\\.net\\/crawler",
"scanner"
],
[
"BLP_bbot",
"seo"
],
[
"BomboraBot",
"seo"
],
[
"Buck\\/",
"monitoring"
],
[
"Companybook-Crawler",
"seo"
],
[
"Genieo",
"seo"
],
[
"magpie-crawler",
"seo"
],
[
"MeltwaterNews",
"seo"
],
[
"Moreover",
"seo"
],
[
"newspaper\\/",
"http-library"
],
[
"ScoutJet",
"seo"
],
[
"(^| )sentry\\/",
"monitoring"
],
[
"StorygizeBot",
"seo"
],
[
"UptimeRobot",
"monitoring"
],
[
"OutclicksBot",
"advertising"
],
[
"seoscanners",
"seo"
],
[
"Hatena",
"social-preview"
],
[
"Google Web Preview",
"social-preview"
],
[
"MauiBot",
"seo"
],
[
"AlphaBot",
"seo"
],
[
"SBL-BOT",
"scanner"
],
[
"IAS crawler",
"advertising"
],
[
"adscanner",
"advertising"
],
[
"Netvibes",
"feed-reader"
],
[
"acapbot",
"seo"
],
[
"Baidu-YunGuanCe",
"monitoring"
],
[
"bitlybot",
"seo"
],
[
"blogmuraBot",
"seo"
],
[
"Bot\\.AraTurka\\.com",
"search-engine"
],
[
"bot-pge\\.chlooe\\.com",
"seo"
],
[
"BoxcarBot",
"seo"
],
[
"BTWebClient",
"http-library"
],
[
"ContextAd Bot",
"advertising"
],
[
"Digincore bot",
"seo"
],
[
"Disqus",
"seo"
],
[
"Feedly",
"feed-reader"
],
[
"Fetch\\/",
"seo"
],
[
"Fever",
"feed-reader"
],
[
"Flamingo_SearchEngine",
"search-engine"
],
[
"FlipboardProxy",
"social-preview"
],
[
"g2reader-bot",
"feed-reader"
],
[
"G2 Web Services",
"seo"
],
[
"imrbot",
"seo"
],
[
"K7MLWCBot",
"scanner"
],
[
"Kemvibot",
"seo"
],
[
"Landau-Media-Spider",
"seo"
],
[
"linkapediabot",
"seo"
],
[
"vkShare",
"social-preview"
],
[
"Siteimprove\\.com",
"monitoring"
],
[
"BLEXBot\\/",
"seo"
],
[
"DareBoost",
"monitoring"
],
[
"ZuperlistBot\\/",
"seo"
],
[
"Miniflux\\/",
"feed-reader"
],
[
"Feedspot",
"feed-reader"
],
[
"Diffbot\\/",
"seo"
],
[
"SEOkicks",
"seo"
],
[
"tracemyfile",
"seo"
],
[
"Nimbostratus-Bot",
"seo"
],
[
"zgrab",
"scanner"
],
[
"PR-CY\\.RU",
"seo"
],
[
"AdsTxtCrawler",
"advertising"
],
[
"Datafeedwatch",
"seo"
],
[
"Zabbix",
"monitoring"
],
[
"TangibleeBot",
"seo"
],
[
"google-xrawler",
"search-engine"
],
[
"axios",
"http-library"
],
[
"Amazon CloudFront",
"http-library"
],
[
"Pulsepoint ",
"advertising"
],
[
"CloudFlare-AlwaysOnline",
"monitoring"
],
[
"Cloudflare-Healthchecks",
"monitoring"
],
[
"Cloudflare-Traffic-Manager",
"monitoring"
],
[
"CloudFlare-Prefetch",
"seo"
],
[
"Cloudflare-SSLDetector",
"scanner"
],
[
"https:\\/\\/developers\\.cloudflare\\.com\\/security-center\\/",
"scanner"
],
[
"Google-Structured-Data-Testing-Tool",
"search-engine"
],
[
"WordupInfoSearch",
"search-engine"
],
[
"WebDataStats",
"seo"
],
[
"HttpUrlConnection",
"http-library"
],
[
"ZoomBot",
"seo"
],
[
"VelenPublicWebCrawler",
"seo"
],
[
"MoodleBot",
"seo"
],
[
"jpg-newsbot",
"seo"
],
[
"outbrain",
"advertising"
],
[
"W3C_Validator",
"monitoring"
],
[
"Validator\\.nu",
"monitoring"
],
[
"W3C-checklink",
"monitoring"
],
[
"W3C-mobileOK",
"monitoring"
],
[
"W3C_I18n-Checker",
"monitoring"
],
[
"FeedValidator",
"monitoring"
],
[
"W3C_CSS_Validator",
"monitoring"
],
[
"W3C_Unicorn",
"monitoring"
],
[
"Google-PhysicalWeb",
"search-engine"
],
[
"Blackboard",
"academic"
],
[
"ICBot\\/",
"seo"
],
[
"BazQux",
"feed-reader"
],
[
"Twingly",
"seo"
],
[
"Rivva",
"seo"
],
[
"Experibot",
"monitoring"
],
[
"awesomecrawler",
"seo"
],
[
"Dataprovider\\.com",
"seo"
],
[
"GroupHigh\\/",
"seo"
],
[
"theoldreader\\.com",
"feed-reader"
],
[
"AnyEvent",
"http-library"
],
[
"Uptimebot\\.org",
"monitoring"
],
[
"Nmap Scripting Engine",
"scanner"
],
[
"2ip\\.ru",
"seo"
],
[
"Clickagy",
"advertising"
],
[
"Caliperbot",
"seo"
],
[
"MBCrawler",
"seo"
],
[
"online-webceo-bot",
"seo"
],
[
"B2B Bot",
"seo"
],
[
"AddSearchBot",
"search-engine"
],
[
"Google Favicon",
"search-engine"
],
[
"HubSpot",
"seo"
],
[
"Chrome-Lighthouse",
"monitoring"
],
[
"HeadlessChrome",
"browser-automation"
],
[
"CheckMarkNetwork\\/",
"monitoring"
],
[
"www\\.uptime\\.com",
"monitoring"
],
[
"Streamline3Bot\\/",
"monitoring"
],
[
"serpstatbot\\/",
"seo"
],
[
"MixnodeCache\\/",
"seo"
],
[
"^curl",
"http-library"
],
[
"SimpleScraper",
"seo"
],
[
"RSSingBot",
"feed-reader"
],
[
"Jooblebot",
"search-engine"
],
[
"fedoraplanet",
"feed-reader"
],
[
"Friendica",
"social-preview"
],
[
"NextCloud",
"feed-reader"
],
[
"Tiny Tiny RSS",
"feed-reader"
],
[
"RegionStuttgartBot",
"seo"
],
[
"Bytespider",
"ai-crawler"
],
[
"Datanyze",
"seo"
],
[
"Google-Site-Verification",
"search-engine"
],
[
"TrendsmapResolver",
"seo"
],
[
"tweetedtimes",
"seo"
],
[
"NTENTbot",
"search-engine"
],
[
"Gwene",
"feed-reader"
],
[
"SimplePie",
"feed-reader"
],
[
"SearchAtlas",
"seo"
],
[
"Superfeedr",
"feed-reader"
],
[
"feedbot",
"feed-reader"
],
[
"UT-Dorkbot",
"scanner"
],
[
"Amazonbot",
"search-engine"
],
[
"AmazonProductDiscovery",
"search-engine"
],
[
"AmazonSellerInitiatedListing",
"search-engine"
],
[
"SerendeputyBot",
"seo"
],
[
"Eyeotabot",
"advertising"
],
[
"officestorebot",
"search-engine"
],
[
"Neticle Crawler",
"seo"
],
[
"SurdotlyBot",
"seo"
],
[
"LinkisBot",
"seo"
],
[
"AwarioSmartBot",
"seo"
],
[
"AwarioRssBot",
"feed-reader,seo"
],
[
"RyteBot",
"seo"
],
[
"FreeWebMonitoring SiteChecker",
"monitoring"
],
[
"AspiegelBot",
"search-engine"
],
[
"NAVER Blog Rssbot",
"feed-reader"
],
[
"zenback bot",
"seo"
],
[
"SentiBot",
"seo"
],
[
"Domains Project\\/",
"seo"
],
[
"Pandalytics",
"seo"
],
[
"VKRobot",
"social-preview"
],
[
"bidswitchbot",
"advertising"
],
[
"tigerbot",
"seo"
],
[
"NIXStatsbot",
"monitoring"
],
[
"Atom Feed Robot",
"feed-reader"
],
[
"[Cc]urebot",
"monitoring"
],
[
"PagePeeker\\/",
"seo"
],
[
"Vigil\\/",
"monitoring"
],
[
"rssbot\\/",
"feed-reader"
],
[
"startmebot\\/",
"seo"
],
[
"JobboerseBot",
"search-engine"
],
[
"seewithkids",
"seo"
],
[
"NINJA bot",
"scanner"
],
[
"Cutbot",
"seo"
],
[
"BublupBot",
"seo"
],
[
"BrandONbot",
"seo"
],
[
"RidderBot",
"seo"
],
[
"Taboolabot",
"advertising"
],
[
"Dubbotbot",
"monitoring"
],
[
"FindITAnswersbot",
"seo"
],
[
"infoobot",
"seo"
],
[
"Refindbot",
"seo"
],
[
"BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher",
"feed-reader"
],
[
"SeobilityBot",
"seo"
],
[
"Cincraw",
"seo"
],
[
"Dragonbot",
"seo"
],
[
"VoluumDSP-content-bot",
"advertising"
],
[
"FreshRSS",
"feed-reader"
],
[
"BitBot",
"seo"
],
[
"^PHP-Curl-Class",
"http-library"
],
[
"Google-Certificates-Bridge",
"search-engine"
],
[
"centurybot",
"seo"
],
[
"Viber",
"social-preview"
],
[
"e\\.ventures Investment Crawler",
"seo"
],
[
"evc-batch",
"seo"
],
[
"PetalBot",
"search-engine"
],
[
"virustotal",
"scanner"
],
[
"(^| )PTST\\/",
"monitoring"
],
[
"minicrawler",
"monitoring"
],
[
"Cookiebot",
"scanner"
],
[
"trovitBot",
"search-engine"
],
[
"seostar\\.co",
"seo"
],
[
"IonCrawl",
"seo"
],
[
"Uptime-Kuma",
"monitoring"
],
[
"Seekport",
"search-engine"
],
[
"FreshpingBot",
"monitoring"
],
[
"Feedbin",
"feed-reader"
],
[
"CriteoBot",
"advertising"
],
[
"Snap URL Preview Service",
"social-preview"
],
[
"Better Uptime Bot",
"monitoring"
],
[
"RuxitSynthetic",
"monitoring"
],
[
"Google-Read-Aloud",
"search-engine"
],
[
"Valve\\/Steam",
"social-preview"
],
[
"OdklBot\\/",
"social-preview"
],
[
"GPTBot",
"ai-crawler"
],
[
"ChatGPT-User",
"ai-crawler"
],
[
"OAI-SearchBot",
"ai-crawler,search-engine"
],
[
"YandexRenderResourcesBot\\/",
"search-engine"
],
[
"LightspeedSystemsCrawler",
"scanner"
],
[
"ev-crawler\\/",
"seo"
],
[
"BitSightBot\\/",
"scanner"
],
[
"woorankreview\\/",
"seo"
],
[
"Google-Safety",
"search-engine"
],
[
"AwarioBot",
"seo"
],
[
"DataForSeoBot",
"seo"
],
[
"Linespider",
"social-preview"
],
[
"WellKnownBot",
"seo"
],
[
"A Patent Crawler",
"academic"
],
[
"StractBot",
"search-engine"
],
[
"search\\.marginalia\\.nu",
"search-engine"
],
[
"YouBot",
"search-engine"
],
[
"Nicecrawler",
"search-engine"
],
[
"Neevabot",
"search-engine"
],
[
"BrightEdge Crawler",
"seo"
],
[
"SiteCheckerBotCrawler",
"seo"
],
[
"TombaPublicWebCrawler",
"seo"
],
[
"CrawlyProjectCrawler",
"seo"
],
[
"KomodiaBot",
"seo"
],
[
"KStandBot",
"seo"
],
[
"CISPA Webcrawler",
"academic"
],
[
"MTRobot",
"seo"
],
[
"hyscore\\.io",
"seo"
],
[
"AlexandriaOrgBot",
"search-engine"
],
[
"2ip bot",
"seo"
],
[
"Yellowbrandprotectionbot",
"seo"
],
[
"SEOlizer",
"seo"
],
[
"vuhuvBot",
"seo"
],
[
"INETDEX-BOT",
"search-engine"
],
[
"Synapse",
"social-preview"
],
[
"t3versionsBot",
"seo"
],
[
"deepnoc",
"monitoring"
],
[
"Cocolyzebot",
"seo"
],
[
"hypestat",
"seo"
],
[
"ReverseEngineeringBot",
"scanner"
],
[
"sempi\\.tech",
"seo"
],
[
"Iframely",
"social-preview"
],
[
"MetaInspector",
"seo"
],
[
"node-fetch",
"http-library"
],
[
"l9explore",
"scanner"
],
[
"python-opengraph",
"http-library"
],
[
"OpenGraphCheck",
"seo"
],
[
"developers\\.google\\.com\\/\\+\\/web\\/snippet",
"search-engine"
],
[
"SenutoBot",
"seo"
],
[
"MaCoCu",
"academic"
],
[
"NewsBlur",
"feed-reader"
],
[
"inoreader",
"feed-reader"
],
[
"NetSystemsResearch",
"scanner"
],
[
"PageThing",
"seo"
],
[
"WordPress\\/",
"seo"
],
[
"PhxBot",
"search-engine"
],
[
"ImagesiftBot",
"search-engine"
],
[
"Expanse",
"scanner"
],
[
"InternetMeasurement",
"academic"
],
[
"^BW\\/",
"seo"
],
[
"GeedoBot",
"search-engine"
],
[
"Audisto Crawler",
"seo"
],
[
"PerplexityBot\\/",
"ai-crawler,search-engine"
],
[
"[cC]laude[bB]ot",
"ai-crawler"
],
[
"Monsidobot",
"monitoring"
],
[
"GroupMeBot",
"social-preview"
],
[
"Vercelbot",
"seo"
],
[
"vercel-screenshot",
"seo"
],
[
"facebookcatalog\\/",
"advertising"
],
[
"meta-externalads\\/",
"advertising"
],
[
"meta-externalagent\\/",
"ai-crawler,social-preview"
],
[
"meta-externalfetcher\\/",
"social-preview"
],
[
"AcademicBotRTU",
"academic"
],
[
"KeybaseBot",
"social-preview"
],
[
"Lemmy",
"social-preview"
],
[
"CookieHubScan",
"scanner"
],
[
"Hydrozen\\.io",
"monitoring"
],
[
"HTTP Banner Detection",
"scanner"
],
[
"SummalyBot",
"social-preview"
],
[
"MicrosoftPreview\\/",
"social-preview"
],
[
"GeedoProductSearch",
"search-engine"
],
[
"TikTokSpider",
"ai-crawler,social-preview"
],
[
"OnCrawl\\/",
"seo"
],
[
"sindresorhus\\/got",
"http-library"
],
[
"CensysInspect\\/",
"scanner"
],
[
"SBIntuitionsBot\\/",
"ai-crawler"
],
[
"sitebulb",
"seo"
],
[
"YextBot\\/",
"seo"
],
[
"DatadogSynthetics",
"monitoring"
],
[
"Google-Ads-Conversions",
"advertising"
],
[
"ObservePoint",
"monitoring"
],
[
"Checkly",
"monitoring"
],
[
"ALittle Client",
"http-library"
],
[
"AliyunSecBot",
"scanner"
],
[
"Claude-Web",
"ai-crawler"
],
[
"anthropic-ai",
"ai-crawler"
],
[
"Claude-User",
"ai-crawler"
],
[
"Claude-SearchBot",
"ai-crawler,search-engine"
],
[
"Google-Extended",
"ai-crawler"
],
[
"cohere-ai",
"ai-crawler"
],
[
"Timpibot",
"search-engine"
],
[
"SERankingBacklinksBot",
"seo"
],
[
"CMSChecker",
"seo"
],
[
"Wayback",
"archiver"
],
[
"Playwright",
"browser-automation"
],
[
"Puppeteer",
"browser-automation"
],
[
"Selenium",
"browser-automation"
],
[
"Nikto",
"scanner"
],
[
"sqlmap",
"scanner"
],
[
"ZmEu",
"scanner"
],
[
"masscan",
"scanner"
],
[
"WPScan",
"scanner"
],
[
"[aA]cunetix",
"scanner"
],
[
"Nessus",
"scanner"
],
[
"[dD]ir[Bb]uster",
"scanner"
],
[
"StatusCake",
"monitoring"
],
[
"colly",
"http-library"
],
[
"[mM]echanize",
"http-library"
],
[
"air\\.ai\\/scanning",
"scanner"
],
[
"asnriskscorer",
"scanner"
],
[
"OICrawler",
"search-engine"
],
[
"l9scan",
"scanner"
],
[
"SlaccaleBot",
"seo"
],
[
"CustomAsyncHttpClient",
"http-library"
],
[
"^HTTPie\\/",
"http-library"
],
[
"Gemini-Deep-Research",
"ai-crawler,search-engine"
],
[
"Perplexity-User",
"ai-crawler,search-engine"
],
[
"PerplexityUser",
"ai-crawler,search-engine"
],
[
"meta-webindexer",
"social-preview"
],
[
"DuckAssistBot",
"search-engine,ai-crawler"
],
[
"MistralAI-User",
"ai-crawler"
],
[
"webzio",
"seo"
],
[
"newsai\\/",
"ai-crawler,feed-reader"
],
[
"^ArenaUnfurlBot",
"social-preview"
],
[
"A360-Search",
"advertising"
],
[
"AASA-Bot",
"social-preview"
],
[
"AccessStatus",
"monitoring"
],
[
"Acquia optimize",
"monitoring"
],
[
"ActiveComply",
"monitoring"
],
[
"AdkernelTopicCrawler",
"advertising"
],
[
"AlertSite",
"monitoring"
],
[
"AllAfrica",
"feed-reader"
],
[
"Amazing-SearchBot",
"advertising"
],
[
"Amazon-Bedrock-AgentCore-Browser",
"browser-automation"
],
[
"AmazonBuyForMe",
"browser-automation"
],
[
"Amzn-SearchBot",
"ai-crawler"
],
[
"Amzn-User",
"ai-crawler"
],
[
"Anchor Browser",
"browser-automation"
],
[
"Anomura",
"ai-crawler"
],
[
"AP3A\\.240617\\.008",
"scanner"
],
[
"ApifyBot",
"ai-crawler"
],
[
"ApifyWebsiteContentCrawler",
"ai-crawler"
],
[
"Archive-It",
"archiver"
],
[
"artemis web reader",
"feed-reader"
],
[
"atlassian-bot",
"ai-crawler"
],
[
"Attracta",
"seo"
],
[
"AudigentAdBot",
"advertising"
],
[
"Authory",
"archiver"
],
[
"Automaton|Newsify Feed Fetcher",
"monitoring"
],
[
"AwarioRendererBot",
"advertising"
],
[
"AzureAI-SearchBot",
"ai-crawler"
],
[
"BestChange",
"feed-reader"
],
[
"bigsur\\.ai",
"ai-crawler"
],
[
"bl\\.uk_lddc_bot",
"archiver"
],
[
"BlingERP",
"feed-reader"
],
[
"Blockaid",
"monitoring"
],
[
"Bloglines",
"feed-reader"
],
[
"BlogVault",
"monitoring"
],
[
"bluesky-domain-status-classifier",
"social-preview"
],
[
"Bluesky\\/",
"social-preview"
],
[
"bne\\.es_bot",
"archiver"
],
[
"Brightbot",
"ai-crawler"
],
[
"BrowserBot-Observer",
"scanner"
],
[
"BufferLinkPreviewBot",
"social-preview"
],
[
"Bugsnag",
"monitoring"
],
[
"Buttondown",
"feed-reader"
],
[
"CapitalOneBot",
"feed-reader"
],
[
"CertChief",
"scanner"
],
[
"channable",
"advertising"
],
[
"Channel3Bot",
"ai-crawler"
],
[
"Chirp|gotosocial",
"monitoring"
],
[
"ClickUpLinkUnfurler",
"social-preview"
],
[
"Cloudflare-AutoRAG",
"ai-crawler"
],
[
"Cloudflare-Custom-Hostname-Verification",
"seo"
],
[
"Cloudflare-Stream-Webhook",
"seo"
],
[
"CloudflareRadarURLScanner",
"monitoring"
],
[
"Cloudtrellis",
"seo"
],
[
"[cC]ludo",
"monitoring"
],
[
"Code\\/1\\.",
"browser-automation"
],
[
"Collapsify",
"feed-reader"
],
[
"ContextualBot[\\s\\S]*outcomes\\.net",
"advertising"
],
[
"Convermax",
"seo"
],
[
"cookie-maestro",
"scanner"
],
[
"CookieHubVerify",
"scanner"
],
[
"CookieYesbot",
"scanner"
],
[
"Crazy Egg",
"monitoring"
],
[
"Current[\\s\\S]*RSS Reader",
"feed-reader"
],
[
"cypex\\.ai\\/scanning",
"scanner"
],
[
"DeepCrawl",
"seo"
],
[
"DigiCert DCV",
"scanner"
],
[
"dlvr\\.it",
"feed-reader"
],
[
"Dotcom-Monitor",
"monitoring"
],
[
"DrataAutopilot",
"monitoring"
],
[
"DreamHost Data Team",
"monitoring"
],
[
"ds9",
"advertising"
],
[
" DVbot",
"advertising"
],
[
"EcoVadisSustainabilityBot",
"advertising"
],
[
"elmah\\.io Uptime Monitoring",
"monitoring"
],
[
"EvernoteRichLinkBot",
"social-preview"
],
[
"EzLynx",
"advertising"
],
[
"EzoicBot",
"advertising"
],
[
"FacebookBot",
"ai-crawler"
],
[
"FastDAST",
"monitoring"
],
[
"Feeder \\/",
"feed-reader"
],
[
"FeedFlow",
"feed-reader"
],
[
"FindFiles\\.net",
"search-engine"
],
[
"FirecrawlAgent",
"ai-crawler"
],
[
"FyndSearchEngine-Crawler",
"search-engine"
],
[
"FyndSearchEngine-ReCrawler",
"search-engine"
],
[
"Goodreads",
"social-preview"
],
[
"Google Trust Services",
"monitoring"
],
[
"Google-Agent",
"browser-automation"
],
[
"Google-Gemini-CLI",
"browser-automation"
],
[
"Google-NotebookLM",
"ai-crawler"
],
[
"GoogleAgent-Mariner",
"browser-automation"
],
[
"Greppr Web Crawler",
"search-engine"
],
[
"Hardenize",
"scanner"
],
[
"HoneybadgerBot",
"monitoring"
],
[
"IbouBot",
"search-engine"
],
[
"imageSpider",
"ai-crawler"
],
[
"Innologica",
"feed-reader"
],
[
"kagi-fetcher",
"ai-crawler"
],
[
"Kangaroo Bot",
"ai-crawler"
],
[
"Known Agent",
"monitoring"
],
[
"KrawlerBot",
"scanner"
],
[
"laion-huggingface-processor",
"ai-crawler"
],
[
"LinkCheckerBot",
"seo"
],
[
"LinkupBot",
"ai-crawler"
],
[
"LMArenaUnfurlBot",
"social-preview"
],
[
"lyonl-asset-proxy",
"search-engine"
],
[
"lyonl-crawler",
"search-engine"
],
[
"MagiBot",
"search-engine"
],
[
"MagpieRSS",
"feed-reader"
],
[
"mail\\.ru",
"feed-reader"
],
[
"MailChimp",
"feed-reader"
],
[
"Manus-User",
"browser-automation"
],
[
"McontextualBot",
"advertising"
],
[
"Mediumbot-MetaTagFetcher",
"social-preview"
],
[
"MetaIAB Facebook",
"social-preview"
],
[
"MixrankBot",
"advertising"
],
[
"ModernizeBot",
"seo"
],
[
"MontasticMonitor",
"monitoring"
],
[
"NanoInteractive",
"advertising"
],
[
"NestDaddybot",
"search-engine"
],
[
"Netcraft SSL Server Survey",
"scanner"
],
[
"Netcraft Web Server Survey",
"advertising"
],
[
"NetSeer crawler",
"advertising"
],
[
"Netumo|netumo",
"monitoring"
],
[
"NewRelicSynthetics",
"monitoring"
],
[
"NewsRoom\\.BI",
"advertising"
],
[
"Nitro-",
"seo"
],
[
"NitroBot",
"seo"
],
[
"Noibu",
"monitoring"
],
[
"NostoCrawlerBot",
"advertising"
],
[
"OneTrust",
"monitoring"
],
[
"opencode-smartfetch",
"browser-automation"
],
[
";Owler",
"advertising"
],
[
"ParselySharesBot",
"monitoring"
],
[
"PhindBot",
"ai-crawler"
],
[
"PodchaserParser",
"feed-reader"
],
[
"Podimo",
"feed-reader"
],
[
"Poggio-Citations",
"ai-crawler"
],
[
"productsup\\.io\\/crawler",
"advertising"
],
[
"qcbot",
"feed-reader"
],
[
"Qualys",
"monitoring"
],
[
"Quora-Bot",
"social-preview"
],
[
"Qwantbot",
"search-engine"
],
[
"Qwarrybot",
"advertising"
],
[
"RSiteAuditor",
"seo"
],
[
"RSS\\.Social",
"feed-reader"
],
[
"Salesforce\\.com",
"http-library"
],
[
"Scope3",
"advertising"
],
[
"scraping@nytimes\\.com",
"advertising"
],
[
"Scrubby",
"search-engine"
],
[
"Scrunchbot",
"monitoring"
],
[
"seo4ajax\\.com",
"seo"
],
[
"SequelWP",
"monitoring"
],
[
"ServerDensity",
"monitoring"
],
[
"ShapBot",
"ai-crawler"
],
[
"ShortPixel",
"feed-reader"
],
[
"Silktide",
"monitoring"
],
[
"SiteLock",
"monitoring"
],
[
"SmarshBot",
"archiver"
],
[
"SMTnetPMBot",
"search-engine"
],
[
"Software-Security-Research",
"scanner,academic"
],
[
"SottopopNone",
"feed-reader"
],
[
"Spider[\\s\\S]*spider\\.com",
"ai-crawler"
],
[
"Splunk",
"monitoring"
],
[
"StatusNestBacklinkSpider",
"seo"
],
[
"stepstoneCrawlBot",
"search-engine"
],
[
"TavilyBot",
"ai-crawler"
],
[
"ThousandEyes",
"monitoring"
],
[
"Trae\\/",
"browser-automation"
],
[
"TwinAgent",
"browser-automation"
],
[
"uipbot",
"advertising"
],
[
"um-FC",
"advertising"
],
[
"um-IC",
"advertising"
],
[
"UptimeStatistics",
"monitoring"
],
[
"Verispider",
"scanner"
],
[
"visionheight\\.com\\/scan",
"scanner"
],
[
"Watchbot monitoring robot",
"monitoring"
],
[
"Watchful",
"monitoring"
],
[
"weborama-fetcher",
"advertising"
],
[
"webspidermount",
"advertising"
],
[
"WepchSearchEngine",
"search-engine"
],
[
"wknd-bot",
"advertising"
],
[
"WPMU DEV Hub",
"monitoring"
],
[
"WTotem",
"scanner"
],
[
"XoviOnpageCrawler",
"seo"
],
[
"yelpspider",
"feed-reader"
],
[
"ZanistaBot",
"ai-crawler"
],
[
"ZoomInfo-",
"search-engine"
],
[
"7Siters",
"search-engine"
],
[
"Accessible Web Bot",
"monitoring"
],
[
"AtVowBot",
"monitoring"
],
[
"Bibliotheque Nacional de France Crawler",
"archiver"
],
[
"Bling ERP",
"monitoring"
],
[
"CDSCbot",
"academic"
],
[
"Critical CSS Bot",
"monitoring"
],
[
"CybaaBot",
"advertising"
],
[
"CyberFindCrawler",
"search-engine"
],
[
"Dark Visitor",
"monitoring"
],
[
"Determ",
"monitoring"
],
[
"DNSScanner",
"scanner"
],
[
"Drupalbot",
"monitoring"
],
[
"eMoney Advisor",
"monitoring"
],
[
"everyfeed-spider",
"feed-reader"
],
[
"ExteContextCrawl",
"ai-crawler"
],
[
"FediDB",
"monitoring"
],
[
"FediIndex",
"search-engine"
],
[
"FediList Agent",
"monitoring"
],
[
"Fedineko",
"academic"
],
[
"FedReporter Bot for FFIEC",
"monitoring"
],
[
"Feedsearch Bot",
"feed-reader"
],
[
"Feedsearch-Crawler",
"feed-reader"
],
[
"fiperbot",
"search-engine"
],
[
"FleebsBot",
"search-engine"
],
[
"Fluid",
"browser-automation"
],
[
"Flyriverbot",
"ai-crawler"
],
[
"Freshbot",
"monitoring"
],
[
"Gaisbot",
"academic"
],
[
"GenomeCrawlerd",
"monitoring"
],
[
"HaloBot",
"monitoring"
],
[
"IRLbot",
"academic"
],
[
"kaikki\\.org-digital-archive",
"archiver"
],
[
"kb\\.dk_bot",
"archiver"
],
[
"Library Of Congress Web Archiving",
"archiver"
],
[
"MagnetmeBot",
"monitoring"
],
[
"MatchorySearch",
"search-engine"
],
[
"Minoru's Fediverse Crawler",
"monitoring"
],
[
"MirrorWebCrawler",
"archiver"
],
[
"mithril-crawler",
"academic"
],
[
"ModatScanner",
"monitoring"
],
[
"NapBot",
"search-engine"
],
[
"New York Times Newsgathering",
"monitoring"
],
[
"NLUX_IAHarvester",
"archiver"
],
[
"NoahBot",
"monitoring"
],
[
"PlagAwareBot",
"monitoring"
],
[
"Rakuten Image extraction bot",
"monitoring"
],
[
"ResearchBot",
"academic"
],
[
"rss-is-dead\\.lol web bot",
"feed-reader"
],
[
"seoLyt",
"seo"
],
[
"SirdataBot",
"advertising"
],
[
"SitesOverPagesBot",
"seo"
],
[
"SleepBot",
"monitoring"
],
[
"Sosospider",
"search-engine"
],
[
"Termly",
"monitoring"
],
[
"TLS tester",
"scanner"
],
[
"trafilatura",
"http-library"
],
[
"UrlBeeBot",
"scanner"
],
[
"videootv Bot",
"monitoring"
],
[
"vmcrawl",
"monitoring"
],
[
"WadooBot",
"search-engine"
],
[
"Website-info\\.net-Robot",
"seo"
],
[
"WebZIP",
"archiver"
],
[
"WikiDo",
"search-engine"
],
[
"WOVN Crawler",
"monitoring"
],
[
"YoudaoBot",
"search-engine"
],
[
"ZyBorg",
"search-engine"
],
[
"Aranet-SearchBot",
"ai-crawler"
],
[
"crawl4ai",
"ai-crawler"
],
[
"DeepSeekBot",
"ai-crawler"
],
[
"iaskspider",
"ai-crawler"
],
[
"KunatoCrawler",
"ai-crawler"
],
[
"TerraCotta",
"ai-crawler"
],
[
"ABEvalBot",
"monitoring"
],
[
"blekkobot",
"search-engine"
],
[
"br-crawler",
"search-engine"
],
[
"BuddyBot",
"monitoring"
],
[
"CapterraBot",
"monitoring"
],
[
"carbon-umbrella-bot",
"monitoring"
],
[
"caveman-hunter",
"monitoring"
],
[
"Centro Ads\\.txt Crawler",
"advertising"
],
[
"WISEbot",
"monitoring"
],
[
"CodaBot",
"social-preview"
],
[
"Corporama matcher",
"monitoring"
],
[
"CyotekWebCopy",
"archiver"
],
[
"Datadog Agent",
"monitoring"
],
[
"Dazzle BlueSky Bot",
"social-preview"
],
[
"DominicBot",
"monitoring"
],
[
"Dow Jones Searchbot",
"monitoring"
],
[
"Download Ninja",
"http-library"
],
[
"EmailWolf",
"scanner"
],
[
"fedistatsCrawler",
"monitoring"
],
[
"GoParserBot",
"http-library"
],
[
"gsa-crawler",
"search-engine"
],
[
"HanaleiBot",
"monitoring"
],
[
"NicheIndex",
"feed-reader"
],
[
"HeadOnlyScraper",
"monitoring"
],
[
"HenkBot",
"ai-crawler"
],
[
"Impact\\.com Agent",
"advertising"
],
[
"Keydrop\\.io",
"scanner"
],
[
"larbin",
"http-library"
],
[
"SENTINEL-LinkCheck",
"monitoring"
],
[
"linko",
"monitoring"
],
[
"LinkpadBot",
"monitoring"
],
[
"lwp-trivial",
"http-library"
],
[
"Magus Bot",
"monitoring"
],
[
"NaverBot",
"search-engine"
],
[
"loopimprovements\\.com",
"monitoring"
],
[
"OpenTheBoxBot",
"search-engine"
],
[
"OWLer-W",
"monitoring"
],
[
"peer39_crawler",
"advertising"
],
[
"Pixalate\\.com",
"advertising"
],
[
"Poduptime",
"monitoring"
],
[
"Pomothy-Bot",
"monitoring"
],
[
"PulsePoint-Crawler",
"advertising"
],
[
"rawweb-bot",
"http-library"
],
[
"semantic-visions",
"ai-crawler"
],
[
"Sindup",
"monitoring"
],
[
"SiteSucker",
"archiver"
],
[
"SpringserveBot",
"advertising"
],
[
"SQWatcher",
"monitoring"
],
[
"Supabase Paired Crawler",
"social-preview"
],
[
"sv-watchagent",
"monitoring"
],
[
"Swiftbot",
"search-engine"
],
[
"SynthesiBot",
"monitoring"
],
[
"TaraGroup Intelligent Bot",
"ai-crawler"
],
[
"Thinkbot",
"ai-crawler"
],
[
"TSMbot",
"monitoring"
],
[
"TSM-turingos",
"ai-crawler"
],
[
"UGAResearchAgent",
"academic"
],
[
"UrlSuMa\\.de crawler",
"monitoring"
],
[
"WanscannerBot",
"scanner"
],
[
"WebCapture",
"archiver"
],
[
"WebCopier",
"archiver"
],
[
"cognitiveseo\\.com",
"seo"
],
[
"Xing Bot",
"social-preview"
],
[
"XML Sitemaps Generator",
"seo"
],
[
"YandoriRSSBot",
"feed-reader"
],
[
"Zealbot",
"search-engine"
],
[
"008\\/",
"seo"
],
[
"monitoring360bot\\/",
"monitoring"
],
[
"AdagioBot",
"seo"
],
[
"adbeat\\.com",
"advertising"
],
[
"AdminLabs",
"seo"
],
[
"advanced_crawler",
"seo"
],
[
"Adventurer",
"seo"
],
[
"AGAKIDSBOT",
"seo"
],
[
"AgencyAnalyticsBot",
"seo"
],
[
"AI2Bot",
"academic,ai-crawler"
],
[
"AkismetBot",
"scanner"
],
[
"alexa site audit",
"seo"
],
[
"Algolia Crawler",
"search-engine"
],
[
"alienfarm",
"seo"
],
[
"allOrigins",
"seo"
],
[
"AmazonAdBot",
"advertising"
],
[
"KendraBot",
"search-engine,ai-crawler"
],
[
"AppSiteAssociation",
"seo"
],
[
"Aragog\\/",
"seo"
],
[
"Aranea",
"academic"
],
[
"ArchiveBox",
"archiver"
],
[
"ArquivoBot",
"archiver"
],
[
"Arquivo-web-crawler",
"archiver"
],
[
"ArtemisBot",
"scanner"
],
[
"Asana\\/",
"seo"
],
[
"AudistoBot",
"seo"
],
[
"Autoconfig Test from USTC",
"academic"
],
[
"tracking-quality-spider",
"advertising"
],
[
"Bad Neighborhood Header Detector",
"scanner"
],
[
"BaiduAdsBot",
"advertising"
],
[
"BDBot\\/",
"seo"
],
[
"BeeperBot",
"seo"
],
[
"BetterUptimeBot",
"monitoring"
],
[
"BnFBot",
"archiver"
],
[
"BigUpDataBot",
"seo"
],
[
"BinaryCanary",
"monitoring"
],
[
"Bitbucket-Webhooks",
"seo"
],
[
"bl\\.uk_ldfc_bot",
"archiver"
],
[
"BlackDuck-FD",
"scanner"
],
[
"Blogtrottr",
"feed-reader"
],
[
"BlueskyPreviewBot",
"social-preview"
],
[
"BoardGamePricesBot",
"seo"
],
[
"BotPoke",
"seo"
],
[
"BDFetch",
"seo"
],
[
"Brandwatch",
"seo"
],
[
"BraveBot",
"search-engine"
],
[
"brokenlinkcheck\\.com",
"seo"
],
[
"BW\\/",
"seo"
],
[
"Bushbaby",
"seo"
],
[
"Butterfly",
"seo"
],
[
"rss-parser",
"feed-reader"
],
[
"CaliberBot",
"seo"
],
[
"CapitalOneShopping",
"seo"
],
[
"Catchpoint",
"monitoring"
],
[
"centuryb\\.o\\.t9",
"seo"
],
[
"CERT PL",
"scanner"
],
[
"certytags",
"seo"
],
[
"ChargeBeeBot",
"seo"
],
[
"Charlotte",
"seo"
],
[
"ChatGLM-Spider",
"ai-crawler"
],
[
"Chatwork LinkPreview",
"social-preview"
],
[
"CheckHost",
"monitoring"
],
[
"Goodzer",
"seo"
],
[
"Chrome Privacy Preserving Prefetch Proxy",
"seo"
],
[
"CirrusExplorer",
"seo"
],
[
"CLASSLA-web",
"academic"
],
[
"Clearscopebot",
"seo"
],
[
"WorldBot",
"seo"
],
[
"Cloudflare-Validator",
"monitoring"
],
[
"cloudflare-csup",
"monitoring"
],
[
"Cloudflare-Custom-Error-Page-Crawler",
"monitoring"
],
[
"Cloudflare-Radar-Scanner",
"scanner"
],
[
"Cloudflare-SpeedTest",
"monitoring"
],
[
"Cloudflare-Stream-Hook",
"seo"
],
[
"cognitiveSEO Bot",
"seo"
],
[
"cohere-training-data-crawler",
"ai-crawler"
],
[
"CommaFeed",
"feed-reader"
],
[
"researchscan\\.comsys\\.rwth-aachen\\.de",
"academic"
],
[
"contentkingapp",
"seo"
],
[
"CookieHub Bot",
"scanner"
],
[
"Cotoyogi",
"seo"
],
[
"Coveobot",
"search-engine"
],
[
"Crawlson",
"seo"
],
[
"RepoLookoutBot",
"seo"
],
[
"Criticalcss\\.com",
"seo"
],
[
"cron-job\\.org",
"monitoring"
],
[
"DnBCrawler",
"seo"
],
[
"DMBrowser",
"monitoring"
],
[
"DomCopBot",
"seo"
],
[
"downnotifier\\.com",
"monitoring"
],
[
"DowntimeDetector\\/",
"monitoring"
],
[
"Dlc\\/",
"seo"
],
[
"Dratabot",
"scanner"
],
[
"EasyBib AutoCite",
"academic"
],
[
"easybill-ImportManager",
"seo"
],
[
"EasyCron\\/",
"monitoring"
],
[
"easyDNS Monitoring",
"monitoring"
],
[
"EchoboxBot\\/",
"seo"
],
[
"Cronless",
"monitoring"
],
[
"crusty\\/",
"seo"
],
[
"csirt\\.cz",
"scanner"
],
[
"CXK_Bot",
"seo"
],
[
"daumoa",
"search-engine"
],
[
"DaspeedBot",
"seo"
],
[
"Dead Link Checker",
"seo"
],
[
"Deskyobot",
"seo"
],
[
"Detectify",
"scanner"
],
[
"Devin",
"browser-automation,ai-crawler"
],
[
"DF Bot",
"seo"
],
[
"DingTalkBot-LinkService",
"social-preview"
],
[
"Discourse Forum Onebox",
"social-preview"
],
[
"Dmbot",
"seo"
],
[
"SustainabilityCrawler",
"seo"
],
[
"edansbot",
"seo"
],
[
"EdgeWatch",
"monitoring"
],
[
"Do Not Track Verifier",
"scanner"
],
[
"elmahio-uptimebot",
"monitoring"
],
[
"eMoneyBot",
"seo"
],
[
"EpivozCrawler",
"seo"
],
[
"eRepublik\\.tools",
"seo"
],
[
"EvoUptimeBot",
"monitoring"
],
[
"ExodusMovement",
"seo"
],
[
"Ezgif",
"seo"
],
[
"factset_spyderbot",
"seo"
],
[
"FastmailUA",
"social-preview"
],
[
"FDL Stats Bot",
"seo"
],
[
"Fedicabot",
"seo"
],
[
"FedReporterDataBot",
"seo"
],
[
"Feed Image Audit",
"scanner"
],
[
"FeedBurner",
"feed-reader"
],
[
"feeder\\.co",
"feed-reader"
],
[
"Feedpresso Content Index Bot",
"feed-reader"
],
[
"Feedwind",
"feed-reader"
],
[
"fidget-spinner-bot",
"seo"
],
[
"FirmoGraph",
"seo"
],
[
"FlipboardRSS",
"feed-reader"
],
[
"Foregenix",
"scanner"
],
[
"Freespoke\\/",
"search-engine"
],
[
"Friendly testing bot",
"seo"
],
[
"friendly-spider",
"seo"
],
[
"FriendlyCrawler\\/",
"seo"
],
[
"FullStoryBot\\/",
"seo"
],
[
"Funnelback",
"search-engine"
],
[
"FuseonBot\\/",
"seo"
],
[
"Gabanzabot\\/",
"search-engine"
],
[
"gdnplus\\.com",
"seo"
],
[
"getthit\\.com",
"seo"
],
[
"GG PeekBot",
"social-preview"
],
[
"Ghost Inspector",
"browser-automation"
],
[
"github-camo",
"social-preview"
],
[
"GlobalWebSearch",
"search-engine"
],
[
"Golfe\\/",
"search-engine"
],
[
"Google-Apps-Script",
"seo"
],
[
"GoogleStackdriverMonitoring",
"monitoring"
],
[
"GoogleAssociationService\\/",
"seo"
],
[
"GoogleImageProxy",
"social-preview"
],
[
"GoogleProducer",
"search-engine"
],
[
"Googlebot-IA\\/",
"search-engine,academic"
],
[
"Google-Trust-Services\\/",
"scanner"
],
[
"Google-Area120",
"seo"
],
[
"Google-CloudVertexBot",
"search-engine,ai-crawler"
],
[
"GoogleAssociationService$",
"seo"
],
[
"GoogleDocs",
"seo"
],
[
"GoPay",
"seo"
],
[
"GotSiteMonitor\\.com",
"monitoring"
],
[
"synthetic-monitoring-agent\\/",
"monitoring"
],
[
"Grammarly\\/",
"seo"
],
[
"gregcrawler",
"seo"
],
[
"GroovinaAdsbot\\/",
"advertising"
],
[
"Grover\\/",
"seo"
],
[
"GTmetrix",
"monitoring"
],
[
"GuestpostsBot\\/",
"seo"
],
[
"Gulper Web Bot",
"academic"
],
[
"Verity\\/",
"advertising"
],
[
"HappyWing",
"seo"
],
[
"harsilbot\\/",
"seo"
],
[
"HawaiiBot",
"seo"
],
[
"hCardValidator",
"seo"
],
[
"Hello World",
"seo"
],
[
"HelloworkJobPostingBot\\/",
"seo"
],
[
"HetrixTools",
"monitoring"
],
[
"HIFIBot\\/",
"seo"
],
[
"Hlidam\\.to robot",
"seo"
],
[
"Honeybadger Uptime Check",
"monitoring"
],
[
"HostTracker\\/",
"monitoring"
],
[
"Hotjar",
"seo"
],
[
"hstspreload-bot",
"scanner"
],
[
"Huckabot\\/",
"seo"
],
[
"Hype Machine\\/",
"seo"
],
[
"Web Screen Service By hyperhost",
"seo"
],
[
"AdsBot-IAB",
"advertising"
],
[
"iAskBot\\/",
"search-engine,ai-crawler"
],
[
"IBM Crawler",
"seo"
],
[
"IFTTT\\/",
"seo"
],
[
"ImageFetcher\\/",
"seo"
],
[
"ImageMind",
"ai-crawler"
],
[
"img2dataset",
"ai-crawler"
],
[
"imgproxy\\/",
"seo"
],
[
"impendoom-bot\\/",
"seo"
],
[
"IndeedJobBot",
"search-engine"
],
[
"Innguma\\/",
"seo"
],
[
"Instapaper\\/",
"seo"
],
[
"Integromat\\/",
"seo"
],
[
"intelx\\.io_bot",
"search-engine"
],
[
"internetVista monitor",
"monitoring"
],
[
"Irokez\\.cz monitoring",
"monitoring"
],
[
"IsDownBot\\/",
"monitoring"
],
[
"ISSCyberRiskCrawler\\/",
"scanner"
],
[
"iubenda-radar\\/",
"scanner"
],
[
"UptimeBot\\/",
"monitoring"
],
[
"jetmon\\/",
"monitoring"
],
[
"jobswithgptcom-bot",
"search-engine"
],
[
"Jumio",
"seo"
],
[
"Kagibot\\/",
"search-engine"
],
[
"KangarooBot\\/",
"seo"
],
[
"KargoBot-Artemis",
"advertising"
],
[
"kazbtbot\\/",
"seo"
],
[
"keycdn-tools\\/",
"monitoring"
],
[
"keys-so-bot",
"seo"
],
[
"kinsta-bot",
"seo"
],
[
"Klaviyo\\/",
"seo"
],
[
"Kukei\\.eu-Bot\\/",
"seo"
],
[
"LAC_IAHarvester",
"archiver"
],
[
"LastModBot\\/",
"seo"
],
[
"LegalMonster",
"scanner"
],
[
"Let's Encrypt",
"scanner"
],
[
"Level9SearchBot\\/",
"search-engine"
],
[
"loc\\.gov\\/programs\\/web-archiving",
"archiver"
],
[
"LinerBot\\/",
"ai-crawler"
],
[
"LinkTiger",
"seo"
],
[
"LinkAce\\/",
"archiver"
],
[
"LinksIndexerBot\\/",
"seo"
],
[
"LinkWalker\\/",
"seo"
],
[
"LogicMonitor",
"monitoring"
],
[
"LoomlyBot",
"social-preview"
],
[
"Macrobondbot",
"seo"
],
[
"MADBbot\\/",
"seo"
],
[
"Magellan",
"seo"
],
[
"magicsearchdev\\/",
"seo"
],
[
"Magnet\\.me-web\\/",
"seo"
],
[
"MainWP\\/",
"seo"
],
[
"Make\\/",
"seo"
],
[
"ManageWP",
"seo"
],
[
"MarketGoo\\/",
"seo"
],
[
"MarketingMiner",
"seo"
],
[
"dbot\\)",
"social-preview"
],
[
"Mattermost-Bot\\/",
"social-preview"
],
[
"Mavifinds",
"seo"
],
[
"MB-LinkChecker",
"seo"
],
[
"MedialogiaBot",
"seo"
],
[
"MediaMonitoringBot\\/",
"seo"
],
[
"MediavineMetadataParser\\/",
"advertising"
],
[
"Pywikibot\\/",
"seo"
],
[
"CentComBot\\/",
"seo"
],
[
"MergadoBot",
"seo"
],
[
"Meta-ExternalHit\\/",
"social-preview"
],
[
"Metorik",
"seo"
],
[
"MgidBot",
"advertising"
],
[
"Miniature\\.io\\/",
"archiver"
],
[
"mirrorweb\\.com",
"archiver"
],
[
"MissinglettrBot\\/",
"seo"
],
[
"crawler_eb_germany",
"seo"
],
[
"ModularConnector\\/",
"seo"
],
[
"Mollie HTTP client",
"seo"
],
[
"Monibot",
"seo"
],
[
"monitis -",
"monitoring"
],
[
"MonitoRSS\\/",
"feed-reader"
],
[
"MonSpark\\/",
"monitoring"
],
[
"montastic-monitor",
"monitoring"
],
[
"MonTools\\.com",
"seo"
],
[
"MotoMinerBot\\/",
"seo"
],
[
"MRGbot\\/",
"scanner"
],
[
"MxToolbox",
"monitoring"
],
[
"my-tiny-bot",
"seo"
],
[
"MyBot\\/",
"seo"
],
[
"nbertaupete95",
"seo"
],
[
"NetAPI",
"seo"
],
[
"NetpeakCheckerBot\\/",
"seo"
],
[
"NetShelter ContentScan",
"advertising"
],
[
"NETVIGIE",
"monitoring"
],
[
"NewRelicbot\\/",
"monitoring"
],
[
"nyt_scraping",
"seo"
],
[
"NewsNow\\/",
"seo"
],
[
"NLNZ_IAHarvester",
"archiver"
],
[
"NodePing",
"monitoring"
],
[
"nomore404\\.com robot",
"seo"
],
[
"noorobot",
"seo"
],
[
"Nooshub\\/",
"seo"
],
[
"Notabot",
"seo"
],
[
"Novaact\\/",
"seo"
],
[
"Novellum",
"ai-crawler"
],
[
"NsToolsBot\\/",
"scanner"
],
[
"nvdorz",
"seo"
],
[
"Odin;",
"seo"
],
[
"Offline Explorer",
"archiver"
],
[
"OhDear\\/",
"monitoring"
],
[
"Omnisend\\/",
"seo"
],
[
"Online Domain Tools",
"seo"
],
[
"WebCEO Online\\/",
"seo"
],
[
"OnlineOrNot\\.com_bot",
"monitoring"
],
[
"OpenGraph\\.io\\/",
"social-preview"
],
[
"OpenRSS",
"feed-reader"
],
[
"OpenVAS",
"scanner"
],
[
"Owler \\(ows\\.eu",
"search-engine"
],
[
"Operator\\/",
"seo"
],
[
"Orbbot\\/",
"seo"
],
[
"zebra-v2-bot",
"seo"
],
[
"Orlo-LinkPreview\\/",
"social-preview"
],
[
"Cozi-iCalendar-FeedReader",
"feed-reader"
],
[
"OutsellURLValidator",
"seo"
],
[
"Overcast\\/",
"feed-reader"
],
[
"PRTGCloudBot\\/",
"monitoring"
],
[
"Pagespeed\\/",
"monitoring"
],
[
"PanguBot",
"search-engine"
],
[
"Panopta",
"monitoring"
],
[
"Paqlebot\\/",
"seo"
],
[
"parse\\.ly scraper\\/",
"seo"
],
[
"PayPal\\/",
"seo"
],
[
"PDF24 URL To PDF",
"seo"
],
[
"PingAdmin\\.Ru\\/",
"monitoring"
],
[
"pingping\\.io\\/",
"monitoring"
],
[
"PlayStore-Google",
"search-engine"
],
[
"Plesk screenshot bot",
"seo"
],
[
"PocketCasts\\/",
"feed-reader"
],
[
"Potions\\/",
"seo"
],
[
"PressEngineBot",
"seo"
],
[
"PricedroneShoppingBot\\/",
"seo"
],
[
"PriEcoBot\\/",
"seo"
],
[
"PrintFriendly\\.com",
"seo"
],
[
"Pro-Sitemaps\\/",
"seo"
],
[
"ProbelySPDR\\/",
"scanner"
],
[
"ProjectShield-UrlCheck",
"scanner"
],
[
"Blackbox Exporter\\/",
"monitoring"
],
[
"Protopage\\/",
"feed-reader"
],
[
"PS_Daily\\/",
"seo"
],
[
"pulsetic\\.com",
"monitoring"
],
[
"PWABuilderHttpAgent",
"seo"
],
[
"QualifiedBot\\/",
"seo"
],
[
"Quantcastbot\\/",
"advertising"
],
[
"Rackspace Monitoring\\/",
"monitoring"
],
[
"rakutenusabot-image\\/",
"seo"
],
[
"top100\\.rambler\\.ru crawler",
"search-engine"
],
[
"RankurBot\\/",
"seo"
],
[
"RavenCrawler\\/",
"seo"
],
[
"Readable\\/",
"seo"
],
[
"Recurly Webhooks\\/",
"seo"
],
[
"RED\\/",
"monitoring"
],
[
"Reelevant\\/",
"seo"
],
[
"remove\\.bg\\/",
"seo"
],
[
"Retool\\/",
"seo"
],
[
"RetroListeCOM\\/",
"seo"
],
[
"RevvimGort\\/",
"seo"
],
[
"reward-gateway",
"seo"
],
[
"Riddler \\(http:\\/\\/riddler\\.io",
"scanner"
],
[
"RobotsChecker\\/",
"seo"
],
[
"RSSAPI\\/",
"feed-reader"
],
[
"rss2tg",
"feed-reader"
],
[
"RssReaderBot",
"feed-reader"
],
[
"s4a-probe-bot\\/",
"seo"
],
[
"SFDC-Callout\\/",
"seo"
],
[
"page-preview-tool",
"seo"
],
[
"SandobaCrawler\\/",
"seo"
],
[
"Sansec Security Monitor\\/",
"scanner"
],
[
"GIFTEDVISITOR SCAN",
"seo"
],
[
"Schema-Markup-Validator",
"seo"
],
[
"Scoop\\.it\\/",
"seo"
],
[
"ScourRSSBot\\/",
"feed-reader"
],
[
"ScrapeheroBot\\/",
"seo"
],
[
"screeenly-bot",
"seo"
],
[
"SEBot-WA",
"seo"
],
[
"Searcherweb",
"search-engine"
],
[
"Searcherxweb",
"search-engine"
],
[
"SearchExpress",
"search-engine"
],
[
"SecurityHeaders",
"scanner"
],
[
"semaltbot\\/",
"seo"
],
[
"SendGrid Event API",
"seo"
],
[
"SentryUptimeBot\\/",
"monitoring"
],
[
"seo-audit-check-bot\\/",
"seo"
],
[
"s4a\\/",
"seo"
],
[
"ClarityBot\\/",
"seo"
],
[
"SeoSiteCheckup",
"seo"
],
[
"SeoulBot",
"seo"
],
[
"SERPtimizerBot",
"seo"
],
[
"Server Density Service Monitoring",
"monitoring"
],
[
"ServerHunterSpider\\/",
"seo"
],
[
"SeznamHomepageCrawler\\/",
"search-engine"
],
[
"Shopify-Captain-Hook",
"seo"
],
[
"Shortwave Image Fetcher",
"social-preview"
],
[
"linkReader\\/",
"ai-crawler"
],
[
"Sidetrade indexer bot",
"seo"
],
[
"^Silk\\/",
"seo"
],
[
"SinceraSyntheticUser\\/",
"seo"
],
[
"Optimizer\\)",
"seo"
],
[
"Site24x7",
"monitoring"
],
[
"SiteAuditBot\\/",
"seo"
],
[
"SiteCheck-sitecrawl",
"seo"
],
[
"SiteScoreBot",
"seo"
],
[
"SiteSearch360\\/",
"search-engine"
],
[
"SiteUptime\\.com",
"monitoring"
],
[
"Konturbot\\/",
"seo"
],
[
"SkroutzBot",
"seo"
],
[
"SkyworkSpider",
"seo"
],
[
"SlickBot\\/",
"seo"
],
[
"SmartologyBot\\/",
"advertising"
],
[
"SnapURLPreview\\/",
"social-preview"
],
[
"SnapchatAds\\/",
"advertising"
],
[
"Snipcart\\/",
"seo"
],
[
"solarwinds\\/",
"monitoring"
],
[
"Sora POS\\/",
"seo"
],
[
"SparkShipping",
"seo"
],
[
"SparkPost",
"seo"
],
[
"Spawning-AI",
"ai-crawler"
],
[
"IDG\\/EU",
"seo"
],
[
"Specificfeeds",
"feed-reader"
],
[
"Spectate\\/",
"monitoring"
],
[
"SpiderLing",
"academic"
],
[
"splash Version\\/",
"browser-automation"
],
[
"Rigor\\)",
"monitoring"
],
[
"TwinWaveScanner",
"scanner"
],
[
"SSL Labs \\(https:\\/\\/www\\.ssllabs\\.com",
"scanner"
],
[
"SSSSBot\\/",
"seo"
],
[
"Stape\\/",
"seo"
],
[
"StartpagePrivateImageProxy\\/",
"search-engine"
],
[
"Statabot\\/",
"seo"
],
[
"StatistikAustria\\/",
"seo"
],
[
"StatsDroneBot",
"seo"
],
[
"Stripe\\/",
"seo"
],
[
"Sucuri",
"scanner"
],
[
"Svix-Webhooks\\/",
"seo"
],
[
"SwifteqLinkChecker",
"seo"
],
[
"Swisscows",
"search-engine"
],
[
"Datadog Synthetic",
"monitoring"
],
[
"TactiScout\\/",
"seo"
],
[
"tchelebi\\/",
"seo"
],
[
"bitdiscovery",
"scanner"
],
[
"Test Certificate Info",
"scanner"
],
[
"Testcrawler",
"seo"
],
[
"test-bot",
"seo"
],
[
"TestLocally\\/",
"browser-automation"
],
[
"TestURI",
"seo"
],
[
"TextRazor",
"seo"
],
[
"The Knowledge AI",
"ai-crawler"
],
[
"TheInternetSearchx",
"search-engine"
],
[
"thesis-research-bot",
"academic"
],
[
"Trellis-Services",
"seo"
],
[
"trentwil\\.es",
"seo"
],
[
"Trustly\\/",
"seo"
],
[
"TTD-Content",
"advertising"
],
[
"Tweakers",
"seo"
],
[
"TwilioProxy\\/",
"social-preview"
],
[
"UASlinkChecker\\/",
"seo"
],
[
"hgfAlphaXCrawl\\/",
"academic"
],
[
"Unshorten\\.It\\!",
"scanner"
],
[
"updown\\.io",
"monitoring"
],
[
"Uptime\\/",
"monitoring"
],
[
"uptimedoctor",
"monitoring"
],
[
"Uptimia",
"monitoring"
],
[
"uptrends",
"monitoring"
],
[
"Urlcheckr\\/",
"seo"
],
[
"URLSuMaBot",
"seo"
],
[
"useeBookChecker\\/",
"seo"
],
[
"Vagabondo\\/",
"seo"
],
[
"VaultPress",
"scanner"
],
[
"videootvBot",
"seo"
],
[
"VsuSearchSpider\\/",
"search-engine"
],
[
"vu-server-health-scanner\\/",
"academic"
],
[
"WARDBot\\/",
"seo"
],
[
"WebsiteOps",
"monitoring"
],
[
"WatchMouse",
"monitoring"
],
[
"Web Measure\\/",
"academic"
],
[
"Webflow",
"seo"
],
[
"webgains-bot",
"advertising"
],
[
"webprosbot\\/",
"seo"
],
[
"websitepulse",
"monitoring"
],
[
"WebSniffer\\/",
"seo"
],
[
"WSM\\/",
"seo"
],
[
"WebwikiBot\\/",
"seo"
],
[
"WEDOS OnLine",
"monitoring"
],
[
"WhatsMyIP\\.org",
"seo"
],
[
"WhatWeb\\/",
"scanner"
],
[
"Wheregoes\\.com",
"seo"
],
[
"wheresitup\\.com\\/",
"monitoring"
],
[
"Citoid",
"academic"
],
[
"WireReaderBot\\/",
"seo"
],
[
"ZoteroTranslationServer\\/WMF",
"academic"
],
[
"wmtips\\.com\\/",
"seo"
],
[
"WordCountBot\\/",
"seo"
],
[
"Wordup-1",
"seo"
],
[
"workona-favicon-service\\/",
"seo"
],
[
"Indy Library",
"seo"
],
[
"WJHRO\\/",
"seo"
],
[
"WormlyBot",
"monitoring"
],
[
"WovnCrawler\\/",
"seo"
],
[
"wowLink Crawler\\/",
"seo"
],
[
"WP Time Capsule",
"seo"
],
[
"WPUmbrella",
"monitoring"
],
[
"wpbot\\/",
"seo"
],
[
"WPMU DEV Broken Link Checker",
"seo"
],
[
"WPMUDEV Uptime Monitor",
"monitoring"
],
[
"WPSec\\/",
"scanner"
],
[
"WRTNBot",
"seo"
],
[
"abuse\\.xmco\\.fr",
"scanner"
],
[
"XY-Archive-Compliance",
"archiver"
],
[
"Yahoo Ad monitoring",
"advertising"
],
[
"YahooMailProxy",
"social-preview"
],
[
"YahooCacheSystem",
"seo"
],
[
"YLT Chrome",
"monitoring"
],
[
"YokoyGroupAG\\/",
"seo"
],
[
"Yuuperbot",
"seo"
],
[
"Zapier",
"seo"
],
[
"Zendesk Webhook",
"seo"
],
[
"Zombiebot\\/",
"seo"
],
[
"zzhbot",
"seo"
],
[
"Penthouse Critical Path CSS Generator",
"seo"
],
[
"Google-AdWords-Express",
"advertising"
],
[
"Notion\\/",
"social-preview"
],
[
"SSL Labs$",
"scanner"
],
[
"Skroutz ImageBot",
"seo"
],
[
"Tumblr\\/",
"social-preview"
],
[
"upday\\/",
"feed-reader"
],
[
"watchTowr",
"scanner"
],
[
"PRTG Network Monitor",
"monitoring"
],
[
"GeedoShopProductFinder",
"search-engine"
],
[
"WatchForBot",
"monitoring"
],
[
"ListGemBot",
"social-preview"
],
[
"IsonSearchBot",
"search-engine"
],
[
"9bot\\/",
"social-preview"
],
[
"IsonAI-User",
"ai-crawler"
],
[
"InfoSifter",
"monitoring"
],
[
"AIWebIndex",
"ai-crawler"
]
],
"source": "https://github.com/monperrus/crawler-user-agents @ db662d2787f984ed4fe3043744a12eb067b89218"
},
"generated": "2026-10-31",
"ranges": [
{
"group": "bot",
"kind": "search-engine",
"name": "Googlebot",
"prefixes": [
"192.178.4.0/27",
"192.178.4.128/27",
"192.178.4.160/27",
"192.178.4.192/27",
"192.178.4.224/27",
"192.178.4.32/27",
"192.178.4.64/27",
"192.178.4.96/27",
"192.178.5.0/27",
"192.178.6.0/27",
"192.178.6.128/27",
"192.178.6.160/27",
"192.178.6.192/27",
"192.178.6.224/27",
"192.178.6.32/27",
"192.178.6.64/27",
"192.178.6.96/27",
"192.178.7.0/27",
"192.178.7.128/27",
"192.178.7.160/27",
"192.178.7.192/27",
"192.178.7.224/27",
"192.178.7.32/27",
"192.178.7.64/27",
"192.178.7.96/27",
"2001:4860:4801:10::/64",
"2001:4860:4801:11::/64",
"2001:4860:4801:12::/64",
"2001:4860:4801:13::/64",
"2001:4860:4801:14::/64",
"2001:4860:4801:15::/64",
"2001:4860:4801:16::/64",
"2001:4860:4801:17::/64",
"2001:4860:4801:18::/64",
"2001:4860:4801:19::/64",
"2001:4860:4801:1a::/64",
"2001:4860:4801:1b::/64",
"2001:4860:4801:1c::/64",
"2001:4860:4801:1d::/64",
"2001:4860:4801:1e::/64",
"2001:4860:4801:1f::/64",
"2001:4860:4801:20::/64",
"2001:4860:4801:21::/64",
"2001:4860:4801:22::/64",
"2001:4860:4801:23::/64",
"2001:4860:4801:24::/64",
"2001:4860:4801:25::/64",
"2001:4860:4801:26::/64",
"2001:4860:4801:27::/64",
"2001:4860:4801:28::/64",
"2001:4860:4801:29::/64",
"2001:4860:4801:2::/64",
"2001:4860:4801:2a::/64",
"2001:4860:4801:2b::/64",
"2001:4860:4801:2c::/64",
"2001:4860:4801:2d::/64",
"2001:4860:4801:2e::/64",
"2001:4860:4801:2f::/64",
"2001:4860:4801:30::/64",
"2001:4860:4801:31::/64",
"2001:4860:4801:32::/64",
"2001:4860:4801:33::/64",
"2001:4860:4801:34::/64",
"2001:4860:4801:35::/64",
"2001:4860:4801:36::/64",
"2001:4860:4801:37::/64",
"2001:4860:4801:38::/64",
"2001:4860:4801:39::/64",
"2001:4860:4801:3a::/64",
"2001:4860:4801:3b::/64",
"2001:4860:4801:3c::/64",
"2001:4860:4801:3d::/64",
"2001:4860:4801:3e::/64",
"2001:4860:4801:3f::/64",
"2001:4860:4801:40::/64",
"2001:4860:4801:41::/64",
"2001:4860:4801:42::/64",
"2001:4860:4801:43::/64",
"2001:4860:4801:44::/64",
"2001:4860:4801:45::/64",
"2001:4860:4801:46::/64",
"2001:4860:4801:47::/64",
"2001:4860:4801:48::/64",
"2001:4860:4801:49::/64",
"2001:4860:4801:4a::/64",
"2001:4860:4801:4b::/64",
"2001:4860:4801:4c::/64",
"2001:4860:4801:4d::/64",
"2001:4860:4801:4e::/64",
"2001:4860:4801:50::/64",
"2001:4860:4801:51::/64",
"2001:4860:4801:52::/64",
"2001:4860:4801:53::/64",
"2001:4860:4801:54::/64",
"2001:4860:4801:55::/64",
"2001:4860:4801:56::/64",
"2001:4860:4801:57::/64",
"2001:4860:4801:58::/64",
"2001:4860:4801:59::/64",
"2001:4860:4801:60::/64",
"2001:4860:4801:61::/64",
"2001:4860:4801:62::/64",
"2001:4860:4801:63::/64",
"2001:4860:4801:64::/64",
"2001:4860:4801:65::/64",
"2001:4860:4801:66::/64",
"2001:4860:4801:67::/64",
"2001:4860:4801:68::/64",
"2001:4860:4801:69::/64",
"2001:4860:4801:6a::/64",
"2001:4860:4801:6b::/64",
"2001:4860:4801:6c::/64",
"2001:4860:4801:6d::/64",
"2001:4860:4801:6e::/64",
"2001:4860:4801:6f::/64",
"2001:4860:4801:70::/64",
"2001:4860:4801:71::/64",
"2001:4860:4801:72::/64",
"2001:4860:4801:73::/64",
"2001:4860:4801:74::/64",
"2001:4860:4801:75::/64",
"2001:4860:4801:76::/64",
"2001:4860:4801:77::/64",
"2001:4860:4801:78::/64",
"2001:4860:4801:79::/64",
"2001:4860:4801:7a::/64",
"2001:4860:4801:7b::/64",
"2001:4860:4801:7c::/64",
"2001:4860:4801:7d::/64",
"2001:4860:4801:7e::/64",
"2001:4860:4801:7f::/64",
"2001:4860:4801:80::/64",
"2001:4860:4801:81::/64",
"2001:4860:4801:82::/64",
"2001:4860:4801:83::/64",
"2001:4860:4801:84::/64",
"2001:4860:4801:85::/64",
"2001:4860:4801:86::/64",
"2001:4860:4801:87::/64",
"2001:4860:4801:88::/64",
"2001:4860:4801:90::/64",
"2001:4860:4801:91::/64",
"2001:4860:4801:92::/64",
"2001:4860:4801:93::/64",
"2001:4860:4801:94::/64",
"2001:4860:4801:95::/64",
"2001:4860:4801:96::/64",
"2001:4860:4801:97::/64",
"2001:4860:4801:a0::/64",
"2001:4860:4801:a1::/64",
"2001:4860:4801:a2::/64",
"2001:4860:4801:a3::/64",
"2001:4860:4801:a4::/64",
"2001:4860:4801:a5::/64",
"2001:4860:4801:a6::/64",
"2001:4860:4801:a7::/64",
"2001:4860:4801:a8::/64",
"2001:4860:4801:a9::/64",
"2001:4860:4801:aa::/64",
"2001:4860:4801:ab::/64",
"2001:4860:4801:ac::/64",
"2001:4860:4801:ad::/64",
"2001:4860:4801:ae::/64",
"2001:4860:4801:b0::/64",
"2001:4860:4801:b1::/64",
"2001:4860:4801:b2::/64",
"2001:4860:4801:b3::/64",
"2001:4860:4801:b4::/64",
"2001:4860:4801:b5::/64",
"2001:4860:4801:b6::/64",
"2001:4860:4801:c::/64",
"2001:4860:4801:f::/64",
"34.100.182.96/28",
"34.101.50.144/28",
"34.118.254.0/28",
"34.118.66.0/28",
"34.126.178.96/28",
"34.146.150.144/28",
"34.147.110.144/28",
"34.151.74.144/28",
"34.152.50.64/28",
"34.154.114.144/28",
"34.155.98.32/28",
"34.165.18.176/28",
"34.175.160.64/28",
"34.176.130.16/28",
"34.22.85.0/27",
"34.64.82.64/28",
"34.65.242.112/28",
"34.80.50.80/28",
"34.88.194.0/28",
"34.89.10.80/28",
"34.89.198.80/28",
"34.96.162.48/28",
"35.247.243.240/28",
"66.249.64.0/27",
"66.249.64.128/27",
"66.249.64.160/27",
"66.249.64.192/27",
"66.249.64.224/27",
"66.249.64.32/27",
"66.249.64.64/27",
"66.249.64.96/27",
"66.249.65.0/27",
"66.249.65.128/27",
"66.249.65.160/27",
"66.249.65.192/27",
"66.249.65.224/27",
"66.249.65.32/27",
"66.249.65.64/27",
"66.249.65.96/27",
"66.249.66.0/27",
"66.249.66.128/27",
"66.249.66.160/27",
"66.249.66.192/27",
"66.249.66.224/27",
"66.249.66.32/27",
"66.249.66.64/27",
"66.249.66.96/27",
"66.249.67.0/27",
"66.249.67.32/27",
"66.249.67.64/27",
"66.249.68.0/27",
"66.249.68.128/27",
"66.249.68.160/27",
"66.249.68.192/27",
"66.249.68.224/27",
"66.249.68.32/27",
"66.249.68.64/27",
"66.249.68.96/27",
"66.249.69.0/27",
"66.249.69.128/27",
"66.249.69.160/27",
"66.249.69.192/27",
"66.249.69.224/27",
"66.249.69.32/27",
"66.249.69.64/27",
"66.249.69.96/27",
"66.249.70.0/27",
"66.249.70.128/27",
"66.249.70.160/27",
"66.249.70.192/27",
"66.249.70.224/27",
"66.249.70.32/27",
"66.249.70.64/27",
"66.249.70.96/27",
"66.249.71.0/27",
"66.249.71.128/27",
"66.249.71.160/27",
"66.249.71.192/27",
"66.249.71.224/27",
"66.249.71.32/27",
"66.249.71.64/27",
"66.249.71.96/27",
"66.249.72.0/27",
"66.249.72.128/27",
"66.249.72.160/27",
"66.249.72.192/27",
"66.249.72.224/27",
"66.249.72.32/27",
"66.249.72.64/27",
"66.249.72.96/27",
"66.249.73.0/27",
"66.249.73.128/27",
"66.249.73.160/27",
"66.249.73.192/27",
"66.249.73.224/27",
"66.249.73.32/27",
"66.249.73.64/27",
"66.249.73.96/27",
"66.249.74.0/27",
"66.249.74.128/27",
"66.249.74.160/27",
"66.249.74.192/27",
"66.249.74.224/27",
"66.249.74.32/27",
"66.249.74.64/27",
"66.249.74.96/27",
"66.249.75.0/27",
"66.249.75.128/27",
"66.249.75.160/27",
"66.249.75.192/27",
"66.249.75.224/27",
"66.249.75.32/27",
"66.249.75.64/27",
"66.249.75.96/27",
"66.249.76.0/27",
"66.249.76.128/27",
"66.249.76.160/27",
"66.249.76.192/27",
"66.249.76.224/27",
"66.249.76.32/27",
"66.249.76.64/27",
"66.249.76.96/27",
"66.249.77.0/27",
"66.249.77.128/27",
"66.249.77.160/27",
"66.249.77.192/27",
"66.249.77.224/27",
"66.249.77.32/27",
"66.249.77.64/27",
"66.249.77.96/27",
"66.249.78.0/27",
"66.249.78.128/27",
"66.249.78.160/27",
"66.249.78.192/27",
"66.249.78.224/27",
"66.249.78.32/27",
"66.249.78.64/27",
"66.249.78.96/27",
"66.249.79.0/27",
"66.249.79.128/27",
"66.249.79.160/27",
"66.249.79.192/27",
"66.249.79.224/27",
"66.249.79.32/27",
"66.249.79.64/27"
],
"source": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
"verifies": "googlebot"
},
{
"group": "bot",
"kind": "search-engine",
"name": "Google special crawlers",
"prefixes": [
"108.177.2.0/27",
"108.177.2.128/27",
"108.177.2.160/27",
"108.177.2.192/27",
"108.177.2.224/27",
"108.177.2.32/27",
"108.177.2.64/27",
"108.177.2.96/27",
"192.178.16.0/27",
"192.178.16.128/27",
"192.178.16.160/27",
"192.178.16.192/27",
"192.178.16.224/27",
"192.178.16.32/27",
"192.178.16.64/27",
"192.178.16.96/27",
"192.178.17.0/27",
"2001:4860:4801:2008::/64",
"2001:4860:4801:200c::/64",
"2001:4860:4801:200d::/64",
"2001:4860:4801:2010::/64",
"2001:4860:4801:2011::/64",
"2001:4860:4801:2012::/64",
"2001:4860:4801:2013::/64",
"2001:4860:4801:2014::/64",
"2001:4860:4801:2015::/64",
"2001:4860:4801:2016::/64",
"2001:4860:4801:2017::/64",
"2001:4860:4801:2018::/64",
"2001:4860:4801:2019::/64",
"2001:4860:4801:201a::/64",
"2001:4860:4801:201b::/64",
"2001:4860:4801:201c::/64",
"2001:4860:4801:201d::/64",
"2001:4860:4801:201e::/64",
"2001:4860:4801:201f::/64",
"2001:4860:4801:2020::/64",
"2001:4860:4801:2021::/64",
"2001:4860:4801:2022::/64",
"2001:4860:4801:2023::/64",
"2001:4860:4801:2024::/64",
"2001:4860:4801:2025::/64",
"2001:4860:4801:2026::/64",
"2001:4860:4801:2027::/64",
"2001:4860:4801:2028::/64",
"2001:4860:4801:2029::/64",
"2001:4860:4801:202a::/64",
"2001:4860:4801:202b::/64",
"2001:4860:4801:202c::/64",
"2001:4860:4801:202d::/64",
"2001:4860:4801:202e::/64",
"2001:4860:4801:202f::/64",
"2001:4860:4801:2030::/64",
"2001:4860:4801:2031::/64",
"2001:4860:4801:2032::/64",
"2001:4860:4801:2033::/64",
"2001:4860:4801:2034::/64",
"2001:4860:4801:2035::/64",
"2001:4860:4801:2036::/64",
"2001:4860:4801:2037::/64",
"2001:4860:4801:2038::/64",
"2001:4860:4801:2039::/64",
"2001:4860:4801:203a::/64",
"2001:4860:4801:203b::/64",
"2001:4860:4801:203c::/64",
"2001:4860:4801:203d::/64",
"2001:4860:4801:203e::/64",
"2001:4860:4801:203f::/64",
"2001:4860:4801:2040::/64",
"2001:4860:4801:2041::/64",
"2001:4860:4801:2042::/64",
"2001:4860:4801:2043::/64",
"2001:4860:4801:2044::/64",
"2001:4860:4801:2045::/64",
"2001:4860:4801:2046::/64",
"2001:4860:4801:2047::/64",
"2001:4860:4801:2048::/64",
"2001:4860:4801:2049::/64",
"2001:4860:4801:204a::/64",
"2001:4860:4801:204b::/64",
"2001:4860:4801:204c::/64",
"2001:4860:4801:204d::/64",
"2001:4860:4801:204e::/64",
"2001:4860:4801:2050::/64",
"2001:4860:4801:2051::/64",
"2001:4860:4801:2052::/64",
"2001:4860:4801:2053::/64",
"2001:4860:4801:2054::/64",
"2001:4860:4801:2055::/64",
"2001:4860:4801:2056::/64",
"2001:4860:4801:2060::/64",
"2001:4860:4801:2061::/64",
"2001:4860:4801:2062::/64",
"2001:4860:4801:2063::/64",
"2001:4860:4801:2064::/64",
"2001:4860:4801:2065::/64",
"2001:4860:4801:2066::/64",
"2001:4860:4801:2067::/64",
"2001:4860:4801:2068::/64",
"2001:4860:4801:2069::/64",
"2001:4860:4801:206a::/64",
"2001:4860:4801:206b::/64",
"2001:4860:4801:206c::/64",
"2001:4860:4801:206d::/64",
"2001:4860:4801:206e::/64",
"2001:4860:4801:206f::/64",
"2001:4860:4801:2070::/64",
"2001:4860:4801:2071::/64",
"2001:4860:4801:2072::/64",
"2001:4860:4801:2073::/64",
"2001:4860:4801:2074::/64",
"2001:4860:4801:2075::/64",
"2001:4860:4801:2076::/64",
"2001:4860:4801:2077::/64",
"2001:4860:4801:2078::/64",
"2001:4860:4801:2079::/64",
"2001:4860:4801:207a::/64",
"2001:4860:4801:207b::/64",
"2001:4860:4801:207c::/64",
"2001:4860:4801:207d::/64",
"2001:4860:4801:207e::/64",
"2001:4860:4801:207f::/64",
"2001:4860:4801:2080::/64",
"2001:4860:4801:2081::/64",
"2001:4860:4801:2082::/64",
"2001:4860:4801:2083::/64",
"2001:4860:4801:2084::/64",
"2001:4860:4801:2085::/64",
"2001:4860:4801:2086::/64",
"2001:4860:4801:2087::/64",
"2001:4860:4801:2088::/64",
"2001:4860:4801:2090::/64",
"2001:4860:4801:2091::/64",
"2001:4860:4801:2092::/64",
"2001:4860:4801:2093::/64",
"2001:4860:4801:2094::/64",
"2001:4860:4801:2095::/64",
"2001:4860:4801:2096::/64",
"2001:4860:4801:2097::/64",
"2001:4860:4801:20a0::/64",
"2001:4860:4801:20a1::/64",
"2001:4860:4801:20a2::/64",
"2001:4860:4801:20a3::/64",
"2001:4860:4801:20a4::/64",
"2001:4860:4801:20a5::/64",
"2001:4860:4801:20a6::/64",
"2001:4860:4801:20b0::/64",
"2001:4860:4801:20b1::/64",
"2001:4860:4801:20b2::/64",
"2001:4860:4801:20b3::/64",
"2001:4860:4801:20b4::/64",
"2001:4860:4801:20b5::/64",
"2001:4860:4801:20b6::/64",
"209.85.238.0/27",
"209.85.238.128/27",
"209.85.238.160/27",
"209.85.238.192/27",
"209.85.238.224/27",
"209.85.238.32/27",
"209.85.238.64/27",
"209.85.238.96/27",
"66.249.87.0/27",
"66.249.87.128/27",
"66.249.87.160/27",
"66.249.87.192/27",
"66.249.87.224/27",
"66.249.87.32/27",
"66.249.87.64/27",
"66.249.87.96/27",
"66.249.89.0/27",
"66.249.89.128/27",
"66.249.89.160/27",
"66.249.89.192/27",
"66.249.89.224/27",
"66.249.89.32/27",
"66.249.89.64/27",
"66.249.89.96/27",
"66.249.90.0/27",
"66.249.90.128/27",
"66.249.90.160/27",
"66.249.90.192/27",
"66.249.90.224/27",
"66.249.90.32/27",
"66.249.90.64/27",
"66.249.90.96/27",
"66.249.91.0/27",
"66.249.91.128/27",
"66.249.91.160/27",
"66.249.91.192/27",
"66.249.91.224/27",
"66.249.91.32/27",
"66.249.91.64/27",
"66.249.91.96/27",
"66.249.92.0/27",
"66.249.92.128/27",
"66.249.92.160/27",
"66.249.92.192/27",
"66.249.92.224/27",
"66.249.92.32/27",
"66.249.92.64/27",
"66.249.92.96/27",
"72.14.199.0/27",
"72.14.199.128/27",
"72.14.199.160/27",
"72.14.199.192/27",
"72.14.199.224/27",
"72.14.199.32/27",
"72.14.199.64/27",
"72.14.199.96/27",
"74.125.148.0/27",
"74.125.148.128/27",
"74.125.148.160/27",
"74.125.148.192/27",
"74.125.148.224/27",
"74.125.148.32/27",
"74.125.148.64/27",
"74.125.148.96/27",
"74.125.149.0/27",
"74.125.149.128/27",
"74.125.149.160/27",
"74.125.149.192/27",
"74.125.149.224/27",
"74.125.149.32/27",
"74.125.149.64/27",
"74.125.149.96/27",
"74.125.150.0/27",
"74.125.150.128/27",
"74.125.150.160/27",
"74.125.150.192/27",
"74.125.150.32/27",
"74.125.150.64/27",
"74.125.150.96/27",
"74.125.151.0/27",
"74.125.151.128/27",
"74.125.151.160/27",
"74.125.151.192/27",
"74.125.151.224/27",
"74.125.151.32/27",
"74.125.151.64/27",
"74.125.151.96/27",
"74.125.216.0/27",
"74.125.216.128/27",
"74.125.216.160/27",
"74.125.216.192/27",
"74.125.216.224/27",
"74.125.216.32/27",
"74.125.216.64/27",
"74.125.216.96/27",
"74.125.217.0/27",
"74.125.217.128/27",
"74.125.217.160/27",
"74.125.217.192/27",
"74.125.217.224/27",
"74.125.217.32/27",
"74.125.217.64/27",
"74.125.217.96/27",
"74.125.218.0/27",
"74.125.218.128/27",
"74.125.218.160/27",
"74.125.218.192/27",
"74.125.218.224/27",
"74.125.218.32/27",
"74.125.218.64/27",
"74.125.218.96/27",
"74.125.219.0/27",
"74.125.219.128/27",
"74.125.219.160/27",
"74.125.219.192/27",
"74.125.219.224/27",
"74.125.219.32/27",
"74.125.219.64/27",
"74.125.219.96/27"
],
"source": "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json",
"verifies": ""
},
{
"group": "bot",
"kind": "fetcher",
"name": "Google user-triggered fetchers",
"prefixes": [
"107.178.192.0/27",
"107.178.192.128/27",
"107.178.192.160/27",
"107.178.192.192/27",
"107.178.192.224/27",
"107.178.192.32/27",
"107.178.192.64/27",
"107.178.192.96/27",
"107.178.193.0/27",
"107.178.193.128/27",
"107.178.193.160/27",
"107.178.193.192/27",
"107.178.193.224/27",
"107.178.193.32/27",
"107.178.193.64/27",
"107.178.193.96/27",
"107.178.194.0/27",
"107.178.194.128/27",
"107.178.194.160/27",
"107.178.194.192/27",
"107.178.194.224/27",
"107.178.194.32/27",
"107.178.194.64/27",
"107.178.194.96/27",
"107.178.195.0/27",
"107.178.195.128/27",
"107.178.195.160/27",
"107.178.195.192/27",
"107.178.195.224/27",
"107.178.195.32/27",
"107.178.195.64/27",
"107.178.195.96/27",
"107.178.196.0/27",
"107.178.196.128/27",
"107.178.196.160/27",
"107.178.196.192/27",
"107.178.196.224/27",
"107.178.196.32/27",
"107.178.196.64/27",
"107.178.196.96/27",
"107.178.197.0/27",
"107.178.197.128/27",
"107.178.197.160/27",
"107.178.197.192/27",
"107.178.197.224/27",
"107.178.197.32/27",
"107.178.197.64/27",
"107.178.197.96/27",
"107.178.198.0/27",
"107.178.198.128/27",
"107.178.198.160/27",
"107.178.198.192/27",
"107.178.198.224/27",
"107.178.198.32/27",
"107.178.198.64/27",
"107.178.198.96/27",
"107.178.199.0/27",
"107.178.199.128/27",
"107.178.199.160/27",
"107.178.199.192/27",
"107.178.199.224/27",
"107.178.199.32/27",
"107.178.199.64/27",
"107.178.199.96/27",
"107.178.202.0/27",
"107.178.202.128/27",
"107.178.202.160/27",
"107.178.202.192/27",
"107.178.202.224/27",
"107.178.202.32/27",
"107.178.202.64/27",
"107.178.202.96/27",
"107.178.203.0/27",
"107.178.203.128/27",
"107.178.203.160/27",
"107.178.203.192/27",
"107.178.203.224/27",
"107.178.203.32/27",
"107.178.203.64/27",
"107.178.224.0/27",
"107.178.224.128/27",
"107.178.224.160/27",
"107.178.224.192/27",
"107.178.224.224/27",
"107.178.224.32/27",
"107.178.224.64/27",
"107.178.224.96/27",
"107.178.225.0/27",
"107.178.225.128/27",
"107.178.225.160/27",
"107.178.225.192/27",
"107.178.225.224/27",
"107.178.225.32/27",
"107.178.225.64/27",
"107.178.225.96/27",
"136.124.136.0/27",
"136.124.136.32/27",
"136.124.136.64/27",
"2404:f340:10:4000::/64",
"2404:f340:10:4001::/64",
"2404:f340:10:4002::/64",
"2404:f340:10:4003::/64",
"2404:f340:10:4004::/64",
"2404:f340:10:4005::/64",
"2404:f340:10:4006::/64",
"2404:f340:10:4007::/64",
"2404:f340:4010:4000::/64",
"2404:f340:4010:4001::/64",
"2404:f340:4010:4002::/64",
"2404:f340:4010:4003::/64",
"2404:f340:4010:4004::/64",
"2404:f340:4010:4005::/64",
"2404:f340:4010:4006::/64",
"2404:f340:4010:4007::/64",
"2600:1900:0:100::/64",
"2600:1900:0:101::/64",
"2600:1900:0:102::/64",
"2600:1900:0:103::/64",
"2600:1900:0:104::/64",
"2600:1900:0:105::/64",
"2600:1900:0:106::/64",
"2600:1900:0:107::/64",
"2600:1900:0:108::/64",
"2600:1900:0:109::/64",
"2600:1900:0:10::/64",
"2600:1900:0:10a::/64",
"2600:1900:0:10b::/64",
"2600:1900:0:10c::/64",
"2600:1900:0:10d::/64",
"2600:1900:0:10e::/64",
"2600:1900:0:10f::/64",
"2600:1900:0:110::/64",
"2600:1900:0:111::/64",
"2600:1900:0:112::/64",
"2600:1900:0:113::/64",
"2600:1900:0:114::/64",
"2600:1900:0:115::/64",
"2600:1900:0:116::/64",
"2600:1900:0:117::/64",
"2600:1900:0:118::/64",
"2600:1900:0:119::/64",
"2600:1900:0:11a::/64",
"2600:1900:0:11b::/64",
"2600:1900:0:11c::/64",
"2600:1900:0:11d::/64",
"2600:1900:0:11e::/64",
"2600:1900:0:11f::/64",
"2600:1900:0:120::/64",
"2600:1900:0:121::/64",
"2600:1900:0:122::/64",
"2600:1900:0:123::/64",
"2600:1900:0:130::/64",
"2600:1900:0:131::/64",
"2600:1900:0:132::/64",
"2600:1900:0:133::/64",
"2600:1900:0:138::/64",
"2600:1900:0:139::/64",
"2600:1900:0:13a::/64",
"2600:1900:0:13b::/64",
"2600:1900:0:140::/64",
"2600:1900:0:141::/64",
"2600:1900:0:142::/64",
"2600:1900:0:143::/64",
"2600:1900:0:144::/64",
"2600:1900:0:145::/64",
"2600:1900:0:146::/64",
"2600:1900:0:150::/64",
"2600:1900:0:151::/64",
"2600:1900:0:152::/64",
"2600:1900:0:153::/64",
"2600:1900:0:154::/64",
"2600:1900:0:155::/64",
"2600:1900:0:156::/64",
"2600:1900:0:157::/64",
"2600:1900:0:158::/64",
"2600:1900:0:159::/64",
"2600:1900:0:15::/64",
"2600:1900:0:15a::/64",
"2600:1900:0:15b::/64",
"2600:1900:0:15c::/64",
"2600:1900:0:15d::/64",
"2600:1900:0:160::/64",
"2600:1900:0:161::/64",
"2600:1900:0:162::/64",
"2600:1900:0:163::/64",
"2600:1900:0:164::/64",
"2600:1900:0:165::/64",
"2600:1900:0:166::/64",
"2600:1900:0:167::/64",
"2600:1900:0:168::/64",
"2600:1900:0:169::/64",
"2600:1900:0:16::/64",
"2600:1900:0:16a::/64",
"2600:1900:0:16b::/64",
"2600:1900:0:16c::/64",
"2600:1900:0:16d::/64",
"2600:1900:0:16e::/64",
"2600:1900:0:16f::/64",
"2600:1900:0:170::/64",
"2600:1900:0:171::/64",
"2600:1900:0:172::/64",
"2600:1900:0:173::/64",
"2600:1900:0:17::/64",
"2600:1900:0:180::/64",
"2600:1900:0:181::/64",
"2600:1900:0:182::/64",
"2600:1900:0:183::/64",
"2600:1900:0:184::/64",
"2600:1900:0:185::/64",
"2600:1900:0:186::/64",
"2600:1900:0:187::/64",
"2600:1900:0:188::/64",
"2600:1900:0:189::/64",
"2600:1900:0:18::/64",
"2600:1900:0:18a::/64",
"2600:1900:0:18b::/64",
"2600:1900:0:18c::/64",
"2600:1900:0:18d::/64",
"2600:1900:0:18e::/64",
"2600:1900:0:18f::/64",
"2600:1900:0:190::/64",
"2600:1900:0:191::/64",
"2600:1900:0:192::/64",
"2600:1900:0:193::/64",
"2600:1900:0:194::/64",
"2600:1900:0:195::/64",
"2600:1900:0:196::/64",
"2600:1900:0:1a0::/64",
"2600:1900:0:1a1::/64",
"2600:1900:0:1a2::/64",
"2600:1900:0:1a3::/64",
"2600:1900:0:1a::/64",
"2600:1900:0:1b::/64",
"2600:1900:0:1c0::/64",
"2600:1900:0:1c1::/64",
"2600:1900:0:1c2::/64",
"2600:1900:0:1c3::/64",
"2600:1900:0:1c4::/64",
"2600:1900:0:1c5::/64",
"2600:1900:0:1c6::/64",
"2600:1900:0:1c::/64",
"2600:1900:0:1d0::/64",
"2600:1900:0:1d1::/64",
"2600:1900:0:1e0::/64",
"2600:1900:0:1e1::/64",
"2600:1900:0:1e2::/64",
"2600:1900:0:1e3::/64",
"2600:1900:0:1e4::/64",
"2600:1900:0:1f0::/64",
"2600:1900:0:1f1::/64",
"2600:1900:0:1f2::/64",
"2600:1900:0:1f3::/64",
"2600:1900:0:1f4::/64",
"2600:1900:0:1f5::/64",
"2600:1900:0:1f6::/64",
"2600:1900:0:1f7::/64",
"2600:1900:0:1f8::/64",
"2600:1900:0:200::/64",
"2600:1900:0:201::/64",
"2600:1900:0:202::/64",
"2600:1900:0:203::/64",
"2600:1900:0:204::/64",
"2600:1900:0:205::/64",
"2600:1900:0:206::/64",
"2600:1900:0:207::/64",
"2600:1900:0:208::/64",
"2600:1900:0:209::/64",
"2600:1900:0:20::/64",
"2600:1900:0:20a::/64",
"2600:1900:0:20b::/64",
"2600:1900:0:210::/64",
"2600:1900:0:211::/64",
"2600:1900:0:212::/64",
"2600:1900:0:213::/64",
"2600:1900:0:214::/64",
"2600:1900:0:215::/64",
"2600:1900:0:216::/64",
"2600:1900:0:217::/64",
"2600:1900:0:21::/64",
"2600:1900:0:220::/64",
"2600:1900:0:221::/64",
"2600:1900:0:222::/64",
"2600:1900:0:223::/64",
"2600:1900:0:22::/64",
"2600:1900:0:230::/64",
"2600:1900:0:231::/64",
"2600:1900:0:232::/64",
"2600:1900:0:233::/64",
"2600:1900:0:234::/64",
"2600:1900:0:235::/64",
"2600:1900:0:236::/64",
"2600:1900:0:237::/64",
"2600:1900:0:238::/64",
"2600:1900:0:239::/64",
"2600:1900:0:23::/64",
"2600:1900:0:23a::/64",
"2600:1900:0:23b::/64",
"2600:1900:0:240::/64",
"2600:1900:0:241::/64",
"2600:1900:0:242::/64",
"2600:1900:0:24::/64",
"2600:1900:0:250::/64",
"2600:1900:0:251::/64",
"2600:1900:0:252::/64",
"2600:1900:0:253::/64",
"2600:1900:0:254::/64",
"2600:1900:0:255::/64",
"2600:1900:0:25::/64",
"2600:1900:0:260::/64",
"2600:1900:0:261::/64",
"2600:1900:0:262::/64",
"2600:1900:0:263::/64",
"2600:1900:0:264::/64",
"2600:1900:0:26::/64",
"2600:1900:0:270::/64",
"2600:1900:0:271::/64",
"2600:1900:0:272::/64",
"2600:1900:0:273::/64",
"2600:1900:0:274::/64",
"2600:1900:0:275::/64",
"2600:1900:0:276::/64",
"2600:1900:0:27::/64",
"2600:1900:0:280::/64",
"2600:1900:0:281::/64",
"2600:1900:0:282::/64",
"2600:1900:0:283::/64",
"2600:1900:0:28::/64",
"2600:1900:0:290::/64",
"2600:1900:0:291::/64",
"2600:1900:0:292::/64",
"2600:1900:0:293::/64",
"2600:1900:0:294::/64",
"2600:1900:0:295::/64",
"2600:1900:0:296::/64",
"2600:1900:0:297::/64",
"2600:1900:0:29::/64",
"2600:1900:0:2a0::/64",
"2600:1900:0:2a1::/64",
"2600:1900:0:2a2::/64",
"2600:1900:0:2a3::/64",
"2600:1900:0:2a4::/64",
"2600:1900:0:2a::/64",
"2600:1900:0:2b0::/64",
"2600:1900:0:2b1::/64",
"2600:1900:0:2b2::/64",
"2600:1900:0:2b3::/64",
"2600:1900:0:2b::/64",
"2600:1900:0:2c0::/64",
"2600:1900:0:2c1::/64",
"2600:1900:0:2c2::/64",
"2600:1900:0:2c3::/64",
"2600:1900:0:2c4::/64",
"2600:1900:0:2c5::/64",
"2600:1900:0:2c6::/64",
"2600:1900:0:2d0::/64",
"2600:1900:0:2d1::/64",
"2600:1900:0:2d2::/64",
"2600:1900:0:2d3::/64",
"2600:1900:0:2d4::/64",
"2600:1900:0:2d5::/64",
"2600:1900:0:2d6::/64",
"2600:1900:0:2d7::/64",
"2600:1900:0:2d8::/64",
"2600:1900:0:2d9::/64",
"2600:1900:0:2da::/64",
"2600:1900:0:2db::/64",
"2600:1900:0:2dc::/64",
"2600:1900:0:2dd::/64",
"2600:1900:0:2de::/64",
"2600:1900:0:2df::/64",
"2600:1900:0:2e0::/64",
"2600:1900:0:2e1::/64",
"2600:1900:0:2e2::/64",
"2600:1900:0:2f0::/64",
"2600:1900:0:2f1::/64",
"2600:1900:0:2f2::/64",
"2600:1900:0:2f3::/64",
"2600:1900:0:2f4::/64",
"2600:1900:0:2f5::/64",
"2600:1900:0:2f6::/64",
"2600:1900:0:300::/64",
"2600:1900:0:301::/64",
"2600:1900:0:302::/64",
"2600:1900:0:303::/64",
"2600:1900:0:30::/64",
"2600:1900:0:310::/64",
"2600:1900:0:311::/64",
"2600:1900:0:312::/64",
"2600:1900:0:313::/64",
"2600:1900:0:314::/64",
"2600:1900:0:315::/64",
"2600:1900:0:316::/64",
"2600:1900:0:317::/64",
"2600:1900:0:31::/64",
"2600:1900:0:320::/64",
"2600:1900:0:321::/64",
"2600:1900:0:322::/64",
"2600:1900:0:32::/64",
"2600:1900:0:330::/64",
"2600:1900:0:331::/64",
"2600:1900:0:332::/64",
"2600:1900:0:333::/64",
"2600:1900:0:334::/64",
"2600:1900:0:335::/64",
"2600:1900:0:33::/64",
"2600:1900:0:340::/64",
"2600:1900:0:34::/64",
"2600:1900:0:350::/64",
"2600:1900:0:351::/64",
"2600:1900:0:352::/64",
"2600:1900:0:353::/64",
"2600:1900:0:354::/64",
"2600:1900:0:35::/64",
"2600:1900:0:360::/64",
"2600:1900:0:361::/64",
"2600:1900:0:362::/64",
"2600:1900:0:36::/64",
"2600:1900:0:370::/64",
"2600:1900:0:371::/64",
"2600:1900:0:372::/64",
"2600:1900:0:373::/64",
"2600:1900:0:374::/64",
"2600:1900:0:37::/64",
"2600:1900:0:380::/64",
"2600:1900:0:381::/64",
"2600:1900:0:382::/64",
"2600:1900:0:383::/64",
"2600:1900:0:384::/64",
"2600:1900:0:385::/64",
"2600:1900:0:386::/64",
"2600:1900:0:387::/64",
"2600:1900:0:388::/64",
"2600:1900:0:389::/64",
"2600:1900:0:38::/64",
"2600:1900:0:38a::/64",
"2600:1900:0:38b::/64",
"2600:1900:0:38c::/64",
"2600:1900:0:38d::/64",
"2600:1900:0:38e::/64",
"2600:1900:0:390::/64",
"2600:1900:0:391::/64",
"2600:1900:0:392::/64",
"2600:1900:0:393::/64",
"2600:1900:0:39::/64",
"2600:1900:0:3a0::/64",
"2600:1900:0:3a1::/64",
"2600:1900:0:3a2::/64",
"2600:1900:0:3a::/64",
"2600:1900:0:3b0::/64",
"2600:1900:0:3b1::/64",
"2600:1900:0:3b2::/64",
"2600:1900:0:3b3::/64",
"2600:1900:0:3b4::/64",
"2600:1900:0:3b5::/64",
"2600:1900:0:3b6::/64",
"2600:1900:0:3b::/64",
"2600:1900:0:3c0::/64",
"2600:1900:0:3c1::/64",
"2600:1900:0:3c3::/64",
"2600:1900:0:3c4::/64",
"2600:1900:0:3c::/64",
"2600:1900:0:3d0::/64",
"2600:1900:0:3d1::/64",
"2600:1900:0:3d2::/64",
"2600:1900:0:3d3::/64",
"2600:1900:0:3d4::/64",
"2600:1900:0:3d5::/64",
"2600:1900:0:3d6::/64",
"2600:1900:0:3d7::/64",
"2600:1900:0:40::/64",
"2600:1900:0:410::/64",
"2600:1900:0:411::/64",
"2600:1900:0:412::/64",
"2600:1900:0:41::/64",
"2600:1900:0:42::/64",
"2600:1900:0:43::/64",
"2600:1900:0:44::/64",
"2600:1900:0:45::/64",
"2600:1900:0:46::/64",
"2600:1900:0:47::/64",
"2600:1900:0:48::/64",
"2600:1900:0:49::/64",
"2600:1900:0:4a::/64",
"2600:1900:0:4b::/64",
"2600:1900:0:4c::/64",
"2600:1900:0:4d::/64",
"2600:1900:0:4e::/64",
"2600:1900:0:4f::/64",
"2600:1900:0:50::/64",
"2600:1900:0:51::/64",
"2600:1900:0:52::/64",
"2600:1900:0:53::/64",
"2600:1900:0:54::/64",
"2600:1900:0:55::/64",
"2600:1900:0:56::/64",
"2600:1900:0:57::/64",
"2600:1900:0:58::/64",
"2600:1900:0:59::/64",
"2600:1900:0:5a::/64",
"2600:1900:0:5b::/64",
"2600:1900:0:5c::/64",
"2600:1900:0:5d::/64",
"2600:1900:0:5e::/64",
"2600:1900:0:5f::/64",
"2600:1900:0:60::/64",
"2600:1900:0:61::/64",
"2600:1900:0:62::/64",
"2600:1900:0:63::/64",
"2600:1900:0:64::/64",
"2600:1900:0:65::/64",
"2600:1900:0:66::/64",
"2600:1900:0:67::/64",
"2600:1900:0:68::/64",
"2600:1900:0:69::/64",
"2600:1900:0:6a::/64",
"2600:1900:0:6b::/64",
"2600:1900:0:6c::/64",
"2600:1900:0:6d::/64",
"2600:1900:0:6e::/64",
"2600:1900:0:70::/64",
"2600:1900:0:71::/64",
"2600:1900:0:72::/64",
"2600:1900:0:73::/64",
"2600:1900:0:74::/64",
"2600:1900:0:75::/64",
"2600:1900:0:76::/64",
"2600:1900:0:77::/64",
"2600:1900:0:78::/64",
"2600:1900:0:79::/64",
"2600:1900:0:7a::/64",
"2600:1900:0:7b::/64",
"2600:1900:0:7c::/64",
"2600:1900:0:7d::/64",
"2600:1900:0:7e::/64",
"2600:1900:0:7f::/64",
"2600:1900:0:80::/64",
"2600:1900:0:81::/64",
"2600:1900:0:82::/64",
"2600:1900:0:83::/64",
"2600:1900:0:84::/64",
"2600:1900:0:85::/64",
"2600:1900:0:86::/64",
"2600:1900:0:87::/64",
"2600:1900:0:88::/64",
"2600:1900:0:89::/64",
"2600:1900:0:8::/64",
"2600:1900:0:8a::/64",
"2600:1900:0:8b::/64",
"2600:1900:0:90::/64",
"2600:1900:0:91::/64",
"2600:1900:0:92::/64",
"2600:1900:0:93::/64",
"2600:1900:0:94::/64",
"2600:1900:0:95::/64",
"2600:1900:0:96::/64",
"2600:1900:0:97::/64",
"2600:1900:0:98::/64",
"2600:1900:0:99::/64",
"2600:1900:0:9::/64",
"2600:1900:0:a0::/64",
"2600:1900:0:a1::/64",
"2600:1900:0:a2::/64",
"2600:1900:0:a3::/64",
"2600:1900:0:a4::/64",
"2600:1900:0:a5::/64",
"2600:1900:0:a6::/64",
"2600:1900:0:b0::/64",
"2600:1900:0:b1::/64",
"2600:1900:0:b2::/64",
"2600:1900:0:b3::/64",
"2600:1900:0:b4::/64",
"2600:1900:0:b5::/64",
"2600:1900:0:b6::/64",
"2600:1900:0:b7::/64",
"2600:1900:0:b8::/64",
"2600:1900:0:b9::/64",
"2600:1900:0:ba::/64",
"2600:1900:0:bb::/64",
"2600:1900:0:bc::/64",
"2600:1900:0:bd::/64",
"2600:1900:0:c0::/64",
"2600:1900:0:c1::/64",
"2600:1900:0:c2::/64",
"2600:1900:0:c3::/64",
"2600:1900:0:c4::/64",
"2600:1900:0:c5::/64",
"2600:1900:0:c6::/64",
"2600:1900:0:c::/64",
"2600:1900:0:d0::/64",
"2600:1900:0:d1::/64",
"2600:1900:0:d2::/64",
"2600:1900:0:d3::/64",
"2600:1900:0:d4::/64",
"2600:1900:0:d5::/64",
"2600:1900:0:d6::/64",
"2600:1900:0:d7::/64",
"2600:1900:0:d8::/64",
"2600:1900:0:d9::/64",
"2600:1900:0:da::/64",
"2600:1900:0:db::/64",
"2600:1900:0:dc::/64",
"2600:1900:0:dd::/64",
"2600:1900:0:de::/64",
"2600:1900:0:df::/64",
"2600:1900:0:e0::/64",
"2600:1900:0:e1::/64",
"2600:1900:0:e2::/64",
"2600:1900:0:e3::/64",
"2600:1900:0:e4::/64",
"2600:1900:0:e5::/64",
"2600:1900:0:e6::/64",
"2600:1900:0:e7::/64",
"2600:1900:0:e8::/64",
"2600:1900:0:e9::/64",
"2600:1900:0:ea::/64",
"2600:1900:0:eb::/64",
"2600:1900:0:ec::/64",
"2600:1900:0:ed::/64",
"2600:1900:0:ee::/64",
"2600:1900:0:ef::/64",
"2600:1900:0:f0::/64",
"2600:1900:0:f1::/64",
"2600:1900:0:f2::/64",
"2600:1900:0:f3::/64",
"2600:1900:0:f4::/64",
"2600:1900:0:f5::/64",
"2600:1900:0:f6::/64",
"2600:1900:0:f7::/64",
"2600:1900:0:f8::/64",
"2600:1900:0:f9::/64",
"2600:1900:0:f::/64",
"2600:1900:0:fa::/64",
"34.101.6.0/27",
"34.101.6.128/27",
"34.101.6.160/27",
"34.101.6.192/27",
"34.101.6.224/27",
"34.101.6.32/27",
"34.101.6.64/27",
"34.101.6.96/27",
"34.116.16.0/27",
"34.116.16.128/27",
"34.116.16.160/27",
"34.116.16.32/27",
"34.116.16.64/27",
"34.116.16.96/27",
"34.116.17.0/27",
"34.116.17.128/27",
"34.116.17.32/27",
"34.116.17.64/27",
"34.116.17.96/27",
"34.116.18.0/27",
"34.116.18.128/27",
"34.116.18.160/27",
"34.116.18.192/27",
"34.116.18.32/27",
"34.116.18.64/27",
"34.116.18.96/27",
"34.116.19.0/27",
"34.116.19.128/27",
"34.116.19.160/27",
"34.116.19.192/27",
"34.116.19.224/27",
"34.116.19.32/27",
"34.116.19.64/27",
"34.116.19.96/27",
"34.116.20.0/27",
"34.116.20.32/27",
"34.116.20.64/27",
"34.116.20.96/27",
"34.116.21.0/27",
"34.116.21.128/27",
"34.116.21.160/27",
"34.116.21.192/27",
"34.116.21.224/27",
"34.116.21.32/27",
"34.116.21.64/27",
"34.116.21.96/27",
"34.116.22.0/27",
"34.116.22.128/27",
"34.116.22.32/27",
"34.116.22.64/27",
"34.116.22.96/27",
"34.116.23.0/27",
"34.116.23.32/27",
"34.116.23.64/27",
"34.116.23.96/27",
"34.116.28.0/27",
"34.116.28.128/27",
"34.116.28.160/27",
"34.116.28.192/27",
"34.116.28.224/27",
"34.116.28.32/27",
"34.116.28.64/27",
"34.116.28.96/27",
"34.116.29.0/27",
"34.116.29.32/27",
"34.116.29.64/27",
"34.116.29.96/27",
"34.116.30.0/27",
"34.116.30.128/27",
"34.116.30.160/27",
"34.116.30.192/27",
"34.116.30.224/27",
"34.116.30.32/27",
"34.116.30.64/27",
"34.116.30.96/27",
"34.116.31.0/27",
"34.116.31.32/27",
"34.116.31.64/27",
"34.116.32.0/27",
"34.116.32.128/27",
"34.116.32.160/27",
"34.116.32.192/27",
"34.116.32.32/27",
"34.116.32.64/27",
"34.116.32.96/27",
"34.116.33.0/27",
"34.116.33.128/27",
"34.116.33.160/27",
"34.116.33.192/27",
"34.116.33.224/27",
"34.116.33.32/27",
"34.116.33.64/27",
"34.116.33.96/27",
"34.116.34.0/27",
"34.116.34.32/27",
"34.116.34.64/27",
"34.116.35.0/27",
"34.116.35.128/27",
"34.116.35.160/27",
"34.116.35.192/27",
"34.116.35.32/27",
"34.116.35.64/27",
"34.116.35.96/27",
"34.116.36.0/27",
"34.116.36.128/27",
"34.116.36.160/27",
"34.116.36.32/27",
"34.116.36.64/27",
"34.116.36.96/27",
"34.116.37.0/27",
"34.116.38.0/27",
"34.116.38.32/27",
"34.116.39.0/27",
"34.116.39.128/27",
"34.116.39.160/27",
"34.116.39.192/27",
"34.116.39.224/27",
"34.116.39.32/27",
"34.116.39.64/27",
"34.116.39.96/27",
"34.116.40.0/27",
"34.116.40.128/27",
"34.116.40.160/27",
"34.116.40.192/27",
"34.116.40.224/27",
"34.116.40.32/27",
"34.116.40.64/27",
"34.116.40.96/27",
"34.116.41.0/27",
"34.116.41.128/27",
"34.116.41.160/27",
"34.116.41.192/27",
"34.116.41.224/27",
"34.116.41.32/27",
"34.116.41.64/27",
"34.116.41.96/27",
"34.116.42.0/27",
"34.116.42.128/27",
"34.116.42.32/27",
"34.116.42.64/27",
"34.116.42.96/27",
"34.116.43.0/27",
"34.116.43.128/27",
"34.116.43.160/27",
"34.116.43.192/27",
"34.116.43.224/27",
"34.116.43.32/27",
"34.116.43.64/27",
"34.116.43.96/27",
"34.127.128.0/27",
"34.127.128.128/27",
"34.127.128.32/27",
"34.127.128.64/27",
"34.127.128.96/27",
"34.127.129.0/27",
"34.127.129.128/27",
"34.127.129.32/27",
"34.127.129.64/27",
"34.127.129.96/27",
"34.127.130.0/27",
"34.127.130.32/27",
"34.127.130.64/27",
"34.127.130.96/27",
"34.127.131.0/27",
"34.127.131.32/27",
"34.127.131.64/27",
"34.127.132.0/27",
"34.127.132.128/27",
"34.127.132.160/27",
"34.127.132.192/27",
"34.127.132.224/27",
"34.127.132.32/27",
"34.127.132.64/27",
"34.127.132.96/27",
"34.127.133.0/27",
"34.127.133.128/27",
"34.127.133.160/27",
"34.127.133.192/27",
"34.127.133.224/27",
"34.127.133.32/27",
"34.127.133.64/27",
"34.127.133.96/27",
"34.127.134.0/27",
"34.127.134.32/27",
"34.127.134.64/27",
"34.127.136.0/27",
"34.127.136.128/27",
"34.127.136.160/27",
"34.127.136.192/27",
"34.127.136.224/27",
"34.127.136.32/27",
"34.127.136.64/27",
"34.127.136.96/27",
"34.127.137.0/27",
"34.127.137.32/27",
"34.127.137.64/27",
"34.127.137.96/27",
"34.127.139.0/27",
"34.127.140.0/27",
"34.127.140.128/27",
"34.127.140.160/27",
"34.127.140.192/27",
"34.127.140.224/27",
"34.127.140.32/27",
"34.127.140.64/27",
"34.127.140.96/27",
"34.127.142.0/27",
"34.127.142.128/27",
"34.127.142.32/27",
"34.127.142.96/27",
"34.127.143.0/27",
"34.127.143.32/27",
"34.127.143.64/27",
"34.64.6.0/27",
"34.64.6.128/27",
"34.64.6.160/27",
"34.64.6.192/27",
"34.64.6.224/27",
"34.64.6.32/27",
"34.64.6.64/27",
"34.64.6.96/27",
"34.98.136.0/27",
"34.98.136.128/27",
"34.98.136.160/27",
"34.98.136.192/27",
"34.98.136.224/27",
"34.98.136.32/27",
"34.98.136.64/27",
"34.98.136.96/27",
"34.98.137.0/27",
"34.98.137.32/27",
"34.98.137.64/27",
"34.98.137.96/27",
"34.98.138.0/27",
"34.98.138.128/27",
"34.98.138.32/27",
"34.98.138.64/27",
"34.98.138.96/27",
"34.98.139.0/27",
"34.98.139.128/27",
"34.98.139.160/27",
"34.98.139.192/27",
"34.98.139.224/27",
"34.98.139.32/27",
"34.98.139.64/27",
"34.98.139.96/27",
"34.98.140.0/27",
"34.98.140.128/27",
"34.98.140.160/27",
"34.98.140.192/27",
"34.98.140.224/27",
"34.98.140.32/27",
"34.98.140.64/27",
"34.98.140.96/27",
"34.98.141.0/27",
"34.98.141.128/27",
"34.98.141.160/27",
"34.98.141.192/27",
"34.98.141.224/27",
"34.98.141.32/27",
"34.98.141.64/27",
"34.98.141.96/27",
"34.98.142.0/27",
"34.98.142.32/27",
"34.98.142.64/27",
"34.98.142.96/27",
"34.98.143.0/27",
"34.98.143.128/27",
"34.98.143.160/27",
"34.98.143.192/27",
"34.98.143.224/27",
"34.98.143.32/27",
"34.98.143.64/27",
"34.98.143.96/27",
"35.187.128.0/27",
"35.187.128.128/27",
"35.187.128.160/27",
"35.187.128.192/27",
"35.187.128.224/27",
"35.187.128.32/27",
"35.187.128.64/27",
"35.187.128.96/27",
"35.187.129.0/27",
"35.187.129.128/27",
"35.187.129.160/27",
"35.187.129.192/27",
"35.187.129.224/27",
"35.187.129.32/27",
"35.187.129.64/27",
"35.187.129.96/27",
"35.187.130.0/27",
"35.187.130.128/27",
"35.187.130.160/27",
"35.187.130.192/27",
"35.187.130.224/27",
"35.187.130.32/27",
"35.187.130.64/27",
"35.187.130.96/27",
"35.187.131.0/27",
"35.187.131.128/27",
"35.187.131.160/27",
"35.187.131.192/27",
"35.187.131.224/27",
"35.187.131.32/27",
"35.187.131.64/27",
"35.187.131.96/27",
"35.187.132.0/27",
"35.187.132.128/27",
"35.187.132.160/27",
"35.187.132.192/27",
"35.187.132.224/27",
"35.187.132.32/27",
"35.187.132.64/27",
"35.187.132.96/27",
"35.187.133.0/27",
"35.187.133.128/27",
"35.187.133.160/27",
"35.187.133.192/27",
"35.187.133.224/27",
"35.187.133.32/27",
"35.187.133.64/27",
"35.187.133.96/27",
"35.187.134.0/27",
"35.187.134.128/27",
"35.187.134.160/27",
"35.187.134.192/27",
"35.187.134.224/27",
"35.187.134.32/27",
"35.187.134.64/27",
"35.187.134.96/27",
"35.187.135.0/27",
"35.187.135.128/27",
"35.187.135.160/27",
"35.187.135.192/27",
"35.187.135.224/27",
"35.187.135.32/27",
"35.187.135.64/27",
"35.187.135.96/27",
"35.187.136.0/27",
"35.187.136.128/27",
"35.187.136.160/27",
"35.187.136.192/27",
"35.187.136.224/27",
"35.187.136.32/27",
"35.187.138.0/27",
"35.187.138.128/27",
"35.187.138.160/27",
"35.187.138.192/27",
"35.187.138.224/27",
"35.187.138.32/27",
"35.187.138.64/27",
"35.187.138.96/27",
"35.187.139.0/27",
"35.187.139.128/27",
"35.187.139.160/27",
"35.187.139.192/27",
"35.187.139.224/27",
"35.187.139.32/27",
"35.187.139.64/27",
"35.187.139.96/27",
"35.187.140.0/27",
"35.187.140.128/27",
"35.187.140.160/27",
"35.187.140.192/27",
"35.187.140.32/27",
"35.187.140.64/27",
"35.187.140.96/27",
"35.187.141.0/27",
"35.187.141.128/27",
"35.187.141.160/27",
"35.187.141.192/27",
"35.187.141.224/27",
"35.187.141.32/27",
"35.187.141.64/27",
"35.187.142.0/27",
"35.187.142.32/27",
"35.187.142.64/27",
"35.187.142.96/27",
"35.187.143.0/27",
"35.187.143.128/27",
"35.187.143.160/27",
"35.187.143.192/27",
"35.187.143.224/27",
"35.187.143.32/27",
"35.187.143.64/27",
"35.187.143.96/27",
"35.243.16.0/27",
"35.243.16.128/27",
"35.243.16.32/27",
"35.243.16.64/27",
"35.243.16.96/27",
"35.243.17.0/27",
"35.243.17.128/27",
"35.243.17.160/27",
"35.243.17.192/27",
"35.243.17.32/27",
"35.243.17.64/27",
"35.243.17.96/27",
"35.243.18.0/27",
"35.243.18.32/27",
"35.243.18.64/27",
"35.243.18.96/27",
"35.243.19.0/27",
"35.243.19.128/27",
"35.243.19.160/27",
"35.243.19.192/27",
"35.243.19.32/27",
"35.243.19.64/27",
"35.243.19.96/27",
"35.243.20.0/27",
"35.243.20.128/27",
"35.243.20.160/27",
"35.243.20.192/27",
"35.243.20.224/27",
"35.243.20.32/27",
"35.243.20.64/27",
"35.243.20.96/27",
"35.243.21.0/27",
"35.243.21.32/27",
"35.243.21.64/27",
"35.243.21.96/27",
"35.243.22.0/27",
"35.243.22.128/27",
"35.243.22.160/27",
"35.243.22.192/27",
"35.243.22.32/27",
"35.243.22.64/27",
"35.243.22.96/27",
"35.243.23.0/27",
"35.243.23.128/27",
"35.243.23.160/27",
"35.243.23.192/27",
"35.243.23.224/27",
"35.243.23.32/27",
"35.243.23.64/27",
"35.243.23.96/27"
],
"source": "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
"verifies": ""
},
{
"group": "bot",
"kind": "search-engine",
"name": "Bingbot",
"prefixes": [
"13.66.139.0/24",
"13.66.144.0/24",
"13.67.10.16/28",
"13.69.66.240/28",
"13.71.172.224/28",
"139.217.52.0/28",
"157.55.39.0/24",
"191.233.204.224/28",
"199.30.24.0/23",
"20.125.163.80/28",
"20.15.133.160/27",
"20.36.108.32/28",
"20.43.120.16/28",
"20.74.197.0/28",
"20.79.107.240/28",
"207.46.13.0/24",
"40.77.139.0/25",
"40.77.167.0/24",
"40.77.177.0/24",
"40.77.178.0/23",
"40.77.188.0/22",
"40.77.202.0/24",
"40.79.131.208/28",
"40.79.186.176/28",
"51.105.67.0/28",
"52.167.144.0/24",
"52.231.148.0/28",
"65.55.210.0/24"
],
"source": "https://www.bing.com/toolbox/bingbot.json",
"verifies": "bingbot"
},
{
"group": "bot",
"kind": "search-engine",
"name": "Applebot",
"prefixes": [
"17.166.150.0/24",
"17.166.151.0/24",
"17.166.152.0/24",
"17.166.153.0/24",
"17.166.154.0/24",
"17.166.155.0/24",
"17.166.192.48/28",
"17.166.20.0/24",
"17.166.21.0/24",
"17.166.22.0/24",
"17.166.23.0/24",
"17.166.232.0/24",
"17.166.233.0/24",
"17.166.234.0/24",
"17.166.235.0/24",
"17.166.236.0/24",
"17.166.237.0/24",
"17.166.24.0/24",
"17.166.240.48/28",
"17.166.248.48/28",
"17.166.25.0/24",
"17.241.193.160/27",
"17.241.200.160/27",
"17.241.208.160/27"
],
"source": "https://search.developer.apple.com/applebot.json",
"verifies": "applebot"
},
{
"group": "bot",
"kind": "search-engine",
"name": "DuckDuckBot",
"prefixes": [
"104.43.54.127/32",
"104.43.55.116/32",
"104.43.55.117/32",
"104.43.55.166/32",
"104.43.55.167/32",
"108.141.83.74/32",
"128.203.132.152/32",
"128.203.181.79/32",
"128.203.214.241/32",
"128.203.236.110/32",
"128.203.82.154/32",
"13.86.35.212/32",
"13.89.106.77/32",
"13.90.178.139/32",
"130.107.228.224/32",
"131.145.39.118/32",
"131.189.183.241/32",
"132.164.209.198/32",
"132.196.106.146/32",
"132.220.1.71/32",
"134.33.207.189/32",
"135.171.251.169/32",
"135.233.63.237/32",
"135.234.221.112/32",
"135.235.183.95/32",
"172.168.115.250/32",
"172.168.12.93/32",
"172.168.133.76/32",
"172.168.137.102/32",
"172.168.152.88/32",
"172.168.173.230/32",
"172.168.190.186/32",
"172.168.227.120/32",
"172.168.254.119/32",
"172.168.43.43/32",
"172.168.51.216/32",
"172.168.53.53/32",
"172.168.81.217/32",
"172.168.94.160/32",
"172.169.164.109/32",
"172.169.17.165/32",
"172.169.177.131/32",
"172.169.181.237/32",
"172.169.28.184/32",
"172.169.36.43/32",
"172.169.53.208/32",
"172.169.60.134/32",
"172.169.7.33/32",
"172.169.93.148/32",
"172.171.37.221/32",
"172.179.105.87/32",
"172.179.128.118/32",
"172.188.250.220/32",
"172.189.40.18/32",
"172.193.184.42/32",
"172.193.197.146/32",
"172.193.245.229/32",
"172.194.141.49/32",
"172.199.55.212/32",
"172.210.111.68/32",
"191.233.3.197/32",
"191.233.3.202/32",
"191.234.216.178/32",
"191.234.216.4/32",
"191.235.201.214/32",
"191.235.202.38/32",
"191.235.202.48/32",
"20.100.136.36/32",
"20.100.140.155/32",
"20.101.17.173/32",
"20.102.34.205/32",
"20.109.129.162/32",
"20.109.130.190/32",
"20.109.241.18/32",
"20.112.58.44/32",
"20.113.14.159/32",
"20.113.3.121/32",
"20.115.181.187/32",
"20.118.11.251/32",
"20.119.127.146/32",
"20.12.141.99/32",
"20.12.249.123/32",
"20.12.252.21/32",
"20.121.75.32/32",
"20.13.44.19/32",
"20.15.142.178/32",
"20.166.171.150/32",
"20.170.75.54/32",
"20.175.232.228/32",
"20.185.79.15/32",
"20.185.79.47/32",
"20.190.21.172/32",
"20.191.44.119/32",
"20.191.44.16/32",
"20.191.44.22/32",
"20.191.44.234/32",
"20.191.45.212/32",
"20.193.12.126/32",
"20.193.24.10/32",
"20.193.24.251/32",
"20.193.25.197/32",
"20.193.27.215/32",
"20.193.45.113/32",
"20.195.108.47/32",
"20.195.58.189/32",
"20.197.209.11/32",
"20.197.209.27/32",
"20.198.132.126/32",
"20.198.152.208/32",
"20.201.15.208/32",
"20.204.228.200/32",
"20.204.240.172/32",
"20.204.241.148/32",
"20.204.242.101/32",
"20.204.242.19/32",
"20.204.243.55/32",
"20.204.246.254/32",
"20.204.246.81/32",
"20.207.107.181/32",
"20.207.117.66/32",
"20.207.72.11/32",
"20.207.72.110/32",
"20.207.72.113/32",
"20.207.72.21/32",
"20.207.97.190/32",
"20.207.99.197/32",
"20.212.139.34/32",
"20.212.90.107/32",
"20.216.200.223/32",
"20.219.43.246/32",
"20.219.45.190/32",
"20.219.45.67/32",
"20.221.10.115/32",
"20.221.84.185/32",
"20.226.133.105/32",
"20.23.88.231/32",
"20.231.254.179/32",
"20.232.249.95/32",
"20.232.51.46/32",
"20.236.195.196/32",
"20.236.198.192/32",
"20.236.227.113/32",
"20.236.233.23/32",
"20.236.247.180/32",
"20.236.40.93/32",
"20.237.1.51/32",
"20.241.135.19/32",
"20.241.147.151/32",
"20.241.153.203/32",
"20.241.164.5/32",
"20.241.241.100/32",
"20.242.133.53/32",
"20.242.135.165/32",
"20.242.211.139/32",
"20.242.215.53/32",
"20.242.237.240/32",
"20.246.132.168/32",
"20.246.225.34/32",
"20.247.235.11/32",
"20.250.51.113/32",
"20.252.16.193/32",
"20.252.41.18/32",
"20.252.63.162/32",
"20.252.88.10/32",
"20.252.9.6/32",
"20.253.114.51/32",
"20.253.59.76/32",
"20.253.96.199/32",
"20.29.149.44/32",
"20.29.154.56/32",
"20.3.1.178/32",
"20.3.60.212/32",
"20.40.133.240/32",
"20.40.147.172/32",
"20.43.150.85/32",
"20.43.150.93/32",
"20.43.152.28/32",
"20.43.172.120/32",
"20.44.222.1/32",
"20.49.129.236/32",
"20.49.136.28/32",
"20.50.168.91/32",
"20.50.48.159/32",
"20.50.48.192/32",
"20.50.49.0/32",
"20.50.49.237/32",
"20.50.49.25/32",
"20.50.49.40/32",
"20.50.49.55/32",
"20.50.50.118/32",
"20.50.50.121/32",
"20.50.50.123/32",
"20.50.50.130/32",
"20.50.50.134/32",
"20.50.50.145/32",
"20.50.50.146/32",
"20.50.50.163/32",
"20.50.50.46/32",
"20.53.134.160/32",
"20.53.78.106/32",
"20.53.78.123/32",
"20.53.78.138/32",
"20.53.78.144/32",
"20.53.78.236/32",
"20.53.91.2/32",
"20.53.92.211/32",
"20.54.224.39/32",
"20.56.197.58/32",
"20.56.197.63/32",
"20.59.0.65/32",
"20.61.142.192/32",
"20.61.34.40/32",
"20.62.224.44/32",
"20.64.185.45/32",
"20.69.129.205/32",
"20.69.131.45/32",
"20.69.138.149/32",
"20.69.138.189/32",
"20.69.144.137/32",
"20.69.146.171/32",
"20.69.147.21/32",
"20.69.171.204/32",
"20.69.74.250/32",
"20.71.12.143/32",
"20.71.69.210/32",
"20.72.197.110/32",
"20.72.203.250/32",
"20.72.218.29/32",
"20.72.221.33/32",
"20.72.221.45/32",
"20.72.225.13/32",
"20.72.242.209/32",
"20.72.242.93/32",
"20.73.132.240/32",
"20.73.202.147/32",
"20.75.144.152/32",
"20.75.254.61/32",
"20.77.146.108/32",
"20.79.226.26/32",
"20.79.238.198/32",
"20.79.239.66/32",
"20.8.252.26/32",
"20.80.114.203/32",
"20.80.120.41/32",
"20.80.129.80/32",
"20.80.135.7/32",
"20.80.156.88/32",
"20.81.115.13/32",
"20.82.24.207/32",
"20.84.138.61/32",
"20.84.219.45/32",
"20.84.251.223/32",
"20.85.138.75/32",
"20.85.167.56/32",
"20.85.187.146/32",
"20.88.168.65/32",
"20.9.114.242/32",
"20.93.28.24/32",
"20.93.49.96/32",
"20.99.154.113/32",
"20.99.255.235/32",
"4.144.148.23/32",
"4.144.182.50/32",
"4.144.227.142/32",
"4.144.251.39/32",
"4.149.142.7/32",
"4.149.159.102/32",
"4.149.159.71/32",
"4.149.216.220/32",
"4.150.142.218/32",
"4.154.193.167/32",
"4.156.136.79/32",
"4.156.154.107/32",
"4.156.30.123/32",
"4.157.77.76/32",
"4.172.49.103/32",
"4.177.16.196/32",
"4.182.10.198/32",
"4.182.131.108/32",
"4.195.133.120/32",
"4.201.125.59/32",
"4.201.141.71/32",
"4.201.197.203/32",
"4.201.206.133/32",
"4.201.220.8/32",
"4.207.220.92/32",
"4.209.224.56/32",
"4.213.46.14/32",
"4.226.40.135/32",
"4.228.76.163/32",
"4.237.155.54/32",
"4.237.244.80/32",
"4.242.119.13/32",
"4.246.10.252/32",
"4.248.234.26/32",
"4.249.216.104/32",
"4.249.223.51/32",
"4.249.238.11/32",
"4.254.123.56/32",
"4.255.35.121/32",
"4.255.39.80/32",
"40.114.182.153/32",
"40.114.182.172/32",
"40.114.182.45/32",
"40.114.183.196/32",
"40.114.183.251/32",
"40.114.183.88/32",
"40.119.232.146/32",
"40.119.232.215/32",
"40.119.232.218/32",
"40.119.232.251/32",
"40.119.232.50/32",
"40.121.121.54/32",
"40.127.154.196/32",
"40.64.105.247/32",
"40.64.106.11/32",
"40.64.106.152/32",
"40.64.67.220/32",
"40.64.78.45/32",
"40.64.88.145/32",
"40.65.78.86/32",
"40.76.162.17/32",
"40.76.162.191/32",
"40.76.162.208/32",
"40.76.162.247/32",
"40.76.163.23/32",
"40.76.163.7/32",
"40.76.173.151/32",
"40.81.250.205/32",
"40.82.218.203/32",
"40.88.195.62/32",
"40.88.21.235/32",
"40.89.243.175/32",
"48.192.16.86/32",
"48.192.74.178/32",
"48.200.33.227/32",
"48.202.155.200/32",
"48.202.171.124/32",
"48.204.32.87/32",
"48.206.18.152/32",
"48.217.129.210/32",
"48.217.212.89/32",
"48.217.23.236/32",
"48.223.209.110/32",
"51.104.144.31/32",
"51.104.146.225/32",
"51.104.146.235/32",
"51.104.160.167/32",
"51.104.160.177/32",
"51.104.161.32/32",
"51.104.162.149/32",
"51.104.163.250/32",
"51.104.164.109/32",
"51.104.164.147/32",
"51.104.164.189/32",
"51.104.164.215/32",
"51.104.166.111/32",
"51.104.167.104/32",
"51.104.167.110/32",
"51.104.167.19/32",
"51.104.167.52/32",
"51.104.167.54/32",
"51.104.167.61/32",
"51.104.167.71/32",
"51.104.167.87/32",
"51.104.167.88/32",
"51.104.167.95/32",
"51.104.167.96/32",
"51.104.180.26/32",
"51.104.180.47/32",
"51.104.180.53/32",
"51.107.40.209/32",
"51.116.131.221/32",
"51.120.48.122/32",
"51.138.90.161/32",
"51.138.90.206/32",
"51.138.90.233/32",
"51.143.114.45/32",
"51.145.185.16/32",
"51.8.253.152/32",
"51.8.31.28/32",
"51.8.71.117/32",
"52.142.24.149/32",
"52.142.26.175/32",
"52.143.241.111/32",
"52.143.242.6/32",
"52.143.243.117/32",
"52.143.244.81/32",
"52.143.247.235/32",
"52.143.95.162/32",
"52.143.95.204/32",
"52.146.57.51/32",
"52.146.58.236/32",
"52.146.59.12/32",
"52.146.59.154/32",
"52.146.59.156/32",
"52.146.63.80/32",
"52.148.161.87/32",
"52.148.165.38/32",
"52.149.25.43/32",
"52.149.28.18/32",
"52.149.28.83/32",
"52.149.30.45/32",
"52.149.56.151/32",
"52.149.58.139/32",
"52.149.58.173/32",
"52.149.58.27/32",
"52.149.58.69/32",
"52.149.60.38/32",
"52.149.61.51/32",
"52.154.169.200/32",
"52.154.169.50/32",
"52.154.170.113/32",
"52.154.170.117/32",
"52.154.170.122/32",
"52.154.170.209/32",
"52.154.170.229/32",
"52.154.170.243/32",
"52.154.170.26/32",
"52.154.170.28/32",
"52.154.170.88/32",
"52.154.170.96/32",
"52.154.171.0/32",
"52.154.171.150/32",
"52.154.171.196/32",
"52.154.171.205/32",
"52.154.171.235/32",
"52.154.171.250/32",
"52.154.171.44/32",
"52.154.171.70/32",
"52.154.171.87/32",
"52.154.172.2/32",
"52.154.204.93/32",
"52.154.60.82/32",
"52.185.93.208/32",
"52.186.37.211/32",
"52.188.89.106/32",
"52.189.70.103/32",
"52.190.37.160/32",
"52.191.222.158/32",
"52.191.82.242/32",
"52.224.147.21/32",
"52.224.16.221/32",
"52.224.16.229/32",
"52.224.19.152/32",
"52.224.191.245/32",
"52.224.20.174/32",
"52.224.20.181/32",
"52.224.20.186/32",
"52.224.20.190/32",
"52.224.20.193/32",
"52.224.20.203/32",
"52.224.20.204/32",
"52.224.20.223/32",
"52.224.20.227/32",
"52.224.20.249/32",
"52.224.21.19/32",
"52.224.21.20/32",
"52.224.21.23/32",
"52.224.21.27/32",
"52.224.21.4/32",
"52.224.21.49/32",
"52.224.21.51/32",
"52.224.21.53/32",
"52.224.21.55/32",
"52.224.21.61/32",
"52.242.224.168/32",
"52.250.46.221/32",
"57.151.44.140/32",
"57.152.17.237/32",
"57.152.62.107/32",
"57.152.72.128/32",
"57.155.18.44/32",
"57.165.205.172/32",
"57.167.88.38/32",
"64.236.118.43/32",
"64.236.15.33/32",
"64.236.93.100/32",
"68.219.152.220/32",
"72.144.95.185/32",
"74.163.169.106/32",
"74.163.208.206/32",
"74.179.200.126/32",
"74.179.232.116/32",
"74.242.194.48/32",
"98.70.219.37/32"
],
"source": "https://duckduckgo.com/duckduckbot.json",
"verifies": "duckduckbot"
},
{
"group": "ai",
"kind": "ai-training",
"name": "GPTBot",
"prefixes": [
"132.196.86.0/24",
"172.182.202.0/25",
"172.182.207.0/25",
"20.125.66.80/28",
"20.171.206.0/24",
"20.171.207.0/24",
"4.227.36.0/25",
"52.230.152.0/24",
"74.7.175.128/25",
"74.7.227.0/25",
"74.7.227.128/25",
"74.7.228.0/25",
"74.7.230.0/25",
"74.7.241.0/25",
"74.7.241.128/25",
"74.7.242.0/25",
"74.7.243.128/25",
"74.7.244.0/25"
],
"source": "https://openai.com/gptbot.json",
"verifies": "gptbot"
},
{
"group": "ai",
"kind": "ai-search",
"name": "OAI-SearchBot",
"prefixes": [
"104.210.140.128/28",
"13.66.216.176/28",
"135.234.64.0/24",
"172.182.193.224/28",
"172.182.193.80/28",
"172.182.194.144/28",
"172.182.194.32/28",
"172.182.195.48/28",
"172.182.209.208/28",
"172.182.211.192/28",
"172.182.213.192/28",
"172.182.224.0/28",
"172.203.190.112/28",
"172.203.190.128/28",
"172.203.190.80/28",
"20.14.99.96/28",
"20.168.18.32/28",
"20.169.6.224/28",
"20.169.7.48/28",
"20.169.77.0/25",
"20.171.123.64/28",
"20.171.53.224/28",
"20.25.151.224/28",
"20.42.10.176/28",
"23.102.145.48/28",
"4.227.36.0/25",
"40.67.175.0/25",
"40.90.214.16/28",
"51.8.102.0/24",
"74.7.175.128/25",
"74.7.228.0/25",
"74.7.228.128/25",
"74.7.229.0/25",
"74.7.229.128/25",
"74.7.230.0/25",
"74.7.241.128/25",
"74.7.242.128/25",
"74.7.243.0/25",
"74.7.244.0/25"
],
"source": "https://openai.com/searchbot.json",
"verifies": "oai-searchbot"
},
{
"group": "ai",
"kind": "ai-assistant",
"name": "ChatGPT-User",
"prefixes": [
"104.208.184.192/28",
"104.210.139.192/28",
"104.210.139.224/28",
"128.85.198.32/28",
"13.65.138.112/28",
"13.67.72.16/28",
"13.71.2.208/28",
"13.76.116.80/28",
"13.76.32.208/28",
"13.83.167.128/28",
"13.83.237.176/28",
"132.196.82.48/28",
"134.138.52.128/28",
"134.138.52.16/28",
"134.138.52.64/28",
"134.138.52.96/28",
"134.138.57.176/28",
"134.138.57.64/28",
"134.138.57.80/28",
"134.138.58.32/28",
"134.149.233.80/28",
"134.33.102.192/28",
"135.116.136.160/28",
"135.13.64.240/28",
"135.220.73.208/28",
"135.220.73.240/28",
"135.222.99.32/28",
"135.237.131.208/28",
"135.237.133.48/28",
"137.135.191.176/28",
"138.91.46.96/28",
"145.132.136.96/28",
"158.158.5.32/28",
"168.63.252.240/28",
"172.162.248.64/28",
"172.170.1.80/28",
"172.170.225.0/28",
"172.170.241.80/28",
"172.172.206.48/28",
"172.175.152.224/28",
"172.178.140.144/28",
"172.178.141.112/28",
"172.178.141.128/28",
"172.183.143.224/28",
"172.183.222.128/28",
"172.192.112.208/28",
"172.197.160.192/28",
"172.197.203.16/28",
"172.198.161.128/28",
"172.198.58.176/28",
"172.198.78.192/28",
"172.198.79.112/28",
"172.199.137.80/28",
"172.204.28.224/28",
"172.204.96.80/28",
"172.205.189.192/28",
"172.206.38.240/28",
"172.207.1.32/28",
"172.212.172.160/28",
"172.215.215.32/28",
"191.232.238.96/28",
"191.233.197.0/28",
"191.233.199.160/28",
"191.234.167.144/28",
"191.235.99.160/28",
"191.237.249.64/28",
"20.102.212.144/28",
"20.113.211.112/28",
"20.113.225.112/28",
"20.125.112.224/28",
"20.125.144.144/28",
"20.161.75.208/28",
"20.168.7.192/28",
"20.169.73.32/28",
"20.169.73.64/28",
"20.169.78.128/28",
"20.169.78.160/28",
"20.169.78.192/28",
"20.169.78.48/28",
"20.169.78.64/28",
"20.169.86.224/28",
"20.169.86.240/28",
"20.170.184.16/28",
"20.170.184.32/28",
"20.170.184.48/28",
"20.170.184.64/28",
"20.170.184.80/28",
"20.172.29.32/28",
"20.199.211.160/28",
"20.199.242.0/28",
"20.200.212.240/28",
"20.206.101.192/28",
"20.210.211.192/28",
"20.212.62.208/28",
"20.215.187.208/28",
"20.215.219.128/28",
"20.215.219.160/28",
"20.215.219.208/28",
"20.218.30.240/28",
"20.219.161.192/28",
"20.219.184.96/28",
"20.219.71.192/28",
"20.222.36.192/28",
"20.227.140.32/28",
"20.228.106.176/28",
"20.235.87.224/28",
"20.249.63.208/28",
"20.250.136.64/28",
"20.250.6.128/28",
"20.254.201.208/28",
"20.45.178.144/28",
"20.55.229.144/28",
"20.57.199.192/28",
"20.63.180.96/28",
"20.63.221.64/28",
"20.78.130.48/28",
"20.83.243.176/28",
"20.98.18.80/28",
"23.101.217.176/28",
"23.102.140.144/28",
"23.102.141.32/28",
"23.98.142.176/28",
"23.98.186.176/28",
"23.98.186.192/28",
"23.98.186.64/28",
"23.98.186.96/28",
"4.151.119.48/28",
"4.151.71.176/28",
"4.189.118.208/28",
"4.189.119.48/28",
"4.197.115.112/28",
"4.197.22.112/28",
"4.197.64.0/28",
"4.197.64.16/28",
"4.197.64.48/28",
"4.197.64.64/28",
"4.201.232.64/28",
"4.205.128.176/28",
"4.218.24.64/28",
"4.226.200.16/28",
"4.226.226.32/28",
"40.116.73.208/28",
"40.119.36.240/28",
"40.124.161.0/28",
"40.74.200.208/28",
"40.81.234.144/28",
"40.81.67.96/28",
"40.84.221.208/28",
"40.84.221.224/28",
"48.221.184.80/28",
"48.221.184.96/28",
"48.221.40.176/28",
"51.116.2.80/28",
"51.116.221.96/28",
"51.57.0.96/28",
"51.59.24.64/28",
"51.59.24.80/28",
"51.59.48.80/28",
"51.8.155.112/28",
"51.8.155.48/28",
"52.148.129.32/28",
"52.156.77.144/28",
"52.159.227.32/28",
"52.159.249.96/28",
"52.161.49.96/28",
"52.165.212.48/28",
"52.172.129.160/28",
"52.173.219.112/28",
"52.173.219.96/28",
"52.173.221.16/28",
"52.173.221.176/28",
"52.173.221.208/28",
"52.173.234.16/28",
"52.173.234.80/28",
"52.183.217.240/28",
"52.190.137.144/28",
"52.190.137.16/28",
"52.190.139.48/28",
"52.190.142.64/28",
"52.190.190.16/28",
"52.190.251.112/28",
"52.225.75.208/28",
"52.231.30.48/28",
"52.231.34.176/28",
"52.231.39.144/28",
"52.231.50.64/28",
"52.236.94.144/28",
"52.241.146.208/28",
"52.242.132.224/28",
"52.242.132.240/28",
"52.255.109.144/28",
"52.255.109.80/28",
"52.255.109.96/28",
"52.255.111.0/28",
"52.255.111.80/28",
"57.154.174.112/28",
"57.154.175.0/28",
"57.154.187.32/28",
"68.154.28.96/28",
"68.220.57.64/28",
"70.153.189.192/28",
"70.153.32.16/28",
"70.153.32.32/28",
"70.153.87.224/28",
"70.156.152.96/28",
"74.161.200.96/28",
"74.224.217.64/28",
"74.226.253.160/28",
"74.248.37.160/28",
"74.7.35.112/28",
"74.7.35.48/28",
"74.7.36.64/28",
"74.7.36.80/28",
"74.7.36.96/28",
"85.211.128.16/28",
"85.211.128.32/28",
"9.129.0.0/16",
"9.160.128.16/28",
"9.160.128.64/28",
"9.160.163.128/28",
"9.160.34.144/28",
"9.160.36.16/28",
"9.160.96.16/28",
"9.163.101.48/28",
"9.205.128.32/28",
"9.205.128.48/28",
"9.205.25.128/28",
"9.205.30.128/28",
"9.205.30.144/28",
"9.205.30.176/28",
"9.205.8.48/28",
"9.205.8.64/28",
"9.223.181.208/28",
"9.234.96.192/28",
"9.234.97.96/28"
],
"source": "https://openai.com/chatgpt-user.json",
"verifies": "chatgpt-user"
},
{
"group": "ai",
"kind": "ai-search",
"name": "PerplexityBot",
"prefixes": [
"107.20.236.150/32",
"18.210.92.235/32",
"18.97.1.228/32",
"18.97.9.100/32",
"18.97.9.101/32",
"18.97.9.102/32",
"18.97.9.103/32",
"18.97.9.96/32",
"18.97.9.97/32",
"18.97.9.98/32",
"18.97.9.99/32",
"3.222.232.239/32",
"3.224.62.45/32"
],
"source": "https://www.perplexity.ai/perplexitybot.json",
"verifies": "perplexitybot"
},
{
"group": "ai",
"kind": "ai-assistant",
"name": "Perplexity-User",
"prefixes": [
"18.97.43.80/32",
"18.97.43.81/32",
"18.97.43.82/32",
"18.97.43.83/32",
"18.97.43.84/32",
"18.97.43.85/32",
"18.97.43.86/32",
"18.97.43.87/32",
"34.193.163.52/32",
"44.208.221.197/32"
],
"source": "https://www.perplexity.ai/perplexity-user.json",
"verifies": "perplexity-user"
},
{
"group": "ai",
"kind": "ai-training",
"name": "CCBot",
"prefixes": [
"18.97.14.80/29",
"18.97.14.88/30",
"18.97.9.168/29",
"2600:1f28:365:8000::/56",
"3.41.188.32/29"
],
"source": "https://index.commoncrawl.org/ccbot.json",
"verifies": "ccbot"
}
]
}