iscrawl 1.2.0

[{"pattern":"Googlebot\\/","url":"http://www.google.com/bot.html","description":"Google's main web crawling bot for search indexing","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Googlebot-Mobile","description":"Google's legacy mobile crawler for Google Search indexing","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Googlebot-Image","description":"Google's image-specific web crawling bot for image search indexing","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Googlebot-News","description":"Google's news-specific web crawling bot for Google News indexing","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Googlebot-Video","description":"Google's video crawler for video-related Google Search features","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"AdsBot-Google([^-]|$)","url":"https://support.google.com/webmasters/answer/1061943?hl=en","description":"Google's Ads bot for checking web page ad quality","tags":["advertising"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"AdsBot-Google-Mobile","url":"https://support.google.com/adwords/answer/2404197","description":"Google's mobile Ads bot for crawling mobile pages","tags":["advertising"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Feed[fF]etcher-Google","url":"https://support.google.com/webmasters/answer/178852","description":"Google's feed fetcher for RSS and Atom feeds","tags":["feed-reader"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Mediapartners-Google","url":"https://support.google.com/webmasters/answer/1061943?hl=en","description":"Google's Mediapartners bot for AdSense and AdMob crawling","tags":["advertising"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Mediapartners \\(Googlebot\\)","url":"https://support.google.com/webmasters/answer/1061943?hl=en","description":"Google's Mediapartners bot variant for AdSense and AdMob crawling","tags":["advertising"]},{"pattern":"APIs-Google","url":"https://support.google.com/webmasters/answer/1061943?hl=en","description":"Google's APIs bot for crawling API documentation and services","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Google-InspectionTool","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","description":"Google's inspection tool bot for testing and debugging search indexing","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Storebot-Google","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","description":"Google's Storebot for crawling product and e-commerce pages","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"GoogleOther","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","description":"Google's other bots and services for various Google search features","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"bingbot","url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","description":"Microsoft's web crawling bot for Bing search indexing","tags":["search-engine"],"rdns":[".search.msn.com"]},{"pattern":"Slurp","url":"https://help.yahoo.com/kb/search-for-desktop/SLN22600.html?impressions=true","description":"Yahoo's web crawling bot for Yahoo search indexing","tags":["search-engine"],"rdns":[".yahoo.co.jp",".yahoo.com",".yahoo.net"]},{"pattern":"[wW]get","description":"GNU Wget command-line tool for downloading web content","tags":["http-library"]},{"pattern":"LinkedInBot","description":"LinkedIn's bot for crawling professional content and profiles","tags":["social-preview"],"rdns":[".licdn.com",".linkedin.com"]},{"pattern":"Python-urllib","description":"Python's built-in URL library for HTTP requests","tags":["http-library"]},{"pattern":"python-requests","description":"Popular Python HTTP library for making web requests","tags":["http-library"]},{"pattern":"aiohttp","url":"https://docs.aiohttp.org/en/stable/","description":"Asynchronous HTTP client library for Python","tags":["http-library"]},{"pattern":"httpx","url":"https://www.python-httpx.org","description":"Modern Python HTTP client with async support","tags":["http-library"]},{"pattern":"libwww-perl","description":"Perl library for making HTTP requests and web crawling","tags":["http-library"]},{"pattern":"httpunit","description":"Java library for automated web application testing","tags":["http-library"]},{"pattern":"Nutch","description":"Apache Nutch open-source web crawler framework","tags":["search-engine"]},{"pattern":"Go-http-client","url":"https://pkg.go.dev/net/http","description":"Go programming language HTTP client library","tags":["http-library"]},{"pattern":"phpcrawl","url":"http://phpcrawl.cuab.de/","description":"PHP web crawler library for scraping websites","tags":["http-library"]},{"pattern":"msnbot","url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","description":"Microsoft's search engine bot for web indexing","tags":["search-engine"],"rdns":[".search.msn.com"]},{"pattern":"jyxobot","description":"Jyxo search engine bot for web crawling","tags":["search-engine"]},{"pattern":"FAST-WebCrawler","description":"FAST search engine web crawler for indexing","tags":["search-engine"]},{"pattern":"FAST Enterprise Crawler","description":"FAST enterprise-grade web crawler for search","tags":["search-engine"]},{"pattern":"BIGLOTRON","description":"Biglotron search engine web crawler bot","tags":["search-engine"]},{"pattern":"Teoma","url":"http://about.ask.com/en/docs/about/webmasters.shtml","description":"Ask Jeeves Teoma search engine web crawler","tags":["search-engine"],"rdns":[".ask.com",".teoma.com"]},{"pattern":"seekbot","url":"http://www.seekbot.net/bot.html","description":"Seekbot search engine web crawler for indexing","tags":["search-engine"]},{"pattern":"Gigabot","url":"http://www.gigablast.com/spider.html","description":"Gigablast search engine web crawler bot","tags":["search-engine"]},{"pattern":"Gigablast","url":"https://github.com/gigablast/open-source-search-engine","description":"Gigablast open-source search engine crawler","tags":["search-engine"]},{"pattern":"exabot","description":"Exabot search engine web crawler for indexing","tags":["search-engine"]},{"pattern":"ia_archiver","description":"Internet Archive Wayback Machine web crawler","tags":["archiver"],"rdns":[".archive.org",".internetarchive.org"]},{"pattern":"GingerCrawler","description":"Ginger Software's language assistant web crawler","tags":["seo"]},{"pattern":"webmon ","description":"Webmon website monitoring and crawling bot","tags":["monitoring"]},{"pattern":"HTTrack","description":"HTTrack website copier for offline browsing","tags":["http-library"]},{"pattern":"grub-client","description":"Grub search engine web crawler for indexing","tags":["search-engine"],"rdns":[".grub.org"]},{"pattern":"UsineNouvelleCrawler","description":"Usine Nouvelle news site web crawler","tags":["search-engine"]},{"pattern":"antibot","description":"Antibot web crawler for content discovery","tags":["scanner"]},{"pattern":"netresearchserver","description":"Net Research Server web crawler bot","tags":["seo"]},{"pattern":"speedy","description":"Entireweb Speedy Spider web crawler bot","tags":["search-engine"]},{"pattern":"fluffy","description":"Fluffy search engine web crawler bot","tags":["search-engine"]},{"pattern":"findlink","description":"Findlinks web crawler for link discovery","tags":["seo"]},{"pattern":"msrbot","description":"Microsoft Research web crawler bot","tags":["search-engine"]},{"pattern":"panscient","description":"Panscient web crawler for content analysis","tags":["seo"]},{"pattern":"yacybot","description":"YaCy decentralized search engine web crawler","tags":["search-engine"],"rdns":[".yacy.net",".yacybot.net"]},{"pattern":"AISearchBot","description":"AI-powered search engine web crawler bot","tags":["search-engine","ai-crawler"]},{"pattern":"ips-agent","description":"IPS agent web crawler for content indexing","tags":["seo"]},{"pattern":"tagoobot","description":"Tagoo search engine web crawler bot","tags":["search-engine"]},{"pattern":"MJ12bot","description":"Majestic-12 search engine web crawler bot","tags":["seo"],"rdns":[".majestic.com",".majestic12.co.uk"]},{"pattern":"woriobot","description":"Worio search engine web crawler bot","tags":["search-engine"]},{"pattern":"yanga","description":"Yanga search engine web crawler bot","tags":["search-engine"]},{"pattern":"buzzbot","description":"Buzzstream web crawler for link research","tags":["seo"]},{"pattern":"mlbot","description":"Metadata Labs web crawler for analysis","tags":["seo"]},{"pattern":"YandexBot","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex main search engine web crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexImages","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex image search crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexMobileBot","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex mobile search crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexAccessibilityBot","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex accessibility features crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexMedia","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex media content crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexVideo","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex video search crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexBlogs","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex blog content crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexDirect/","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex advertising platform crawler","tags":["advertising"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexMetrika","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex analytics and metrics crawler","tags":["monitoring"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexFavicons","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex favicon fetcher","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexWebmaster","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex webmaster tools crawler","tags":["seo"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexNews","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex news aggregation crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexMarket","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex marketplace product crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexPagechecker","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex page quality checker","tags":["seo"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexCalendar","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex calendar events crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexScreenshotBot","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex page screenshot generator","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexTurbo","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex Turbo pages crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexVideoParser","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex video content parser","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexVerticals","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex vertical search services crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexOntoDB","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex ontology database crawler","tags":["search-engine"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexAntivirus","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex security and antivirus scanner","tags":["scanner"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexImageResizer","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex image resizing and optimization bot","tags":["search-engine"]},{"pattern":"YandexAdNet","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex advertising network crawler","tags":["advertising"]},{"pattern":"YandexDirectDyn","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex Direct dynamic ads crawler","tags":["advertising"]},{"pattern":"YaDirectFetcher","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex Direct ad content fetcher","tags":["advertising"],"rdns":[".yandex.com",".yandex.net",".yandex.ru"]},{"pattern":"YandexForDomain","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex domain-specific search crawler","tags":["search-engine"]},{"pattern":"YandexMobileScreenShotBot","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex mobile page screenshot generator","tags":["search-engine"]},{"pattern":"YandexPartner","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex partner network crawler","tags":["advertising"]},{"pattern":"YandexRCA","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex RCA service crawler","tags":["search-engine"]},{"pattern":"YandexSearchShop","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex shopping search crawler","tags":["search-engine"]},{"pattern":"YandexSitelinks","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex sitelinks generation crawler","tags":["search-engine"]},{"pattern":"YandexSpravBot","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex directory and reference crawler","tags":["search-engine"]},{"pattern":"YandexTracker","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex tracking and analytics crawler","tags":["monitoring"]},{"pattern":"YandexVertis","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex Vertis classified ads crawler","tags":["search-engine"]},{"pattern":"YandexUserproxy","url":"https://yandex.ru/support/webmaster/ru/robot-workings/check-yandex-robots.html","description":"Yandex user proxy service bot","tags":["search-engine"]},{"pattern":"purebot","description":"Pure web crawler for content discovery","tags":["search-engine"]},{"pattern":"Linguee Bot","url":"http://www.linguee.com/bot","description":"Linguee translation web crawler bot","tags":["search-engine"]},{"pattern":"CyberPatrol","url":"https://www.cyberpatrol.com/","description":"CyberPatrol web content filtering bot","tags":["scanner"]},{"pattern":"voilabot","description":"Voila search engine web crawler bot","tags":["search-engine"]},{"pattern":"Baiduspider","url":"http://www.baidu.jp/spider/","description":"Baidu search engine web crawler bot","tags":["search-engine"],"rdns":[".baidu.com",".baidu.jp"]},{"pattern":"citeseerxbot","description":"CiteSeerX academic web crawler bot","tags":["academic"]},{"pattern":"spbot","url":"https://seoprofiler.com","description":"SEO Profiler web crawler for analysis","tags":["seo"]},{"pattern":"twengabot","url":"http://www.twenga.com/bot.html","description":"Twenga shopping web crawler bot","tags":["seo"]},{"pattern":"postrank","url":"https://marketingplatform.google.com/about/analytics/","description":"PostRank web crawler for content ranking","tags":["seo"]},{"pattern":"Turnitin","url":"http://www.turnitin.com","description":"Turnitin plagiarism detection web crawler","tags":["academic"],"rdns":[".turnitin.com"]},{"pattern":"scribdbot","url":"https://www.scribd.com/","description":"Scribd document web crawler bot","tags":["search-engine"]},{"pattern":"page2rss","url":"http://www.page2rss.com","description":"Page2RSS web crawler for RSS conversion","tags":["feed-reader"]},{"pattern":"sitebot","url":"http://www.sitebot.org","description":"Sitebot web crawler for site analysis","tags":["seo"]},{"pattern":"linkdex","url":"https://www.linkdex.com/en-gb/","description":"Linkdex SEO tool web crawler for link analysis","tags":["seo"]},{"pattern":"Adidxbot","url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","description":"Bing's advertising index web crawler bot","tags":["advertising","search-engine"],"rdns":[".search.msn.com"]},{"pattern":"ezooms","url":"http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289","description":"Ezooms search engine web crawler bot","tags":["seo"]},{"pattern":"dotbot","description":"Moz DotBot web crawler for SEO analysis","tags":["seo"],"rdns":[".dotbot.com",".moz.com",".rogerbot.com"]},{"pattern":"Mail\\.RU_Bot","description":"Mail.RU search engine web crawler bot","tags":["search-engine"],"rdns":[".mail.ru"]},{"pattern":"discobot","url":"http://discoveryengine.com/discobot.html","description":"Discovery Engine web crawler for content discovery","tags":["search-engine"]},{"pattern":"heritrix","url":"https://github.com/internetarchive/heritrix3/wiki","description":"Internet Archive's Heritrix web crawler framework","tags":["archiver"],"rdns":[".archive.org",".internetarchive.org"]},{"pattern":"findthatfile","url":"http://www.findthatfile.com/","description":"FindThatFile web crawler for file discovery","tags":["search-engine"]},{"pattern":"europarchive\\.org","description":"European Archive web crawler for preservation","tags":["archiver"]},{"pattern":"NerdByNature\\.Bot","url":"http://www.nerdbynature.net/bot","description":"NerdByNature web crawler for content indexing","tags":["seo"]},{"pattern":"(sistrix|SISTRIX) [cC]rawler","url":"https://www.sistrix.com/tutorials/crawling-errors-in-the-optimizer/","description":"SISTRIX SEO tool web crawler for analysis","tags":["seo"],"rdns":[".sistrix.com",".sistrix.net"]},{"pattern":"Ahrefs(Bot|SiteAudit)","description":"Ahrefs SEO tool web crawler for link analysis","tags":["seo"],"rdns":[".ahrefs.com",".ahrefs.net"]},{"pattern":"fuelbot","description":"Fuel web crawler for content discovery","tags":["seo"]},{"pattern":"CrunchBot","description":"LeadCrunch web crawler for lead generation","tags":["seo"]},{"pattern":"IndeedBot","description":"Indeed job search web crawler bot","tags":["search-engine"]},{"pattern":"mappydata","description":"Mappy web crawler for mapping data","tags":["seo"]},{"pattern":"woobot","description":"Woo web crawler for content analysis","tags":["seo"]},{"pattern":"ZoominfoBot","description":"ZoomInfo web crawler for business intelligence","tags":["seo"]},{"pattern":"PrivacyAwareBot","description":"PrivacyAware web crawler for privacy analysis","tags":["seo"]},{"pattern":"Multiviewbot","description":"Multiview web crawler for content discovery","tags":["seo"]},{"pattern":"SWIMGBot","description":"SWIMG web crawler for image discovery","tags":["seo"]},{"pattern":"Grobbot","description":"Grob web crawler for content analysis","tags":["seo"]},{"pattern":"eright","description":"Eright web crawler for content discovery","tags":["seo"]},{"pattern":"Apercite","description":"Apercite web crawler for content analysis","tags":["seo"]},{"pattern":"semanticbot","description":"Semantic web crawler for content analysis","tags":["seo"]},{"pattern":"Aboundex","url":"http://www.aboundex.com/crawler/","description":"Aboundex web crawler for content discovery","tags":["search-engine"]},{"pattern":"domaincrawler","description":"Domain Crawler web crawler for analysis","tags":["seo"]},{"pattern":"wbsearchbot","url":"https://warebay.com/password","description":"Warebay search web crawler bot","tags":["search-engine"]},{"pattern":"summify","url":"https://summify.com/","description":"Summify web crawler for content summarization","tags":["seo"]},{"pattern":"CCBot","url":"https://www.commoncrawl.org/ccbot","description":"Common Crawl web crawler for indexing","tags":["ai-crawler"],"rdns":[".commoncrawl.org"]},{"pattern":"edisterbot","description":"Edister web crawler for content discovery","tags":["search-engine"]},{"pattern":"SeznamBot","description":"Seznam search engine web crawler bot","tags":["search-engine"],"rdns":[".seznam.cz"]},{"pattern":"ec2linkfinder","description":"EC2 link finder web crawler bot","tags":["scanner"]},{"pattern":"gslfbot","description":"GSLFBOT web crawler for content discovery","tags":["seo"]},{"pattern":"aiHitBot","description":"AiHit web crawler for data collection","tags":["search-engine"]},{"pattern":"intelium_bot","description":"Intelium web crawler for content discovery","tags":["seo"]},{"pattern":"facebookexternalhit","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","description":"Facebook external hit web crawler bot","tags":["social-preview"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"Yeti","url":"http://naver.me/bot","description":"Naver Yeti search engine web crawler bot","tags":["search-engine"],"rdns":[".naver.com",".naver.jp"]},{"pattern":"RetrevoPageAnalyzer","description":"Retrevo page analyzer web crawler bot","tags":["seo"]},{"pattern":"lb-spider","description":"LB spider web crawler for content discovery","tags":["search-engine"]},{"pattern":"Sogou","url":"http://www.sogou.com/docs/help/webmasters.htm#07","description":"Sogou search engine web crawler bot","tags":["search-engine"],"rdns":[".sogou.com"]},{"pattern":"lssbot","url":"https://www.lssbot.com/","description":"LSS web crawler for content discovery","tags":["seo"]},{"pattern":"careerbot","url":"http://www.career-x.de/bot.html","description":"Career-X web crawler for job discovery","tags":["seo"]},{"pattern":"wotbox","url":"http://www.wotbox.com","description":"Wotbox web crawler for content discovery","tags":["search-engine"]},{"pattern":"wocbot","url":"http://www.wocodi.com/crawler","description":"Wocodi web crawler for content discovery","tags":["search-engine"]},{"pattern":"ichiro","url":"http://help.goo.ne.jp/help/article/1142","description":"Goo ichiro search engine web crawler bot","tags":["search-engine"]},{"pattern":"DuckDuckBot","url":"https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot","description":"DuckDuckGo search engine web crawler bot","tags":["search-engine"],"rdns":[".duckduckgo.com"]},{"pattern":"lssrocketcrawler","description":"LSS Rocket web crawler for content","tags":["seo"]},{"pattern":"drupact","url":"http://www.arocom.de/drupact","description":"Drupact web crawler for content discovery","tags":["seo"]},{"pattern":"webcompanycrawler","description":"Web Company web crawler for analysis","tags":["seo"]},{"pattern":"acoonbot","url":"http://www.acoon.de/robot.asp","description":"Acoon web crawler for content discovery","tags":["search-engine"]},{"pattern":"openindexspider","url":"https://www.openindex.io/about-our-spider/","description":"OpenIndex web crawler for indexing","tags":["search-engine"]},{"pattern":"gnam gnam spider","description":"Gnam Gnam web crawler for discovery","tags":["seo"]},{"pattern":"web-archive-net\\.com\\.bot","description":"Web Archive web crawler for preservation","tags":["archiver"]},{"pattern":"backlinkcrawler","url":"http://www.backlinktest.com/crawler.html","description":"Backlink Crawler web crawler for analysis","tags":["seo"]},{"pattern":"coccoc/","url":"https://coccoc.com/search/console","description":"Coccoc search engine web crawler bot","tags":["search-engine"]},{"pattern":"integromedb","url":"http://www.integromedb.org/Crawler","description":"IntegromeDB web crawler for research","tags":["academic"]},{"pattern":"content crawler spider","description":"Content Crawler web crawler for discovery","tags":["seo"]},{"pattern":"toplistbot","description":"TopList web crawler for ranking","tags":["seo"]},{"pattern":"it2media-domain-crawler","description":"IT2Media domain web crawler bot","tags":["seo"]},{"pattern":"ip-web-crawler\\.com","description":"IP Web Crawler web crawler bot","tags":["scanner"]},{"pattern":"siteexplorer\\.info","description":"Site Explorer web crawler for analysis","tags":["seo"]},{"pattern":"elisabot","description":"Elisa web crawler for content discovery","tags":["seo"]},{"pattern":"proximic","url":"https://www.proximic.com/Web-Crawler","description":"Proximic web crawler for content analysis","tags":["advertising"]},{"pattern":"changedetection","url":"https://visualping.io","description":"ChangeDetection web crawler for monitoring","tags":["monitoring"]},{"pattern":"arabot","description":"Arabot web crawler for content discovery","tags":["search-engine"]},{"pattern":"WeSEE:Search","description":"WeSEE search engine web crawler bot","tags":["search-engine"]},{"pattern":"niki-bot","description":"Niki web crawler for content discovery","tags":["seo"]},{"pattern":"CrystalSemanticsBot","url":"http://www.crystalsemantics.com/user-agent/","description":"Crystal Semantics web crawler bot","tags":["seo"]},{"pattern":"rogerbot","url":"https://moz.com/help/moz-procedures/crawlers/rogerbot","description":"Moz RogerBot web crawler for analysis","tags":["seo"],"rdns":[".dotbot.com",".moz.com",".rogerbot.com"]},{"pattern":"360Spider","url":"http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html","description":"360 search engine web crawler bot","tags":["search-engine"],"rdns":[".360.cn",".haosou.com",".qihoo.com",".so.com"]},{"pattern":"psbot","url":"http://www.picsearch.com/bot.html","description":"PicSearch web crawler for image discovery","tags":["search-engine"]},{"pattern":"InterfaxScanBot","url":"https://scan-interfax.ru/","description":"Interfax scan web crawler bot","tags":["seo"]},{"pattern":"CC Metadata Scaper","url":"https://wiki.creativecommons.org/wiki/Metadata_Scraper","description":"Creative Commons metadata web crawler","tags":["archiver"]},{"pattern":"g00g1e\\.net","url":"http://www.g00g1e.net/","description":"G00g1e web crawler for content discovery","tags":["seo"]},{"pattern":"GrapeshotCrawler","url":"https://www.oracle.com/corporate/acquisitions/","description":"Grapeshot web crawler for content analysis","tags":["advertising"],"rdns":[".grapeshot.co.uk",".grapeshot.com"]},{"pattern":"urlappendbot","url":"http://www.profound.net/urlappendbot.html","description":"URL Append web crawler for analysis","tags":["seo"]},{"pattern":"brainobot","description":"Braino web crawler for content discovery","tags":["seo"]},{"pattern":"fr-crawler","description":"FR Crawler web crawler for content discovery","tags":["search-engine"]},{"pattern":"binlar","description":"Binlar web crawler for content discovery","tags":["scanner"]},{"pattern":"SimpleCrawler","description":"Simple Crawler web crawler framework","tags":["http-library"]},{"pattern":"Twitterbot","url":"https://developer.twitter.com/","description":"Twitter web crawler for link previews","tags":["social-preview"],"rdns":[".twitter.com",".twttr.com",".x.com"]},{"pattern":"cXensebot","url":"http://www.cxense.com/bot.html","description":"CXense web crawler for content analysis","tags":["advertising"]},{"pattern":"SMTBot","url":"http://www.similartech.com/smtbot","description":"SimilarTech web crawler for technology detection","tags":["seo"],"rdns":[".smtbot.com"]},{"pattern":"bnf\\.fr_bot","url":"https://www.bnf.fr/fr/capture-de-votre-site-web-par-le-robot-de-la-bnf","description":"BNF French National Library web crawler","tags":["archiver"]},{"pattern":"A6-Indexer","url":"http://www.a6corp.com/a6-web-scraping-policy/","description":"A6 Corporation web crawler for indexing","tags":["search-engine"]},{"pattern":"ADmantX","url":"https://integralads.com","description":"ADmantX semantic analyzer web crawler","tags":["advertising"]},{"pattern":"Facebot","url":"https://developers.facebook.com/docs/sharing/best-practices#crawl","description":"Facebook's web crawler for social sharing","tags":["social-preview"]},{"pattern":"OrangeBot\\/","description":"Orange search engine web crawler bot","tags":["search-engine"]},{"pattern":"memorybot","url":"http://mignify.com/bot.htm","description":"Mignify memory web crawler bot","tags":["seo"]},{"pattern":"AdvBot","url":"http://advbot.net/bot.html","description":"AdvBot web crawler for advertising analysis","tags":["advertising"]},{"pattern":"MegaIndex","url":"https://www.megaindex.ru/create_account.php","description":"MegaIndex SEO tool web crawler for analysis","tags":["seo"]},{"pattern":"SemanticScholarBot","url":"https://www.semanticscholar.org/crawler","description":"Semantic Scholar web crawler for academic content","tags":["academic"]},{"pattern":"ltx71","url":"http://ltx71.com/","description":"LTX71 web crawler for content discovery","tags":["scanner"]},{"pattern":"nerdybot","url":"http://nerdybot.com/","description":"NerdyBot web crawler for content analysis","tags":["seo"]},{"pattern":"xovibot","url":"https://www.xovibot.net/","description":"Xovi SEO tool web crawler for analysis","tags":["seo"],"rdns":[".xovibot.net"]},{"pattern":"BUbiNG","url":"https://law.di.unimi.it/BUbiNG.html","description":"BUbiNG web crawler framework for research","tags":["academic"]},{"pattern":"Qwantify","url":"https://www.qwant.com/","description":"Qwant search engine web crawler bot","tags":["search-engine"],"rdns":[".qwant.com"]},{"pattern":"archive\\.org_bot","url":"http://archive.org/details/archive.org_bot","description":"Archive.org web crawler for preservation","tags":["archiver"],"depends_on":["heritrix"],"rdns":[".archive.org",".internetarchive.org"]},{"pattern":"Applebot","url":"https://support.apple.com/en-us/119829","description":"Apple's web crawler for Siri and search","tags":["search-engine"],"rdns":[".applebot.apple.com"]},{"pattern":"TweetmemeBot","url":"https://www.meltwater.com/en?redirected-from=datasift.com%2Fbot.html","description":"TweetMeme web crawler for social content","tags":["social-preview"]},{"pattern":"crawler4j","url":"https://github.com/yasserg/crawler4j","description":"Crawler4j Java web crawler framework","tags":["http-library"]},{"pattern":"findxbot","url":"http://www.findxbot.com","description":"FindX web crawler for content discovery","tags":["search-engine"]},{"pattern":"S[eE][mM]rushBot","url":"https://www.semrush.com:443/bot/","description":"Semrush SEO tool web crawler for analysis","tags":["seo"],"rdns":[".semrush.com"]},{"pattern":"yoozBot","url":"http://yooz.ir","description":"Yooz web crawler for content analysis","tags":["seo"]},{"pattern":"lipperhey","url":"https://www.dataprovider.com/","description":"Lipperhey SEO tool web crawler for analysis","tags":["seo"]},{"pattern":"Y!J","url":"https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6","description":"Yahoo Japan search engine web crawler bot","tags":["search-engine"],"rdns":[".yahoo.co.jp",".yahoo.com",".yahoo.net"]},{"pattern":"Domain Re-Animator Bot","url":"http://domainreanimator.com","description":"Domain Re-Animator web crawler for domain analysis","tags":["seo"]},{"pattern":"AddThis","url":"https://www.oracle.com/","description":"AddThis social sharing web crawler bot","tags":["social-preview"]},{"pattern":"Screaming Frog SEO Spider","url":"https://www.screamingfrog.co.uk/seo-spider/","description":"Screaming Frog SEO tool web crawler","tags":["seo"],"rdns":[".screamingfrog.co.uk"]},{"pattern":"MetaURI","url":"https://www.useragentstring.com/MetaURI_id_17683.php","description":"MetaURI API web crawler for metadata","tags":["seo"]},{"pattern":"Scrapy","url":"https://www.scrapy.org/","description":"Scrapy Python web crawler framework","tags":["http-library"]},{"pattern":"Livelap[bB]ot","url":"https://site.livelap.com/crawler","description":"Livelap web crawler for content discovery","tags":["seo"]},{"pattern":"OpenHoseBot","url":"http://www.openhose.org/bot.html","description":"OpenHose web crawler for content analysis","tags":["search-engine"]},{"pattern":"CapsuleChecker","url":"https://www.capsulink.com/features","description":"Capsule web crawler for link checking","tags":["monitoring"]},{"pattern":"collection@infegy\\.com","url":"https://www.infegy.com/","description":"Infegy web crawler for social listening","tags":["seo"]},{"pattern":"IstellaBot","url":"http://www.tiscali.it/","description":"Istella web crawler for search indexing","tags":["search-engine"]},{"pattern":"DeuSu\\/","url":"https://deusu.de/robot.html","description":"DeuSu web crawler for search indexing","tags":["search-engine"]},{"pattern":"betaBot","description":"BetaBot web crawler for testing","tags":["seo"]},{"pattern":"Cliqzbot","url":"http://cliqz.com/company/cliqzbot","description":"Cliqz search engine web crawler bot","tags":["search-engine"]},{"pattern":"MojeekBot\\/","url":"https://www.mojeek.com/bot.html","description":"Mojeek search engine web crawler bot","tags":["search-engine"],"rdns":[".mojeek.com"]},{"pattern":"netEstate NE Crawler","url":"http://www.website-datenbank.de/","description":"NetEstate web crawler for domain analysis","tags":["seo"]},{"pattern":"SafeSearch microdata crawler","url":"https://safesearch.avira.com","description":"Avira SafeSearch web crawler for safety","tags":["search-engine"]},{"pattern":"Gluten Free Crawler\\/","url":"http://glutenfreepleasure.com/","description":"Gluten Free Pleasure web crawler bot","tags":["search-engine"]},{"pattern":"Sonic","url":"https://www.yama.info.waseda.ac.jp/~crawler/info.html","description":"Sonic web crawler for ranking analysis","tags":["seo"]},{"pattern":"Sysomos","url":"https://www.meltwater.com/en","description":"Sysomos web crawler for social media analysis","tags":["seo"]},{"pattern":"Trove","url":"http://www.trove.com","description":"Trove web crawler for content discovery","tags":["archiver"]},{"pattern":"deadlinkchecker","url":"https://www.deadlinkchecker.com/","description":"Dead Link Checker web crawler for link validation","tags":["seo"]},{"pattern":"Slack-ImgProxy","url":"https://api.slack.com/robots","description":"Slack's image proxy web crawler bot","tags":["social-preview"],"rdns":[".slack-edge.com",".slack.com"]},{"pattern":"Embedly","url":"https://support.embed.ly/hc/en-us","description":"Embedly web crawler for content embedding","tags":["social-preview"]},{"pattern":"RankActiveLinkBot","url":"https://rankactive.com/resources/rankactive-linkbot","description":"RankActive web crawler for link analysis","tags":["seo"]},{"pattern":"iskanie","url":"http://www.iskanie.com","description":"Iskanie web crawler for content discovery","tags":["seo"]},{"pattern":"SafeDNSBot","url":"https://www.safedns.com/searchbot","description":"SafeDNS web crawler for security analysis","tags":["scanner"]},{"pattern":"SkypeUriPreview","description":"Skype's URI preview web crawler bot","tags":["social-preview"],"rdns":[".skype.com"]},{"pattern":"Veoozbot","url":"http://www.veooz.com/veoozbot.html","description":"Veooz web crawler for marketing analysis","tags":["seo"],"rdns":[".veooz.com"]},{"pattern":"Slackbot","url":"https://api.slack.com/robots","description":"Slack's link expansion web crawler bot","tags":["social-preview"],"rdns":[".slack-edge.com",".slack.com"]},{"pattern":"redditbot","url":"https://www.reddit.com/feedback","description":"Reddit's web crawler for content sharing","tags":["social-preview"],"rdns":[".reddit.com",".redditmedia.com"]},{"pattern":"datagnionbot","url":"http://www.datagnion.com/bot.html","description":"Datagnion web crawler for data analysis","tags":["seo"]},{"pattern":"Google-Adwords-Instant","url":"http://www.google.com/adsbot.html","description":"Google AdWords instant web crawler bot","tags":["advertising"]},{"pattern":"adbeat_bot","description":"AdBeat web crawler for advertising analysis","tags":["advertising"]},{"pattern":"WhatsApp","url":"https://www.whatsapp.com/","description":"WhatsApp's web crawler for link previews","tags":["social-preview"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net",".whatsapp.com",".whatsapp.net"]},{"pattern":"contxbot","description":"Contx web crawler for content discovery","tags":["seo"]},{"pattern":"pinterest\\.com\\/bot","url":"https://help.pinterest.com/en/business/article/pinterestbot","description":"Pinterest web crawler for content discovery","tags":["social-preview"],"rdns":[".pinimg.com",".pinterest.com"]},{"pattern":"electricmonk","url":"https://www.duedil.com/our-crawler/","description":"DueDil electricmonk web crawler bot","tags":["seo"]},{"pattern":"GarlikCrawler","url":"http://garlik.com/","description":"Garlik web crawler for content discovery","tags":["seo"]},{"pattern":"BingPreview\\/","url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","description":"Bing preview web crawler bot","tags":["social-preview"],"rdns":[".search.msn.com"]},{"pattern":"vebidoobot","url":"https://blog.vebidoo.de/vebidoobot/","description":"Vebidoo web crawler for content discovery","tags":["seo"]},{"pattern":"FemtosearchBot","url":"http://femtosearch.com","description":"Femtosearch web crawler for content discovery","tags":["search-engine"]},{"pattern":"Yahoo Link Preview","url":"https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html","description":"Yahoo link preview web crawler bot","tags":["social-preview"],"rdns":[".yahoo.co.jp",".yahoo.com",".yahoo.net"]},{"pattern":"MetaJobBot","url":"https://www.metajob.de/the/crawler","description":"MetaJob web crawler for job discovery","tags":["seo"]},{"pattern":"DomainStatsBot","url":"http://domainstats.io/our-bot","description":"DomainStats web crawler for domain analysis","tags":["seo"]},{"pattern":"mindUpBot","url":"https://www.datenbutler.de/","description":"MindUp web crawler for content discovery","tags":["seo"]},{"pattern":"Daum\\/","url":"http://cs.daum.net/faq/15/4118.html?faqId=28966","description":"Daum search engine web crawler bot","tags":["search-engine"]},{"pattern":"Jugendschutzprogramm-Crawler","url":"https://www.jugendschutzprogramm.de/","description":"Jugendschutzprogramm web crawler bot","tags":["scanner"]},{"pattern":"Xenu Link Sleuth","url":"http://home.snafu.de/tilman/xenulink.html","description":"Xenu link checker web crawler tool","tags":["seo"]},{"pattern":"Pcore-HTTP","url":"https://bitbucket.org/softvisio/pcore/overview","description":"Pcore HTTP web crawler library","tags":["http-library"]},{"pattern":"moatbot","url":"https://moat.com","description":"Moat web crawler for advertising analysis","tags":["advertising"]},{"pattern":"KosmioBot","url":"https://kosm.io/bot.html","description":"Kosmio web crawler for content discovery","tags":["seo"]},{"pattern":"[pP]ingdom","url":"http://www.pingdom.com","description":"Pingdom website monitoring web crawler","tags":["monitoring"],"rdns":[".pingdom.com",".pingdom.net"]},{"pattern":"AppInsights","url":"https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview","description":"Microsoft AppInsights web crawler bot","tags":["monitoring"]},{"pattern":"PhantomJS","url":"https://phantomjs.org/","description":"PhantomJS headless browser web crawler","tags":["browser-automation"]},{"pattern":"Gowikibot","url":"http://www.gowikibot.com","description":"GoWiki web crawler for content discovery","tags":["seo"]},{"pattern":"PiplBot","url":"https://pipl.com/bot","description":"Pipl web crawler for people search","tags":["seo"]},{"pattern":"Discordbot","url":"https://discord.com/","description":"Discord web crawler for link previews","tags":["social-preview"],"rdns":[".discord.com",".discordapp.com"]},{"pattern":"TelegramBot","description":"Telegram web crawler for link previews","tags":["social-preview"],"rdns":[".t.me",".telegram.org"]},{"pattern":"Jetslide","url":"http://jetsli.de/crawler","description":"Jetslide web crawler for content discovery","tags":["social-preview"]},{"pattern":"newsharecounts","url":"http://newsharecounts.com/crawler","description":"NewShareCounts web crawler for sharing","tags":["social-preview"]},{"pattern":"Bark[rR]owler","url":"http://www.exensa.com/crawl","description":"Barkrowler web crawler for content discovery","tags":["seo"]},{"pattern":"TinEye","url":"http://www.tineye.com/crawler.html","description":"TinEye reverse image search web crawler","tags":["seo"]},{"pattern":"SocialRankIOBot","url":"http://socialrank.io/about","description":"SocialRank web crawler for social analysis","tags":["seo"]},{"pattern":"trendictionbot","url":"https://www.trendiction.com/bot","description":"Trendiction web crawler for trend analysis","tags":["seo"]},{"pattern":"Ocarinabot","description":"Ocarina web crawler for content discovery","tags":["seo"]},{"pattern":"epicbot","url":"http://www.epictions.com/epicbot","description":"Epic web crawler for content discovery","tags":["seo"]},{"pattern":"Primalbot","url":"https://www.primal.com","description":"Primal web crawler for content discovery","tags":["seo"]},{"pattern":"DuckDuckGo-Favicons-Bot","url":"http://duckduckgo.com","description":"DuckDuckGo favicon web crawler bot","tags":["search-engine"],"rdns":[".duckduckgo.com"]},{"pattern":"GnowitNewsbot","url":"https://www.gnowit.com/","description":"Gnowit news web crawler bot","tags":["seo"]},{"pattern":"Leikibot","url":"http://www.leiki.com","description":"Leiki web crawler for content discovery","tags":["seo"]},{"pattern":"LinkArchiver","url":"https://github.com/thisisparker/linkarchiver","description":"LinkArchiver web crawler for archiving","tags":["archiver"]},{"pattern":"YaK\\/","url":"https://www.meltwater.com/en/use-case/consumer-intelligence?utm_source=direct&utm_medium=linkfluence","description":"Linkfluence YaK web crawler bot","tags":["seo"],"rdns":[".linkfluence.com",".radarly.com"]},{"pattern":"PaperLiBot","url":"http://support.paper.li/entries/20023257-what-is-paper-li","description":"Paper.li web crawler for content curation","tags":["seo"]},{"pattern":"Digg Deeper","url":"http://digg.com/about","description":"Digg web crawler for content discovery","tags":["seo"]},{"pattern":"dcrawl","url":"https://github.com/kgretzky/dcrawl","description":"dcrawl web crawler framework","tags":["seo"]},{"pattern":"Snacktory","url":"https://github.com/karussell/snacktory","description":"Snacktory web crawler for content extraction","tags":["seo"]},{"pattern":"AndersPinkBot","url":"https://www.go1.com/","description":"AndersPink web crawler for content discovery","tags":["seo"]},{"pattern":"Fyrebot","description":"Fyre web crawler for content discovery","tags":["seo"]},{"pattern":"EveryoneSocialBot","url":"https://everyonesocial.com/","description":"EveryoneSocial web crawler for sharing","tags":["social-preview"]},{"pattern":"Mediatoolkitbot","url":"https://determ.com/","description":"Mediatoolkit web crawler for monitoring","tags":["monitoring"]},{"pattern":"Luminator-robots","description":"Luminator web crawler for content discovery","tags":["seo"]},{"pattern":"ExtLinksBot","url":"https://extlinks.com/Bot.html","description":"ExtLinks web crawler for link analysis","tags":["seo"]},{"pattern":"SurveyBot","description":"DomainTools survey web crawler bot","tags":["seo"]},{"pattern":"NING\\/","description":"Ning web crawler for content discovery","tags":["social-preview"]},{"pattern":"okhttp","description":"OkHttp Java HTTP client library","tags":["http-library"]},{"pattern":"Nuzzel","description":"Nuzzel web crawler for news discovery","tags":["seo"]},{"pattern":"omgili","url":"http://omgili.com","description":"Omgili web crawler for content discovery","tags":["seo"],"rdns":[".omgili.com",".webz.io"]},{"pattern":"PocketParser","url":"https://getpocket.com/pocketparser_ua","description":"Pocket web crawler for content parsing","tags":["seo"],"rdns":[".getpocket.com",".pocket.com"]},{"pattern":"YisouSpider","description":"Yisou search engine web crawler bot","tags":["search-engine"],"rdns":[".sm.cn",".yisou.com"]},{"pattern":"um-LN","description":"Ubermetrics web crawler for monitoring","tags":["seo"]},{"pattern":"ToutiaoSpider","url":"https://www.toutiao.com/media_cooperation/","description":"Toutiao news platform web crawler bot","tags":["search-engine"],"rdns":[".toutiao.com"]},{"pattern":"MuckRack","url":"https://muckrack.com/","description":"MuckRack journalist database web crawler","tags":["seo"]},{"pattern":"Jamie's Spider","url":"https://uk.linkedin.com/in/jamiebrown","description":"Jamie Brown's personal web crawler bot","tags":["seo"]},{"pattern":"AHC\\/","url":"https://github.com/AsyncHttpClient/async-http-client","description":"Async HTTP Client Java library for HTTP requests","tags":["http-library"]},{"pattern":"NetcraftSurveyAgent","description":"Netcraft survey web crawler bot","tags":["scanner"]},{"pattern":"Laserlikebot","description":"Laserlike web crawler for content discovery","tags":["seo"]},{"pattern":"^Apache-HttpClient","description":"Apache HTTP Client Java library for requests","tags":["http-library"],"rdns":[".nutch.apache.org"]},{"pattern":"AppEngine-Google","description":"Google App Engine web crawler bot","tags":["search-engine"]},{"pattern":"Jetty","description":"Jetty web server HTTP client library","tags":["http-library"]},{"pattern":"Upflow","description":"Upflow web crawler for content analysis","tags":["seo"]},{"pattern":"Thinklab","url":"https://thinklab.com","description":"Thinklab web crawler for research","tags":["academic"]},{"pattern":"Traackr\\.com","url":"https://www.traackr.com/","description":"Traackr influencer marketing web crawler","tags":["seo"]},{"pattern":"Twurly","url":"https://www.thaicrystal.restaurant/our-menu/","description":"Twurly Ruby web crawler framework","tags":["seo"]},{"pattern":"Mastodon","description":"Mastodon social network web crawler bot","tags":["social-preview"]},{"pattern":"http_get","description":"HTTP GET command-line tool for requests","tags":["http-library"]},{"pattern":"DnyzBot","description":"Dnyz web crawler for content discovery","tags":["seo"]},{"pattern":"botify","description":"Botify SEO tool web crawler for analysis","tags":["seo"],"rdns":[".botify.com"]},{"pattern":"007ac9 Crawler","description":"007ac9 web crawler for content discovery","tags":["seo"]},{"pattern":"BehloolBot","description":"Behlool web crawler for content analysis","tags":["seo"]},{"pattern":"BrandVerity","url":"https://www.brandverity.com/why-is-brandverity-visiting-me","description":"BrandVerity web crawler for brand monitoring","tags":["seo"]},{"pattern":"check_http","description":"Nagios check_http monitoring plugin","tags":["monitoring"]},{"pattern":"BDCbot","description":"BigDataCorp web crawler for data collection","tags":["seo"]},{"pattern":"ZumBot","description":"Zum search engine web crawler bot","tags":["search-engine"]},{"pattern":"EZID","description":"EZID link checker web crawler bot","tags":["seo"]},{"pattern":"ICC-Crawler","url":"https://ucri.nict.go.jp/en/icccrawler.html","description":"ICC web crawler for language research","tags":["academic"]},{"pattern":"ArchiveBot","url":"https://github.com/ArchiveTeam/ArchiveBot","description":"Archive Team web crawler for preservation","tags":["archiver"]},{"pattern":"^LCC ","url":"http://corpora.informatik.uni-leipzig.de/crawler_faq.html","description":"Leipzig Corpora Collection web crawler","tags":["academic"]},{"pattern":"filterdb\\.iss\\.net\\/crawler","url":"http://filterdb.iss.net/crawler/","description":"ISS filter database web crawler","tags":["scanner"]},{"pattern":"BLP_bbot","description":"BLP web crawler for content discovery","tags":["seo"]},{"pattern":"BomboraBot","url":"https://bombora.com/bot/","description":"Bombora web crawler for business intelligence","tags":["seo"]},{"pattern":"Buck\\/","url":"https://app.hypefactors.com/media-monitoring/about.html","description":"Hypefactors media monitoring web crawler","tags":["monitoring"]},{"pattern":"Companybook-Crawler","url":"https://www.companybooknetworking.com/","description":"Companybook networking web crawler","tags":["seo"]},{"pattern":"Genieo","url":"https://www.genieo.com/webfilter.html","description":"Genieo web filter web crawler bot","tags":["seo"]},{"pattern":"magpie-crawler","url":"https://www.brandwatch.com/","description":"Brandwatch magpie web crawler bot","tags":["seo"]},{"pattern":"MeltwaterNews","url":"https://www.meltwater.com/en","description":"Meltwater news monitoring web crawler","tags":["seo"]},{"pattern":"Moreover","url":"http://www.moreover.com","description":"Moreover news aggregation web crawler","tags":["seo"]},{"pattern":"newspaper\\/","description":"Newspaper Python web scraping library","tags":["http-library"]},{"pattern":"ScoutJet","url":"http://www.scoutjet.com/","description":"ScoutJet web crawler for content discovery","tags":["seo"]},{"pattern":"(^| )sentry\\/","url":"https://sentry.io/welcome/","description":"Sentry error tracking web crawler bot","tags":["monitoring"],"rdns":[".sentry.dev",".sentry.io"]},{"pattern":"StorygizeBot","url":"https://www.storygize.com/","description":"Storygize web crawler for content analysis","tags":["seo"]},{"pattern":"UptimeRobot","url":"https://www.uptimerobot.com/","description":"UptimeRobot website monitoring web crawler","tags":["monitoring"],"rdns":[".uptimerobot.com"]},{"pattern":"OutclicksBot","url":"https://www.sovrn.com/commerce/","description":"Outclicks web crawler for link tracking","tags":["advertising"]},{"pattern":"seoscanners","url":"https://github.com/monperrus/crawler-user-agents/issues/384#issuecomment-2575367162","description":"SEO Scanners web crawler for analysis","tags":["seo"]},{"pattern":"Hatena","description":"Hatena web services web crawler bot","tags":["social-preview"]},{"pattern":"Google Web Preview","description":"Google web preview web crawler bot","tags":["social-preview"]},{"pattern":"MauiBot","description":"Maui web crawler for content discovery","tags":["seo"]},{"pattern":"AlphaBot","url":"http://alphaseobot.com/bot.html","description":"AlphaBot SEO web crawler for analysis","tags":["seo"]},{"pattern":"SBL-BOT","url":"http://sbl.net","description":"Bot of SoftByte BlackWidow","tags":["scanner"]},{"pattern":"IAS crawler","url":"https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/","description":"Bot of Integral Ad Science, Inc.","tags":["advertising"]},{"pattern":"adscanner","description":"AdScanner web crawler for ad analysis","tags":["advertising"]},{"pattern":"Netvibes","url":"https://www.3ds.com/products/netvibes/","description":"Netvibes web crawler for content aggregation","tags":["feed-reader"]},{"pattern":"acapbot","description":"ACAP web crawler for content analysis","tags":["seo"]},{"pattern":"Baidu-YunGuanCe","url":"https://ce.baidu.com/topic/topic20150908","description":"Baidu Cloud Watch","tags":["monitoring"]},{"pattern":"bitlybot","url":"https://bitly.com/","description":"Bit.ly web crawler for link tracking","tags":["seo"]},{"pattern":"blogmuraBot","url":"https://blogmura.com/","description":"Blog ranking site linking to blogs across all themes","tags":["seo"]},{"pattern":"Bot\\.AraTurka\\.com","url":"http://www.araturka.com","description":"AraTurka web crawler for content discovery","tags":["search-engine"]},{"pattern":"bot-pge\\.chlooe\\.com","description":"Chlooe web crawler for content analysis","tags":["seo"]},{"pattern":"BoxcarBot","url":"https://boxcar.io/","description":"Boxcar web crawler for notifications","tags":["seo"]},{"pattern":"BTWebClient","url":"https://www.utorrent.com/","description":"µTorrent BitTorrent Client","tags":["http-library"]},{"pattern":"ContextAd Bot","description":"ContextAd web crawler for advertising","tags":["advertising"]},{"pattern":"Digincore bot","url":"http://www.digincore.com/crawler.html","description":"Digincore web crawler for content analysis","tags":["seo"]},{"pattern":"Disqus","url":"https://disqus.com/","description":"validate and quality check pages.","tags":["seo"]},{"pattern":"Feedly(?:Bot)?\\/","url":"https://feedly.com/fetcher.html","description":"Feed aggregator fetching RSS and Atom feeds for users","tags":["feed-reader"],"rdns":[".feedly.com"]},{"pattern":"Fetch\\/","description":"Fetch web crawler for CMS detection","tags":["seo"]},{"pattern":"Fever","url":"http://feedafever.com","description":"Fever feed reader web crawler bot","tags":["feed-reader"]},{"pattern":"Flamingo_SearchEngine","description":"Flamingo search engine web crawler bot","tags":["search-engine"]},{"pattern":"FlipboardProxy","url":"https://about.flipboard.com/browserproxy/","description":"Proxy fetching and preparing website elements for Flipboard","tags":["social-preview"],"rdns":[".flipboard.com",".flipboard.net"]},{"pattern":"g2reader-bot","url":"https://www.g2reader.com/","description":"G2Reader web crawler for content discovery","tags":["feed-reader"]},{"pattern":"G2 Web Services","url":"https://g2risksolutions.com:443/","description":"G2 web services web crawler bot","tags":["seo"]},{"pattern":"imrbot","url":"http://www.mignify.com","description":"Mignify imrbot web crawler bot","tags":["seo"]},{"pattern":"K7MLWCBot","url":"https://www.k7computing.com/","description":"Virus scanner","tags":["scanner"]},{"pattern":"Kemvibot","url":"http://kemvi.com","description":"Kemvi web crawler for content discovery","tags":["seo"]},{"pattern":"Landau-Media-Spider","url":"http://bots.landaumedia.de/bot.html","description":"Landau Media web crawler bot","tags":["seo"]},{"pattern":"linkapediabot","url":"http://www.linkapedia.com","description":"Linkapedia web crawler for link discovery","tags":["seo"]},{"pattern":"vkShare","url":"http://vk.com/dev/Share","description":"VK Share web crawler for social sharing","tags":["social-preview"]},{"pattern":"Siteimprove\\.com|SiteCheck-sitecrawl","description":"Siteimprove web crawler for site analysis","tags":["monitoring"]},{"pattern":"BLEXBot\\/","url":"https://www.link-assistant.com","description":"WebMeUp BLEX web crawler bot","tags":["seo"],"rdns":[".blex.net",".blexbot.com",".webmeup.com"]},{"pattern":"DareBoost","url":"https://www.dareboost.com/en","description":"Bot to test, Analyze and Optimize website","tags":["monitoring"]},{"pattern":"ZuperlistBot\\/","description":"Zuperlist web crawler for content discovery","tags":["seo"]},{"pattern":"Miniflux\\/","url":"https://miniflux.app/","description":"Miniflux is a minimalist and opinionated feed reader.","tags":["feed-reader"]},{"pattern":"Feedspot(?:bot)?\\/","url":"https://www.feedspot.com/fs/bot","description":"Feedspot web crawler for feed discovery","tags":["feed-reader"],"rdns":[".feedspot.com"]},{"pattern":"Diffbot\\/","url":"https://www.diffbot.com/","description":"Diffbot web crawler for content extraction","tags":["seo"],"rdns":[".diffbot.com"]},{"pattern":"SEOkicks","url":"https://www.seokicks.de/robot.html","description":"SEOkicks web crawler for SEO analysis","tags":["seo"]},{"pattern":"tracemyfile","description":"TraceMyFile web crawler for file tracking","tags":["seo"]},{"pattern":"Nimbostratus-Bot","description":"Nimbostratus web crawler for cloud analysis","tags":["seo"]},{"pattern":"zgrab","url":"https://github.com/zmap/zgrab2","description":"Zgrab web crawler for security scanning","tags":["scanner"]},{"pattern":"PR-CY\\.RU","url":"https://pr-cy.ru","description":"PR-CY web crawler for SEO analysis","tags":["seo"]},{"pattern":"AdsTxtCrawler","description":"AdsTxt web crawler for ads.txt validation","tags":["advertising"]},{"pattern":"Datafeedwatch","url":"https://www.datafeedwatch.com/","description":"Datafeedwatch web crawler for feed analysis","tags":["seo"]},{"pattern":"Zabbix","url":"https://www.zabbix.com/documentation/3.4/en/manual/web_monitoring","description":"Zabbix web crawler for monitoring","tags":["monitoring"]},{"pattern":"TangibleeBot","url":"http://tangiblee.com/bot","description":"Tangiblee web crawler for visual search","tags":["seo"]},{"pattern":"google-xrawler","url":"https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for","description":"Google xrawler web crawler bot","tags":["search-engine"]},{"pattern":"axios","url":"https://github.com/axios/axios","description":"Axios HTTP client library for requests","tags":["http-library"]},{"pattern":"Amazon CloudFront","url":"https://aws.amazon.com/cloudfront/","description":"Amazon CloudFront web crawler bot","tags":["http-library"],"rdns":[".amazon.com",".amazonaws.com"]},{"pattern":"Pulsepoint","description":"Pulsepoint web crawler for content discovery","tags":["advertising"]},{"pattern":"CloudFlare-AlwaysOnline","url":"https://www.cloudflare.com/always-online/","description":"CloudFlare always online web crawler","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Healthchecks","url":"https://developers.cloudflare.com/health-checks/","description":"CloudFlare health checks web crawler","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Traffic-Manager","url":"https://developers.cloudflare.com/load-balancing/monitors/","description":"CloudFlare traffic manager web crawler","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"CloudFlare-Prefetch","url":"https://developers.cloudflare.com/speed/optimization/content/prefetch-urls/","description":"CloudFlare prefetch web crawler bot","tags":["seo"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-SSLDetector","url":"https://developers.cloudflare.com/ssl/origin-configuration/ssl-tls-recommender/","description":"CloudFlare SSL detector web crawler","tags":["scanner"],"rdns":[".cloudflare.com"]},{"pattern":"https:\\/\\/developers\\.cloudflare\\.com\\/security-center\\/","url":"https://developers.cloudflare.com/ssl/origin-configuration/ssl-tls-recommender/","description":"CloudFlare security center web crawler","tags":["scanner"]},{"pattern":"Google-Structured-Data-Testing-Tool","url":"https://developers.google.com/search/docs/appearance/structured-data","description":"Google structured data testing web crawler","tags":["search-engine"]},{"pattern":"WordupInfoSearch","description":"Wordup info search web crawler bot","tags":["search-engine"]},{"pattern":"WebDataStats","url":"https://webdatastats.com/","description":"WebDataStats web crawler for data analysis","tags":["seo"]},{"pattern":"HttpUrlConnection","description":"Java HttpUrlConnection HTTP client","tags":["http-library"]},{"pattern":"ZoomBot","url":"https://www.seozoom.it/bot/","description":"SEOZoom web crawler for SEO analysis","tags":["seo"],"rdns":[".zoom.us"]},{"pattern":"VelenPublicWebCrawler","url":"https://velen.io/","description":"Velen web crawler for content discovery","tags":["seo"],"rdns":[".velen.io"]},{"pattern":"MoodleBot","description":"Moodle web crawler for learning platforms","tags":["seo"]},{"pattern":"jpg-newsbot","url":"https://vipnytt.no/bots/","description":"JPG news web crawler bot","tags":["seo"]},{"pattern":"outbrain","url":"https://www.outbrain.com/help/advertisers/invalid-url/","description":"Outbrain web crawler for content discovery","tags":["advertising"]},{"pattern":"W3C_Validator","url":"https://validator.w3.org/services","description":"W3C HTML validator web crawler bot","tags":["monitoring"],"rdns":[".w3.org",".w3c.org"]},{"pattern":"Validator\\.nu","url":"https://validator.w3.org/services","description":"Validator.nu HTML validator web crawler","tags":["monitoring"],"rdns":[".validator.nu"]},{"pattern":"W3C-checklink","url":"https://validator.w3.org/services","description":"W3C checklink web crawler for link validation","tags":["monitoring"],"depends_on":["libwww-perl"],"rdns":[".w3.org",".w3c.org"]},{"pattern":"W3C-mobileOK","url":"https://validator.w3.org/services","description":"W3C mobile OK web crawler bot","tags":["monitoring"],"rdns":[".w3.org",".w3c.org"]},{"pattern":"W3C_I18n-Checker","url":"https://validator.w3.org/services","description":"W3C internationalization web crawler","tags":["monitoring"]},{"pattern":"FeedValidator","url":"https://validator.w3.org/services","description":"Feed validator web crawler for RSS validation","tags":["monitoring"]},{"pattern":"W3C_CSS_Validator","url":"https://validator.w3.org/services","description":"W3C CSS validator web crawler bot","tags":["monitoring"],"rdns":[".w3.org",".w3c.org"]},{"pattern":"W3C_Unicorn","url":"https://validator.w3.org/services","description":"W3C Unicorn validator web crawler bot","tags":["monitoring"],"rdns":[".w3.org",".w3c.org"]},{"pattern":"Google-PhysicalWeb","description":"Google physical web web crawler bot","tags":["search-engine"]},{"pattern":"Blackboard","url":"https://self-hosting.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign.html","description":"Blackboard SafeAssign web crawler bot","tags":["academic"]},{"pattern":"ICBot\\/","url":"https://ideasandcode.xyz","description":"Ideas and Code web crawler bot","tags":["seo"]},{"pattern":"BazQux","url":"https://bazqux.com/fetcher","description":"BazQux RSS reader web crawler bot","tags":["feed-reader"]},{"pattern":"Twingly","url":"https://www.twingly.com/","description":"Twingly blog search web crawler bot","tags":["seo"]},{"pattern":"Rivva","url":"https://rivva.de/","description":"Rivva blog search web crawler bot","tags":["seo"]},{"pattern":"Experibot","url":"https://amirkr.wixsite.com/experibot","description":"Experibot web crawler for testing","tags":["monitoring"]},{"pattern":"awesomecrawler","description":"Awesome web crawler for content discovery","tags":["seo"]},{"pattern":"Dataprovider\\.com","url":"https://www.dataprovider.com/","description":"Dataprovider web crawler for data collection","tags":["seo"]},{"pattern":"GroupHigh\\/","url":"http://www.grouphigh.com/","description":"GroupHigh web crawler for influencer marketing","tags":["seo"]},{"pattern":"theoldreader\\.com","url":"https://www.theoldreader.com/","description":"The Old Reader web crawler bot","tags":["feed-reader"],"rdns":[".theoldreader.com"]},{"pattern":"AnyEvent","url":"http://software.schmorp.de/pkg/AnyEvent.html","description":"AnyEvent Perl HTTP web crawler library","tags":["http-library"]},{"pattern":"Uptimebot\\.org","url":"http://uptimebot.org/","description":"Uptimebot website monitoring web crawler","tags":["monitoring"]},{"pattern":"Nmap Scripting Engine","url":"https://nmap.org/book/nse.html","description":"Nmap NSE web crawler for security scanning","tags":["scanner"],"rdns":[".nmap.org"]},{"pattern":"2ip\\.ru","url":"https://2ip.ru/cms/","description":"2IP CMS detector web crawler bot","tags":["seo"]},{"pattern":"Clickagy","url":"https://www.clickagy.com","description":"Clickagy intelligence web crawler bot","tags":["advertising"]},{"pattern":"Caliperbot","url":"https://www.conductor.com/legal/caliperbot/","description":"Conductor Caliperbot web crawler for analysis","tags":["seo"],"rdns":[".conductor.com"]},{"pattern":"MBCrawler","url":"https://www.seoptimer.com/monitor-backlinks/","description":"Monitor Backlinks web crawler bot","tags":["seo"],"rdns":[".monitorbacklinks.com"]},{"pattern":"online-webceo-bot","url":"http://online.webceo.com","description":"Online WebCEO web crawler for analysis","tags":["seo"]},{"pattern":"B2B Bot","description":"B2B web crawler for business discovery","tags":["seo"]},{"pattern":"AddSearchBot","url":"https://www.addsearch.com/docs/indexing/whitelisting-addsearch-bot/","description":"AddSearch web crawler for site search","tags":["search-engine"]},{"pattern":"Google [fF]avicon","description":"Google favicon web crawler bot","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"HubSpot","description":"HubSpot web crawler for marketing automation","tags":["seo"]},{"pattern":"Chrome-Lighthouse","url":"https://pagespeed.web.dev/?utm_source=psi&utm_medium=redirect","description":"Chrome Lighthouse web crawler for audits","tags":["monitoring"]},{"pattern":"HeadlessChrome","url":"https://developer.chrome.com/blog/headless-chrome/","description":"Headless Chrome web crawler for testing","tags":["browser-automation"],"rdns":[".pptr.dev",".puppeteer.dev"]},{"pattern":"CheckMarkNetwork\\/","url":"https://www.checkmarknetwork.com/","description":"CheckMark Network web crawler bot","tags":["monitoring"]},{"pattern":"www\\.uptime\\.com","url":"https://uptime.com/uptime-bot","description":"Uptime.com website monitoring web crawler","tags":["monitoring"]},{"pattern":"Streamline3Bot\\/","url":"https://www.ubtsupport.com/legal/Streamline3Bot.php","description":"Streamline3 web crawler for monitoring","tags":["monitoring"]},{"pattern":"serpstatbot\\/","url":"https://serpstatbot.com/","description":"Serpstat web crawler for backlink tracking","tags":["seo"],"rdns":[".serpstat.com"]},{"pattern":"MixnodeCache\\/","url":"https://cache.mixnode.com/","description":"Mixnode cache web crawler bot","tags":["seo"]},{"pattern":"^curl","url":"https://curl.se/","description":"cURL command-line tool for HTTP requests","tags":["http-library"]},{"pattern":"SimpleScraper","url":"https://github.com/ramonk2o/simple-scraper","description":"Simple Scraper web scraping tool","tags":["seo"]},{"pattern":"RSSingBot","url":"https://www.rssing.com/","description":"RSSing RSS reader web crawler bot","tags":["feed-reader"]},{"pattern":"Jooblebot","url":"http://jooble.org/jooble-bot","description":"Jooble job search web crawler bot","tags":["search-engine"]},{"pattern":"fedoraplanet","url":"https://fedoraplanet.org/","description":"Fedora Planet web crawler for aggregation","tags":["feed-reader"]},{"pattern":"Friendica","url":"https://hoyer.xyz","description":"Friendica social network web crawler bot","tags":["social-preview"]},{"pattern":"NextCloud","url":"https://nextcloud.com/","description":"NextCloud news reader web crawler bot","tags":["feed-reader"]},{"pattern":"Tiny Tiny RSS","url":"https://tt-rss.org/","description":"Tiny Tiny RSS reader web crawler bot","tags":["feed-reader"]},{"pattern":"RegionStuttgartBot","url":"https://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/","description":"Region Stuttgart web crawler bot","tags":["seo"]},{"pattern":"Bytespider","url":"https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent","description":"ByteDance Bytespider web crawler bot","tags":["ai-crawler"],"rdns":[".bytedance.com",".tiktokv.com"]},{"pattern":"Datanyze","url":"https://www.datanyze.com/dnyzbot/","description":"Datanyze web crawler for technology detection","tags":["seo"]},{"pattern":"Google-Site-Verification","url":"https://support.google.com/webmasters/answer/9008080","description":"Google site verification web crawler bot","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"TrendsmapResolver","url":"https://www.trendsmap.com/","description":"Trendsmap web crawler for trend analysis","tags":["seo"]},{"pattern":"tweetedtimes","url":"https://tweetedtimes.com/","description":"Tweeted Times web crawler for news","tags":["seo"]},{"pattern":"NTENTbot","url":"https://ntent.com/ntentbot/","description":"NTENT web crawler for search results","tags":["search-engine"]},{"pattern":"Gwene","url":"https://gwene.org","description":"Gwene RSS to news gateway web crawler","tags":["feed-reader"]},{"pattern":"SimplePie","url":"http://simplepie.org","description":"SimplePie PHP feed parser web crawler","tags":["feed-reader"]},{"pattern":"SearchAtlas","url":"https://SearchAtlas.com/","description":"SearchAtlas SEO web crawler for analysis","tags":["seo"]},{"pattern":"Superfeedr","url":"https://superfeedr.com/","description":"Superfeedr feed reader web crawler bot","tags":["feed-reader"]},{"pattern":"feedbot","url":"https://wordpress.com/","description":"WordPress.com feed web crawler bot","tags":["feed-reader"]},{"pattern":"UT-Dorkbot","url":"https://security.utexas.edu/dorkbot","description":"University of Texas security web crawler","tags":["scanner"]},{"pattern":"Amazonbot","url":"https://developer.amazon.com/amazonbot","description":"Amazon web crawler for product discovery","tags":["search-engine"],"rdns":[".amazon.com",".amazonaws.com"]},{"pattern":"AmazonProductDiscovery","url":"https://vendorcentral.amazon.com/support/amazonproductbot","description":"Amazon product discovery web crawler","tags":["search-engine"]},{"pattern":"AmazonSellerInitiatedListing","url":"https://vendorcentral.amazon.com/support/amazonproductbot","description":"Amazon seller listing web crawler","tags":["search-engine"]},{"pattern":"SerendeputyBot","url":"http://serendeputy.com/about/serendeputy-bot","description":"Serendeputy web crawler for content discovery","tags":["seo"]},{"pattern":"Eyeotabot","url":"https://www.eyeota.com/","description":"Eyeota web crawler for audience data","tags":["advertising"]},{"pattern":"officestorebot","url":"https://www.bing.com?ref=aka&shorturl=officestorebot","description":"Microsoft Office Store web crawler bot","tags":["search-engine"]},{"pattern":"Neticle Crawler","url":"https://neticle.com/bot/en","description":"Neticle web crawler for content analysis","tags":["seo"]},{"pattern":"SurdotlyBot","url":"http://sur.ly/bot.html","description":"Surly web crawler for link shortening","tags":["seo"]},{"pattern":"LinkisBot","description":"Linkis web crawler for link sharing","tags":["seo"]},{"pattern":"AwarioSmartBot","url":"https://awario.com/bots.html","description":"Awario smart web crawler bot","tags":["seo"]},{"pattern":"AwarioRssBot","url":"https://awario.com/bots.html","description":"Awario RSS feed web crawler bot","tags":["feed-reader","seo"]},{"pattern":"RyteBot","url":"https://www.semrush.com:443/bot/","description":"Ryte web crawler for site analysis","tags":["seo"],"rdns":[".onpage.org",".ryte.com"]},{"pattern":"FreeWebMonitoring SiteChecker","url":"https://www.freewebmonitoring.com/bot.html","description":"FreeWebMonitoring site checker web crawler","tags":["monitoring"]},{"pattern":"AspiegelBot","url":"https://aspiegel.com","description":"Aspiegel web crawler for content discovery","tags":["search-engine"],"rdns":[".aspiegel.com",".huawei.com",".petalsearch.com",".petalsearch.net"]},{"pattern":"NAVER Blog Rssbot","url":"https://www.naver.com/","description":"Naver blog RSS web crawler bot","tags":["feed-reader"],"rdns":[".naver.com",".naver.jp"]},{"pattern":"zenback bot","url":"https://corp.logly.co.jp/","description":"Zenback web crawler for content discovery","tags":["seo"]},{"pattern":"SentiBot","url":"https://sites.google.com/senti1.com/sentibot-eu/home","description":"SentiBot web crawler for sentiment analysis","tags":["seo"]},{"pattern":"Domains Project\\/","url":"https://github.com/tb0hdan/domains","description":"Domains Project web crawler bot","tags":["seo"]},{"pattern":"Pandalytics","url":"https://domainsbot.com/pandalytics/","description":"Pandalytics web crawler for domain analysis","tags":["seo"]},{"pattern":"VKRobot","description":"VK social network web crawler bot","tags":["social-preview"]},{"pattern":"bidswitchbot","url":"https://www.bidswitch.com/about-us/","description":"BidSwitch web crawler for advertising","tags":["advertising"]},{"pattern":"tigerbot","description":"Tiger web crawler for content discovery","tags":["seo"]},{"pattern":"NIXStatsbot","url":"http://www.nixstats.com/bot.html","description":"NIXStats web crawler for monitoring","tags":["monitoring"]},{"pattern":"Atom Feed Robot","url":"https://rssmicro.com","description":"RSSMicro atom feed web crawler bot","tags":["feed-reader"]},{"pattern":"[Cc]urebot","description":"Cure web crawler for content discovery","tags":["monitoring"]},{"pattern":"PagePeeker\\/","url":"https://pagepeeker.com/robots/","description":"PagePeeker web crawler for screenshots","tags":["seo"]},{"pattern":"Vigil\\/","url":"http://vigil-app.com/bot.html","description":"Vigil web crawler for monitoring","tags":["monitoring"]},{"pattern":"rssbot\\/","url":"https://github.com/iovxw/rssbot","description":"RSS bot web crawler for feeds","tags":["feed-reader"]},{"pattern":"startmebot\\/","url":"https://support.start.me/en/articles/9182933-what-is-startmebot","description":"Start.me web crawler for content discovery","tags":["seo"]},{"pattern":"JobboerseBot","url":"https://www.xing.com/","description":"Jobboerse web crawler for job discovery","tags":["search-engine"]},{"pattern":"seewithkids","url":"http://seewithkids.com/bot","description":"SeeWithKids web crawler for content discovery","tags":["seo"]},{"pattern":"NINJA bot","description":"NINJA web crawler for content discovery","tags":["scanner"]},{"pattern":"Cutbot","url":"https://www.cutbot.net/","description":"Cutbot web crawler for content discovery","tags":["seo"]},{"pattern":"BublupBot","url":"https://www.bublup.com/bublup-bot","description":"Bublup web crawler for content discovery","tags":["seo"]},{"pattern":"BrandONbot","url":"https://brandonmedia.net/","description":"BrandON web crawler for brand monitoring","tags":["seo"]},{"pattern":"RidderBot","url":"https://ridder.co/","description":"Ridder web crawler for content discovery","tags":["seo"]},{"pattern":"Taboolabot","url":"https://www.taboola.com/","description":"Taboola web crawler for content discovery","tags":["advertising"]},{"pattern":"Dubbotbot","url":"https://dubbot.com/","description":"Dubbot web crawler for content discovery","tags":["monitoring"]},{"pattern":"FindITAnswersbot","url":"http://search.it-influentials.com/bot.htm","description":"FindITAnswers web crawler for discovery","tags":["seo"]},{"pattern":"infoobot","url":"https://www.infoo.nl/bot.html","description":"Infoo web crawler for content discovery","tags":["seo"]},{"pattern":"Refindbot","url":"https://refind.com/about","description":"Refind web crawler for content discovery","tags":["seo"]},{"pattern":"BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher","url":"http://www.blogtraffic.de/rss-bot.html","description":"BlogTraffic feed fetcher web crawler","tags":["feed-reader"]},{"pattern":"SeobilityBot","url":"https://www.seobility.net/en/bot/","description":"Seobility web crawler for SEO analysis","tags":["seo"]},{"pattern":"Cincraw","url":"https://cincrawdata.net/bot/","description":"Cincraw web crawler for data collection","tags":["seo"]},{"pattern":"Dragonbot","url":"https://www.dragonmetrics.com/","description":"DragonMetrics web crawler for analysis","tags":["seo"]},{"pattern":"VoluumDSP-content-bot","url":"https://cm.tech/","description":"Voluum DSP web crawler for advertising","tags":["advertising"]},{"pattern":"FreshRSS","url":"https://freshrss.org","description":"FreshRSS feed reader web crawler bot","tags":["feed-reader"]},{"pattern":"BitBot","url":"https://bitbot.dev","description":"BitBot web crawler for content discovery","tags":["seo"]},{"pattern":"^PHP-Curl-Class","url":"https://github.com/php-curl-class/php-curl-class","description":"PHP Curl Class HTTP client library","tags":["http-library"]},{"pattern":"Google-Certificates-Bridge","description":"Google certificates bridge web crawler","tags":["search-engine"]},{"pattern":"centurybot","description":"Century web crawler for content discovery","tags":["seo"]},{"pattern":"Viber","url":"https://www.viber.com/","description":"Viber messaging web crawler bot","tags":["social-preview"]},{"pattern":"e\\.ventures Investment Crawler","url":"https://www.eventures.vc/","description":"E.ventures investment web crawler bot","tags":["seo"]},{"pattern":"evc-batch","url":"https://www.eventures.vc/","description":"E.ventures batch web crawler bot","tags":["seo"]},{"pattern":"PetalBot","url":"https://webmaster.petalsearch.com/site/petalbot","description":"Petal search engine web crawler bot","tags":["search-engine"],"rdns":[".aspiegel.com",".huawei.com",".petalsearch.com",".petalsearch.net"]},{"pattern":"virustotal","url":"https://www.virustotal.com/gui/home/url","description":"VirusTotal web crawler for security","tags":["scanner"]},{"pattern":"(^| )PTST\\/","url":"https://www.webpagetest.org","description":"WebPageTest web crawler for testing","tags":["monitoring"]},{"pattern":"minicrawler","url":"http://testomato.com/bot/","description":"Testomato web crawler for testing","tags":["monitoring"]},{"pattern":"Cookiebot","url":"https://www.cookiebot.com/","description":"Cookiebot web crawler for cookie scanning","tags":["scanner"]},{"pattern":"trovitBot","url":"https://www.trovit.com/bot.html","description":"Trovit web crawler for content discovery","tags":["search-engine"]},{"pattern":"seostar\\.co","url":"https://seostar.co/robot/","description":"SEOstar web crawler for SEO analysis","tags":["seo"]},{"pattern":"IonCrawl","url":"https://www.ionos.de/terms-gtc/faq-crawler-en","description":"IONOS IonCrawl web crawler bot","tags":["seo"]},{"pattern":"Uptime-Kuma","url":"https://uptime.kuma.pet/","description":"Uptime Kuma web crawler for monitoring","tags":["monitoring"]},{"pattern":"Seekport","url":"https://bot.seekport.com","description":"Seekport web crawler for content discovery","tags":["search-engine"],"rdns":[".seekport.com"]},{"pattern":"FreshpingBot","url":"https://www.freshworks.com/website-monitoring/","description":"Freshping web crawler for monitoring","tags":["monitoring"]},{"pattern":"Feedbin","url":"https://feedbin.com/","description":"Feedbin feed reader web crawler bot","tags":["feed-reader"],"rdns":[".feedbin.com"]},{"pattern":"CriteoBot","url":"https://www.criteo.com/","description":"Criteo web crawler for advertising","tags":["advertising"]},{"pattern":"Snap URL Preview Service","url":"https://www.snap.com/","description":"Snapchat URL preview web crawler","tags":["social-preview"],"rdns":[".snapchat.com"]},{"pattern":"Better Uptime Bot","url":"https://betterstack.com/uptime","description":"Better Uptime web crawler for monitoring","tags":["monitoring"]},{"pattern":"RuxitSynthetic","url":"https://docs.dynatrace.com/docs/observe/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors","description":"Dynatrace Ruxit synthetic web crawler","tags":["monitoring"],"rdns":[".dynatrace.com"]},{"pattern":"Google-Read-Aloud","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","description":"Google read aloud web crawler bot","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Valve\\/Steam","description":"Steam web crawler for link previews","tags":["social-preview"]},{"pattern":"OdklBot\\/","url":"https://ok.ru/","description":"Odnoklassniki web crawler for sharing","tags":["social-preview"]},{"pattern":"GPTBot","url":"https://platform.openai.com/docs/gptbot","description":"OpenAI GPT web crawler bot","tags":["ai-crawler"],"rdns":[".openai.com"]},{"pattern":"ChatGPT-User","url":"https://openai.com/bot","description":"ChatGPT user web crawler bot","tags":["ai-fetcher"],"rdns":[".openai.com"]},{"pattern":"OAI-SearchBot","url":"https://platform.openai.com/docs/bots","description":"OpenAI search web crawler bot","tags":["ai-fetcher","search-engine"],"rdns":[".openai.com"]},{"pattern":"YandexRenderResourcesBot\\/","url":"https://yandex.com/support/webmaster/en/robot-workings/check-yandex-robots.html?lang=en","description":"Yandex render resources web crawler","tags":["search-engine"]},{"pattern":"LightspeedSystemsCrawler","description":"Lightspeed Systems web crawler bot","tags":["scanner"]},{"pattern":"ev-crawler\\/","url":"https://headline.com/legal/crawler","description":"Headline web crawler for content discovery","tags":["seo"]},{"pattern":"BitSightBot\\/","url":"https://www.bitsight.com","description":"BitSight web crawler for security","tags":["scanner"]},{"pattern":"woorankreview\\/","url":"https://www.woorank.com/","description":"WooRank web crawler for SEO analysis","tags":["seo"]},{"pattern":"Google-Safety","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","description":"Google safety web crawler bot","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"AwarioBot","url":"https://awario.com/bots.html","description":"Awario web crawler for monitoring","tags":["seo"]},{"pattern":"DataForSeoBot","url":"https://dataforseo.com/dataforseo-bot","description":"DataForSEO web crawler for analysis","tags":["seo"],"rdns":[".dataforseo.com"]},{"pattern":"Linespider","url":"https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en","description":"LINE Linespider web crawler bot","tags":["social-preview"]},{"pattern":"WellKnownBot","url":"https://well-known.dev/about/#bot)","description":"WellKnown web crawler for discovery","tags":["seo"]},{"pattern":"A Patent Crawler","url":"https://www.epfl.ch/research/facilities/scitas/","description":"EPFL patent web crawler bot","tags":["academic"]},{"pattern":"StractBot","url":"https://trystract.com/webmasters","description":"Stract search engine web crawler bot","tags":["search-engine"]},{"pattern":"search\\.marginalia\\.nu","url":"https://marginalia-search.com/","description":"Marginalia search web crawler bot","tags":["search-engine"]},{"pattern":"YouBot","url":"https://you.com/home","description":"You.com search engine web crawler bot","tags":["search-engine"],"rdns":[".you.com",".youbot.com"]},{"pattern":"Nicecrawler","url":"http://www.nicecrawler.com/","description":"Nicecrawler web crawler for discovery","tags":["search-engine"]},{"pattern":"Neevabot","url":"https://neeva.com/neevabot","description":"Neeva search engine web crawler bot","tags":["search-engine"],"rdns":[".neeva.com",".neevabot.com"]},{"pattern":"BrightEdge Crawler","url":"https://www.brightedge.com/","description":"BrightEdge web crawler for SEO","tags":["seo"],"rdns":[".brightedge.com"]},{"pattern":"SiteCheckerBotCrawler","url":"https://sitechecker.pro","description":"SiteChecker web crawler for analysis","tags":["seo"]},{"pattern":"TombaPublicWebCrawler","url":"https://tombascraper.com","description":"Tomba web crawler for email discovery","tags":["seo"]},{"pattern":"CrawlyProjectCrawler","url":"https://crawlyproject.digitaldragon.dev/","description":"Crawly Project web crawler framework","tags":["seo"]},{"pattern":"KomodiaBot","url":"https://www.komodia.com/","description":"Komodia web crawler for classification","tags":["seo"]},{"pattern":"KStandBot","url":"https://url-classification.io/","description":"KStand web crawler for classification","tags":["seo"]},{"pattern":"CISPA Webcrawler","url":"https://vuln-notify-checker.cispa.saarland","description":"CISPA web crawler for vulnerability","tags":["academic"]},{"pattern":"MTRobot","url":"https://metrics-tools.de/robot.html","description":"Metrics Tools web crawler for analysis","tags":["seo"]},{"pattern":"hyscore\\.io","url":"https://hyscore.io/crawler/","description":"HyScore web crawler for analysis","tags":["seo"]},{"pattern":"AlexandriaOrgBot","url":"https://www.alexandria.org/bot.html","description":"Alexandria web crawler for discovery","tags":["search-engine"]},{"pattern":"2ip bot","url":"http://2ip.io","description":"2IP web crawler for analysis","tags":["seo"]},{"pattern":"Yellowbrandprotectionbot","url":"https://corsearch.com/yellow/","description":"Yellow brand protection web crawler","tags":["seo"]},{"pattern":"SEOlizer","url":"https://www.seolizer.de/bot.html","description":"SEOlizer web crawler for SEO analysis","tags":["seo"]},{"pattern":"vuhuvBot","url":"https://vuhuv.com/bot.html","description":"Vuhuv web crawler for content discovery","tags":["seo"]},{"pattern":"INETDEX-BOT","url":"https://inetdex.com/bot.html","description":"INETDEX web crawler for indexing","tags":["search-engine"]},{"pattern":"Synapse","url":"https://github.com/matrix-org/synapse","description":"Matrix Synapse web crawler bot","tags":["social-preview"]},{"pattern":"t3versionsBot","url":"https://www.t3versions.com/bot","description":"T3versions web crawler for discovery","tags":["seo"]},{"pattern":"deepnoc","url":"https://deepnoc.com/bot","description":"DeepNOC web crawler for monitoring","tags":["monitoring"]},{"pattern":"Cocolyzebot","url":"https://cocolyze.com/en/bot","description":"Cocolyze web crawler for analysis","tags":["seo"]},{"pattern":"hypestat","url":"https://hypestat.com/bot","description":"Hypestat web crawler for analysis","tags":["seo"]},{"pattern":"ReverseEngineeringBot","url":"https://torus.company/bot.html","description":"Torus reverse engineering web crawler","tags":["scanner"]},{"pattern":"sempi\\.tech","url":"http://sempi.tech/bot.html","description":"Sempi web crawler for analysis","tags":["seo"]},{"pattern":"Iframely","url":"https://iframely.com/docs/about","description":"Iframely web crawler for embeds","tags":["social-preview"]},{"pattern":"MetaInspector","url":"https://github.com/jaimeiniesta/metainspector","description":"MetaInspector web crawler for metadata","tags":["seo"]},{"pattern":"node-fetch","url":"https://github.com/node-fetch/node-fetch","description":"Node-fetch HTTP client library","tags":["http-library"]},{"pattern":"l9explore","url":"https://github.com/LeakIX/l9explore","description":"L9explore web crawler for discovery","tags":["scanner"]},{"pattern":"python-opengraph","url":"https://github.com/jaywink/python-opengraph","description":"Python OpenGraph web crawler bot","tags":["http-library"]},{"pattern":"OpenGraphCheck","url":"https://opengraphcheck.com","description":"OpenGraphCheck web crawler for metadata","tags":["seo"]},{"pattern":"developers\\.google\\.com\\/\\+\\/web\\/snippet","url":"https://developers.google.com/+/api-shutdown","description":"Google snippet web crawler bot","tags":["search-engine"]},{"pattern":"SenutoBot","url":"https://www.senuto.com","description":"Senuto web crawler for SEO analysis","tags":["seo"]},{"pattern":"MaCoCu","url":"https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/","description":"MaCoCu web crawler for language research","tags":["academic"]},{"pattern":"NewsBlur","url":"https://www.newsblur.com/","description":"NewsBlur feed reader web crawler bot","tags":["feed-reader"],"rdns":[".newsblur.com"]},{"pattern":"inoreader","url":"https://www.inoreader.com/","description":"Inoreader feed reader web crawler bot","tags":["feed-reader"],"rdns":[".inoreader.com"]},{"pattern":"NetSystemsResearch","url":"http://netsystemsresearch.com","description":"NetSystemsResearch web crawler bot","tags":["scanner"]},{"pattern":"PageThing","url":"https://domainsestate.com/domains/pagething-com/","description":"PageThing web crawler for content discovery","tags":["seo"]},{"pattern":"WordPress","url":"https://wordpress.org","description":"WordPress web crawler for site discovery","tags":["seo"]},{"pattern":"PhxBot","description":"Phoenix web crawler for content discovery","tags":["search-engine"]},{"pattern":"ImagesiftBot","url":"https://imagesift.com/about","description":"Imagesift bot for image search and indexing","tags":["search-engine"],"rdns":[".imagesift.com"]},{"pattern":"Expanse","url":"https://www.paloaltonetworks.com/cortex/cortex-xpanse","description":"Palo Alto Networks Expanse bot for internet asset discovery","tags":["scanner"]},{"pattern":"InternetMeasurement","url":"https://internet-measurement.com","description":"Internet Measurement bot for network research and analysis","tags":["academic"]},{"pattern":"^BW\\/","url":"https://builtwith.com/biup","description":"BuiltWith web crawler for technology detection","tags":["seo"]},{"pattern":"GeedoBot","url":"https://geedo.com/","description":"Geedo web crawler for content discovery","tags":["search-engine"]},{"pattern":"Audisto Crawler","url":"https://audisto.com/help/crawler/bot/","description":"Audisto SEO web crawler for analysis","tags":["seo"]},{"pattern":"PerplexityBot\\/","url":"https://docs.perplexity.ai/docs/perplexitybot","description":"Perplexity AI web crawler for search","tags":["ai-crawler","search-engine"],"rdns":[".perplexity.ai"]},{"pattern":"[cC]laude[bB]ot","url":"https://www.anthropic.com/","description":"Anthropic Claude web crawler bot","tags":["ai-crawler"],"rdns":[".anthropic.com"]},{"pattern":"Monsidobot","url":"http://monsido.com/bot.html","description":"Monsido web crawler for website monitoring","tags":["monitoring"]},{"pattern":"GroupMeBot","url":"https://groupme.com/","description":"GroupMe web crawler for messaging","tags":["social-preview"]},{"pattern":"Vercelbot","url":"https://github.com/vercel/vercel/discussions/5095#discussioncomment-58705","description":"Vercel web crawler for deployment","tags":["seo"]},{"pattern":"vercel-screenshot","description":"Vercel screenshot web crawler bot","tags":["seo"]},{"pattern":"facebookcatalog\\/","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","description":"Facebook catalog web crawler bot","tags":["advertising"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"meta-externalads\\/","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","description":"Meta external ads web crawler bot","tags":["advertising"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"meta-externalagent\\/","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","description":"Meta external agent web crawler bot","tags":["ai-crawler","social-preview"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"meta-externalfetcher\\/","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","description":"Meta external fetcher web crawler bot","tags":["social-preview"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"AcademicBotRTU","url":"https://academicbot.rtu.lv","description":"Academic Bot RTU web crawler bot","tags":["academic"]},{"pattern":"KeybaseBot","url":"https://book.keybase.io/docs/chat/link-previews","description":"Keybase web crawler for link previews","tags":["social-preview"]},{"pattern":"Lemmy","url":"https://leminal.space","description":"Lemmy social network web crawler bot","tags":["social-preview"]},{"pattern":"CookieHubScan","url":"https://www.cookiehub.com/","description":"CookieHub web crawler for cookie scanning","tags":["scanner"]},{"pattern":"Hydrozen\\.io","url":"https://docs.hydrozen.io/overview/misc/user-agent-and-ip-list","description":"Hydrozen web crawler for monitoring","tags":["monitoring"]},{"pattern":"HTTP Banner Detection","url":"https://security.ipip.net","description":"IPIP HTTP banner detection web crawler","tags":["scanner"]},{"pattern":"SummalyBot","url":"https://github.com/misskey-dev/summaly","description":"Summaly web crawler for content summarization","tags":["social-preview"]},{"pattern":"MicrosoftPreview\\/","url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","description":"Microsoft preview web crawler bot","tags":["social-preview"],"rdns":[".search.msn.com"]},{"pattern":"GeedoProductSearch","url":"https://geedo.com/product-search.html","description":"Geedo product search web crawler bot","tags":["search-engine"]},{"pattern":"TikTokSpider","description":"TikTok web crawler for content discovery","tags":["ai-crawler","social-preview"],"rdns":[".bytedance.com",".tiktokv.com"]},{"pattern":"OnCrawl\\/","url":"https://www.oncrawl.com/","description":"OnCrawl SEO web crawler for analysis","tags":["seo"],"rdns":[".oncrawl.com"]},{"pattern":"sindresorhus\\/got","url":"https://github.com/sindresorhus/got","description":"Got HTTP client library for requests","tags":["http-library"]},{"pattern":"CensysInspect\\/","url":"https://about.censys.io","description":"Censys web crawler for security scanning","tags":["scanner"]},{"pattern":"SBIntuitionsBot\\/","url":"https://www.sbintuitions.co.jp/bot/","description":"SB Intuitions web crawler bot","tags":["ai-crawler"]},{"pattern":"sitebulb","url":"https://sitebulb.com/","description":"Sitebulb SEO web crawler for analysis","tags":["seo"],"rdns":[".sitebulb.com"]},{"pattern":"YextBot\\/","url":"https://hitchhikers.yext.com/guides/create-crawler-connector/","description":"Yext web crawler for site crawling","tags":["seo"]},{"pattern":"DatadogSynthetics","url":"https://docs.datadoghq.com/synthetics/","description":"Datadog synthetics web crawler bot","tags":["monitoring"],"rdns":[".datadog.com",".datadoghq.com"]},{"pattern":"Google-Ads-Conversions","url":"https://developers.google.com/google-ads/api/docs/conversions/upload-online","description":"Google Ads conversions web crawler","tags":["advertising"]},{"pattern":"ObservePoint","url":"https://help.observepoint.com/en/articles/9101465-allow-exclude-observepoint-traffic#h_2a8176c9b9","description":"ObservePoint web crawler for monitoring","tags":["monitoring"]},{"pattern":"Checkly","url":"https://www.checklyhq.com/docs/","description":"Checkly web crawler for monitoring","tags":["monitoring"],"rdns":[".checkly.com",".checklyhq.com"]},{"pattern":"ALittle Client","url":"https://udger.com/resources/ua-list/bot-detail?bot=ALittle+Client","description":"ALittle web crawler for content discovery","tags":["http-library"]},{"pattern":"AliyunSecBot","url":"https://helpcenter.alibaba.com/s/buyer","description":"Alibaba Aliyun security web crawler bot","tags":["scanner"]},{"pattern":"Claude-Web","url":"https://www.anthropic.com/","description":"Anthropic Claude web crawler bot","tags":["ai-crawler"],"rdns":[".anthropic.com"]},{"pattern":"anthropic-ai","url":"https://www.anthropic.com/","description":"Anthropic AI web crawler bot","tags":["ai-crawler"],"rdns":[".anthropic.com"]},{"pattern":"Claude-User","url":"https://useragents.io/uas/mozilla-5-0-applewebkit-537-36-khtml-like-gecko-compatible-claudebot-1-0-supportanthropic-com_954fa13a8e1e46d8267fb56e2d48100e","description":"Anthropic Claude user web crawler bot","tags":["ai-fetcher"],"rdns":[".anthropic.com"]},{"pattern":"Claude-SearchBot","url":"https://useragents.io/uas/mozilla-5-0-applewebkit-537-36-khtml-like-gecko-compatible-claudebot-1-0-supportanthropic-com_954fa13a8e1e46d8267fb56e2d48100e","description":"Anthropic Claude search web crawler bot","tags":["ai-fetcher","search-engine"],"rdns":[".anthropic.com"]},{"pattern":"Google-Extended","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/overview-google-crawlers","description":"Google extended web crawler bot","tags":["ai-crawler"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"cohere-ai","url":"https://cohere.com","description":"Cohere AI web crawler bot","tags":["ai-crawler"],"rdns":[".cohere.ai",".cohere.com"]},{"pattern":"Timpibot","url":"https://timpi.io","description":"Timpi web crawler for content discovery","tags":["search-engine"],"rdns":[".timpi.io"]},{"pattern":"SERankingBacklinksBot","url":"https://help.seranking.com/hc/en-us/articles/23020353159964-SERankingBacklinksBot-Crawler","description":"SEranking backlinks web crawler bot","tags":["seo"],"rdns":[".seranking.com",".sr-srv.net"]},{"pattern":"CMSChecker","description":"CMS Checker web crawler for detection","tags":["seo"]},{"pattern":"Playwright","url":"https://playwright.dev","description":"Playwright browser automation web crawler","tags":["browser-automation"],"rdns":[".playwright.dev"]},{"pattern":"Puppeteer","url":"https://pptr.dev","description":"Puppeteer browser automation web crawler","tags":["browser-automation"],"rdns":[".pptr.dev",".puppeteer.dev"]},{"pattern":"Selenium","url":"https://www.selenium.dev","description":"Selenium browser automation web crawler","tags":["browser-automation"],"rdns":[".selenium.dev"]},{"pattern":"Nikto","url":"http://cirt.net/nikto/","description":"Nikto web server security scanner bot","tags":["scanner"],"rdns":[".cirt.net",".nikto.pl"]},{"pattern":"sqlmap","url":"https://sqlmap.org","description":"SQLMap SQL injection testing web crawler","tags":["scanner"]},{"pattern":"ZmEu","url":"https://en.wikipedia.org/wiki/ZmEu_(vulnerability_scanner)","description":"ZmEu vulnerability scanner web crawler","tags":["scanner"]},{"pattern":"masscan","url":"https://github.com/robertdavidgraham/masscan","description":"Masscan network scanner web crawler bot","tags":["scanner"]},{"pattern":"WPScan","url":"https://wpscan.com","description":"WPScan WordPress security scanner bot","tags":["scanner"],"rdns":[".wpscan.com",".wpscan.org"]},{"pattern":"[aA]cunetix","url":"https://www.acunetix.com","description":"Acunetix web vulnerability scanner bot","tags":["scanner"],"rdns":[".acunetix.com"]},{"pattern":"Nessus","url":"https://www.tenable.com/products/nessus","description":"Nessus vulnerability scanner web crawler","tags":["scanner"],"rdns":[".tenable.com"]},{"pattern":"[dD]ir[Bb]uster","url":"https://github.com/KajanM/DirBuster","description":"DirBuster directory scanner web crawler","tags":["scanner"]},{"pattern":"StatusCake","url":"https://www.statuscake.com","description":"StatusCake uptime monitoring web crawler","tags":["monitoring"],"rdns":[".statuscake.com"]},{"pattern":"colly","url":"https://go-colly.org","description":"Colly Go web crawler framework","tags":["http-library"]},{"pattern":"[mM]echanize","url":"https://github.com/sparklemotion/mechanize","description":"Mechanize Ruby web crawler library","tags":["http-library"]},{"pattern":"air\\.ai\\/scanning","description":"Air.ai web crawler for scanning","tags":["scanner"]},{"pattern":"asnriskscorer","description":"ASN Risk Scorer web crawler bot","tags":["scanner"]},{"pattern":"OICrawler","url":"https://chat.openindex.ai/","description":"OpenIndex web crawler for indexing","tags":["search-engine"]},{"pattern":"l9scan","url":"https://github.com/LeakIX/l9scan","description":"L9scan web crawler for scanning","tags":["scanner"]},{"pattern":"SlaccaleBot","description":"Slaccale web crawler bot","tags":["seo"]},{"pattern":"CustomAsyncHttpClient","description":"Custom async HTTP client web crawler","tags":["http-library"]},{"pattern":"^HTTPie\\/","url":"https://httpie.io","description":"HTTPie command-line HTTP client tool","tags":["http-library"]},{"pattern":"Gemini-Deep-Research","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers","description":"Google Gemini deep research web crawler","tags":["ai-fetcher","search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Perplexity-User","url":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","description":"Perplexity user web crawler bot","tags":["ai-fetcher","search-engine"],"rdns":[".perplexity.ai"]},{"pattern":"PerplexityUser","url":"https://perplexity.ai","description":"Perplexity AI web crawler bot","tags":["ai-fetcher","search-engine"]},{"pattern":"meta-webindexer","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers#meta-webindexer","description":"Meta web indexer bot for Facebook and Instagram content crawling","tags":["social-preview"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"DuckAssistBot","url":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot","description":"DuckDuckGo assistant bot for web crawling and search","tags":["search-engine","ai-fetcher"],"rdns":[".duckduckgo.com"]},{"pattern":"MistralAI-User","description":"Mistral AI web crawler bot for content indexing","tags":["ai-fetcher"],"rdns":[".mistral.ai"]},{"pattern":"webzio","url":"https://webz.io/blog/company/from-omgilibot-to-the-webzbot-duo-a-powerful-leap-for-ethical-and-comprehensive-data-collection/#","description":"Webz.io web crawler bot for ethical data collection","tags":["seo"],"rdns":[".omgili.com",".webz.io"]},{"pattern":"newsai\\/","url":"https://knownagents.com/agents/newsai","description":"NewsAI crawler for news content aggregation and indexing","tags":["ai-crawler","feed-reader"]},{"pattern":"008\\/","url":"https://datadome.co/bots/008-2/","description":"SEO monitoring bot tracks keyword rankings across regions","tags":["seo"]},{"pattern":"monitoring360bot\\/","url":"https://app.360monitoring.com/bot.html","description":"Monitors website uptime and performance at intervals","tags":["monitoring"]},{"pattern":"AccessStatus","url":"https://accessstatus.com/bot-info","description":"Evaluates websites for accessibility compliance standards","tags":["scanner"]},{"pattern":"ActiveComply","url":"https://activecomply.com/social-media-compliance-monitoring/","description":"Compliance monitoring for regulated industries","tags":["scanner"]},{"pattern":"AdagioBot","url":"https://datadome.co/bots/adagio-digital/","description":"SEO analysis bot for performance and optimization","tags":["seo"]},{"pattern":"adbeat\\.com","url":"https://www.adbeat.com/operation_policy","description":"Competitive intelligence bot for digital ad tracking","tags":["advertising"]},{"pattern":"AdminLabs","url":"https://datadome.co/bots/adminlabs/","description":"Web scraping tool for data collection and analysis","tags":["seo"]},{"pattern":"advanced_crawler","url":"https://datadome.co/bots/advanced-crawler/","description":"Sophisticated crawler with dynamic content rendering","tags":["seo"]},{"pattern":"Adventurer","url":"https://datadome.co/bots/adventurer/","description":"Web crawler for indexing content and analysis","tags":["seo"]},{"pattern":"AGAKIDSBOT","url":"https://agakids.ru/project/","description":"Web crawler for indexing and data gathering","tags":["seo"]},{"pattern":"AgencyAnalyticsBot","url":"https://agencyanalytics.com/features/seo-site-audit","description":"SEO audit bot for performance and backlink analysis","tags":["seo"]},{"pattern":"AI2Bot","url":"https://datadome.co/bots/ai2bot/","description":"Academic research crawler for NLP dataset creation","tags":["academic","ai-crawler"],"rdns":[".ai2.org",".allenai.org"]},{"pattern":"AkismetBot","url":"https://akismet.com/developers/getting-started/","description":"Spam filtering service for WordPress and CMS","tags":["scanner"]},{"pattern":"alexa site audit","url":"https://alexa.amazon.com/about","description":"Web crawler for SEO data aggregation and analysis","tags":["seo"]},{"pattern":"Algolia Crawler","url":"https://www.algolia.com/doc","description":"Hosted search engine and site search crawler","tags":["search-engine"]},{"pattern":"alienfarm","url":"https://datadome.co/bots/alienfarm/","description":"Web crawler for data aggregation and intelligence","tags":["seo"]},{"pattern":"AllAfrica","url":"https://allafrica.com/tools/","description":"Content aggregator for African news coverage","tags":["seo"]},{"pattern":"allOrigins","url":"https://allorigins.win/","description":"Web scraping proxy tool bypassing CORS restrictions","tags":["seo"]},{"pattern":"AmazonAdBot","url":"https://advertising.amazon.com/resources/","description":"Ad verification bot for delivery and impressions","tags":["advertising"]},{"pattern":"KendraBot","url":"https://docs.aws.amazon.com/kendra/latest/dg/what-is-kendra.html","description":"AI-powered enterprise search crawler for documents","tags":["search-engine","ai-crawler"]},{"pattern":"AntBot","url":"https://datadome.co/bots/antbot/","description":"Web crawler for data aggregation and research","tags":["seo"]},{"pattern":"AppSiteAssociation","url":"https://developer.apple.com/documentation/applications/allowing-app-linking-to-your-website","description":"Verifies app connection to website domains","tags":["seo"]},{"pattern":"Aragog\\/","url":"https://wordads.co/","description":"Web crawler for indexing and data collection","tags":["seo"]},{"pattern":"Aranea","url":"http://unesco.uniba.sk/guest/","description":"Automated web crawler for content aggregation","tags":["academic"]},{"pattern":"ArchiveBox","url":"https://archivebox.io/","description":"Web archiving tool for long-term content preservation","tags":["archiver"]},{"pattern":"ArquivoBot","url":"https://arquivo.pt/about","description":"Preserves Portuguese websites historical content","tags":["archiver"]},{"pattern":"Arquivo-web-crawler","url":"https://arquivo.pt/robot","description":"Portuguese web archive for digital heritage","tags":["archiver"]},{"pattern":"ArtemisBot","url":"https://datadome.co/bots/artemis-web-reader/","description":"Vulnerability scanning and cybersecurity research bot","tags":["scanner"]},{"pattern":"Asana\\/","url":"https://datadome.co/bots/asana-crawler/","description":"Work management platform metadata retriever bot","tags":["seo"]},{"pattern":"Attracta","url":"https://www.attracta.com/seo-tools","description":"SEO tools and performance insights crawler","tags":["seo"]},{"pattern":"AudistoBot","url":"https://audisto.com/help/crawler/bot/","description":"Technical SEO and website audit bot","tags":["seo"]},{"pattern":"Authory","url":"https://authory.com/profile-unavailable","description":"Content archiving bot for authors and journalists","tags":["archiver"]},{"pattern":"Autoconfig Test from USTC","url":"https://datadome.co/bots/autoconfig-test-from-ustc/","description":"USTC university research crawler for web analysis","tags":["academic"]},{"pattern":"AutomatonBot","url":"https://datadome.co/bots/automaton/","description":"Automation platform for web service workflows","tags":["seo"]},{"pattern":"tracking-quality-spider","url":"https://datadome.co/bots/awin-com-crawler/","description":"Affiliate marketing network tracking and verification","tags":["advertising"]},{"pattern":"Bad Neighborhood Header Detector","url":"https://datadome.co/bots/bad-neighborhood/","description":"Website security risk analysis for link quality","tags":["scanner"]},{"pattern":"BaiduAdsBot","url":"https://datadome.co/bots/baidu-ads-server-proxy/","description":"Baidu ad verification and tracking bot","tags":["advertising"]},{"pattern":"BDBot\\/","url":"https://datadome.co/bots/bdbot/","description":"Web crawler for data aggregation and analysis","tags":["seo"]},{"pattern":"BeeperBot","url":"https://www.beeper.com/","description":"Web crawler for aggregation and business intelligence","tags":["seo"]},{"pattern":"BestChange","url":"https://www.bestchange.com/bot/","description":"Cryptocurrency monitoring and exchange rate aggregator","tags":["seo"]},{"pattern":"BetterUptimeBot","url":"https://betteruptime.com/docs","description":"Website uptime and performance monitoring service","tags":["monitoring"]},{"pattern":"BnFBot","url":"https://www.bnf.fr/en/web-services","description":"French national library archiving cultural content","tags":["archiver"]},{"pattern":"BigUpDataBot","url":"https://datadome.co/bots/bigupdata/","description":"Analytics crawler for SEO and metric tracking","tags":["seo"]},{"pattern":"BinaryCanary","url":"https://www.binarycanary.com/monitoring/","description":"Uptime monitoring for servers and services","tags":["monitoring"]},{"pattern":"Bitbucket-Webhooks","url":"https://support.atlassian.com/bitbucket-cloud","description":"Webhook notifications for CI/CD pipeline automation","tags":["seo"]},{"pattern":"bl\\.uk_ldfc_bot","url":"https://www.bl.uk/legal-deposit/web-archiving","description":"British Library web archiver for content preservation","tags":["archiver"]},{"pattern":"BlackDuck-FD","url":"https://www.blackduck.com","description":"Vulnerability scanner for application security testing","tags":["scanner"]},{"pattern":"BlingERP","url":"https://manuais.bling.com.br/hc/pt-br/articles/360046302354","description":"Brazilian ERP platform for invoicing and inventory","tags":["seo"]},{"pattern":"Blockaid","url":"https://blockaid.io/product","description":"Security scanning for phishing and malware threats","tags":["scanner"]},{"pattern":"Blogtrottr","url":"https://datadome.co/bots/blogtrottr/","description":"Email subscription service for blog and feed updates","tags":["feed-reader"]},{"pattern":"BlogVault","url":"https://blogvault.net/malware-scanner/","description":"WordPress backup and malware scanning service","tags":["scanner"]},{"pattern":"BlueskyPreviewBot","url":"https://docs.bsky.app","description":"Bluesky social platform link preview fetcher","tags":["social-preview"]},{"pattern":"BoardGamePricesBot","url":"https://datadome.co/bots/boardgameprices/","description":"Board game price comparison and availability tracker","tags":["seo"]},{"pattern":"BotPoke","url":"https://datadome.co/bots/botpoke/","description":"Web crawler for data aggregation and analysis","tags":["seo"]},{"pattern":"BDFetch","url":"http://www.brandprotect.com/","description":"Brand protection bot for unauthorized use detection","tags":["seo"]},{"pattern":"Brandwatch","url":"https://www.brandwatch.com/legal/crawlers/","description":"Social listening and sentiment analysis platform","tags":["seo"]},{"pattern":"BraveBot","url":"https://search.brave.com/help/web-discovery-project","description":"Privacy-focused search engine indexing crawler","tags":["search-engine"],"rdns":[".brave.com"]},{"pattern":"brokenlinkcheck\\.com","url":"https://datadome.co/bots/brokenlinkcheck-com/","description":"Broken link detection and website integrity checker","tags":["seo"]},{"pattern":"bugsnag-crawler","url":"https://docs.bugsnag.com","description":"Application monitoring and error reporting bot","tags":["monitoring"]},{"pattern":"BW\\/","url":"https://builtwith.com/biup","description":"Technology profiling crawler for web analytics","tags":["seo"]},{"pattern":"Bushbaby","url":"https://datadome.co/bots/bushbaby/","description":"Content aggregation and market intelligence crawler","tags":["seo"]},{"pattern":"Butterfly","url":"http://labs.topsy.com/butterfly/","description":"Experimental data collection crawler","tags":["seo"]},{"pattern":"rss-parser","url":"https://buttondown.com/about","description":"Newsletter RSS feed parser and aggregator","tags":["feed-reader"]},{"pattern":"CaliberBot","url":"https://calibermind.com/platform/","description":"Market research and analytics crawler","tags":["seo"]},{"pattern":"CapitalOneShopping","url":"https://datadome.co/bots/capital-one-shopping-bot/","description":"E-commerce price comparison and shopping bot","tags":["seo"]},{"pattern":"Catchpoint","url":"https://catchpoint.com/bots","description":"Digital experience and performance monitoring bot","tags":["monitoring"]},{"pattern":"centuryb\\.o\\.t9","url":"https://datadome.co/bots/centurybot9/","description":"General web crawler for data aggregation","tags":["seo"]},{"pattern":"CertChief","url":"https://datadome.co/bots/cert-chief-bot/","description":"SSL/TLS certificate monitoring and auditing","tags":["scanner"]},{"pattern":"CERT PL","url":"https://cert.pl/skanowanie/","description":"Polish cybersecurity vulnerability and threat scanner","tags":["scanner"]},{"pattern":"certytags","url":"https://certybot.certytags.com/","description":"Web content indexing and verification crawler","tags":["seo"]},{"pattern":"ChannableBot","url":"https://help.channable.com","description":"E-commerce feed management and product data","tags":["seo"]},{"pattern":"ChargeBeeBot","url":"https://www.chargebee.com:443/resources","description":"Subscription billing verification and integration bot","tags":["seo"]},{"pattern":"Charlotte","url":"https://datadome.co/bots/charlotte-bot/","description":"Financial institution and robot-friendly crawler","tags":["seo"]},{"pattern":"ChatGLM-Spider","url":"https://chatglm.cn/","description":"AI model training data collection crawler","tags":["ai-crawler"],"rdns":[".zhipuai.cn",".zhipuai.com"]},{"pattern":"Chatwork LinkPreview","url":"https://www.chatwork.com/","description":"Collaboration platform URL preview fetcher","tags":["social-preview"]},{"pattern":"CheckHost","url":"https://datadome.co/bots/check-host/","description":"Website uptime and performance monitoring bot","tags":["monitoring"]},{"pattern":"Goodzer","url":"https://discord.com/discovery/applications/1065250549408223252","description":"Discord bot enhancement and server functionality crawler","tags":["seo"]},{"pattern":"Chrome Privacy Preserving Prefetch Proxy","url":"https://datadome.co/bots/chrome-privacypreserving-prefetch-proxy/","description":"Chrome browser resource prefetching with privacy protection","tags":["seo"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"CirrusExplorer","url":"https://cseu.ro/explorer.php","description":"Web crawler for indexing and competitive analysis","tags":["seo"]},{"pattern":"CLASSLA-web","url":"https://www.clarin.si/info/classla-web-crawler/","description":"Linguistic research data crawler for NLP","tags":["academic"]},{"pattern":"Clearscopebot","url":"https://datadome.co/bots/clearscope-clearscopebot/","description":"Content optimization and keyword performance analysis","tags":["seo"]},{"pattern":"WorldBot","url":"https://datadome.co/bots/clickagy-intelligence-bot/","description":"Behavioral analytics and audience segmentation crawler","tags":["seo"]},{"pattern":"Cloudflare-Validator","url":"https://datadome.co/bots/cloudflare-crawler/","description":"Cloudflare security and performance validation","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"cloudflare-csup","url":"https://datadome.co/bots/cloudflare-csup/","description":"Cache status updates and performance monitoring","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Custom-Error-Page-Crawler","url":"https://developers.cloudflare.com","description":"Custom error page validation and testing","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Radar-Scanner","url":"https://radar.cloudflare.com","description":"URL scanning for traffic and threat intelligence","tags":["scanner"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-SpeedTest","url":"https://www.cloudflare.com/speedtest","description":"Website performance and latency testing bot","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Stream-Hook","url":"https://developers.cloudflare.com/stream/manage-video-library/using-webhooks/","description":"Video lifecycle event notifications and management","tags":["seo"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudtrellis","url":"https://cloudtrellis.com/bots","description":"Infrastructure and performance monitoring bot","tags":["monitoring"]},{"pattern":"CludoBot","url":"https://www.cludo.com/docs/bot/","description":"Site search and analytics bot","tags":["search-engine"]},{"pattern":"cognitiveSEO Bot","url":"https://cognitiveseo.com/bot","description":"SEO analytics and digital marketing crawler","tags":["seo"],"rdns":[".cognitiveseo.com"]},{"pattern":"cohere-training-data-crawler","url":"https://datadome.co/bots/cohere-training-data-crawler/","description":"LLM training data collection for AI models","tags":["ai-crawler"],"rdns":[".cohere.ai",".cohere.com"]},{"pattern":"CommaFeed","url":"https://datadome.co/bots/commafeed/","description":"RSS feed aggregator and reader bot","tags":["feed-reader"]},{"pattern":"researchscan\\.comsys\\.rwth-aachen\\.de","url":"https://researchscan.comsys.rwth-aachen.de/","description":"Academic network measurement and analysis","tags":["academic"]},{"pattern":"contentkingapp","url":"https://whatis.conductor.com/","description":"Real-time SEO auditing and monitoring tool","tags":["seo"]},{"pattern":"Convermax","url":"https://convermax.com/auto-parts-search","description":"Conversion and digital marketing optimization","tags":["seo"]},{"pattern":"CookieHub Bot","url":"https://www.cookiehub.com/docs","description":"Cookie consent and privacy compliance auditor","tags":["scanner"]},{"pattern":"cookie-maestro","url":"https://cookiemaestro.com/docs","description":"Cookie tracking and privacy regulation compliance","tags":["scanner"]},{"pattern":"Cotoyogi","url":"https://ds.rois.ac.jp/center8/crawler/","description":"Web crawler for content and data aggregation","tags":["seo"]},{"pattern":"Coveobot","url":"https://platform.cloud.coveo.com/admin/","description":"Enterprise search and content indexing platform","tags":["search-engine"]},{"pattern":"Crawlson","url":"https://www.crawlson.com/","description":"Web crawler for auditing and monitoring","tags":["seo"]},{"pattern":"Crazy Egg","url":"https://support.crazyegg.com/hc/en-us/articles/22650794175251","description":"Analytics bot for heatmaps and A/B testing","tags":["seo"]},{"pattern":"RepoLookoutBot","url":"https://www.repo-lookout.org/","description":"Repository and web content lookup crawler","tags":["seo"]},{"pattern":"Criticalcss\\.com","url":"https://criticalcss.com/","description":"Critical CSS extraction for performance optimization","tags":["seo"]},{"pattern":"cron-job\\.org","url":"https://cron-job.org/en/","description":"Automated website monitoring and uptime checking","tags":["monitoring"]},{"pattern":"DnBCrawler","url":"https://datadome.co/bots/dnbcrawler/","description":"Business data analytics crawler gathering company information insights","tags":["seo"]},{"pattern":"DMBrowser/","url":"https://www.dotcom-monitor.com/wiki/knowledge-base-main/","description":"Synthetic monitoring browser checking website performance and availability","tags":["monitoring"]},{"pattern":"DomCopBot","url":"https://www.domcop.com/bot","description":"Domain research crawler analyzing website metrics and SEO data","tags":["seo"]},{"pattern":"downnotifier\\.com","url":"https://datadome.co/bots/downnotifier-com-monitoring/","description":"Uptime monitoring service alerting on website downtime events","tags":["monitoring"]},{"pattern":"DowntimeDetector\\/","url":"https://datadome.co/bots/downtimedetector/","description":"Availability monitor detecting and reporting website downtime incidents","tags":["monitoring"]},{"pattern":"Dlc\\/","url":"https://www.drlinkcheck.com/","description":"Link checking crawler auditing broken URLs across websites","tags":["seo"]},{"pattern":"Dratabot","url":"https://help.drata.com/en/articles/5411182-dratabot","description":"Compliance automation crawler auditing security controls and documentation","tags":["scanner"]},{"pattern":"EasyBib AutoCite","url":"https://www.easybib.com/","description":"Citation generator fetching metadata for bibliography and reference creation","tags":["academic"]},{"pattern":"easybill-ImportManager","url":"https://www.easybill.de/api/","description":"Billing platform crawler importing product data from ecommerce stores","tags":["seo"]},{"pattern":"EasyCron\\/","url":"https://www.easycron.com","description":"Cron job service executing scheduled HTTP requests and monitoring endpoints","tags":["monitoring"]},{"pattern":"easyDNS Monitoring","url":"http://easyurl.net/monitoring","description":"DNS monitoring service verifying domain resolution and availability","tags":["monitoring"]},{"pattern":"EchoboxBot\\/","url":"https://www.echobox.com/","description":"Social media automation crawler fetching content for publishing optimization","tags":["seo"]},{"pattern":"Cronless","url":"https://datadome.co/bots/cronless/","description":"Dynamic task execution crawler without fixed scheduling","tags":["monitoring"]},{"pattern":"crusty\\/","url":"https://github.com/let4be/crusty","description":"Undocumented web crawler of unknown origin","tags":["seo"]},{"pattern":"csirt\\.cz","url":"https://csirt.cz/cs/dns-crawler/","description":"Czech cybersecurity team vulnerability scanning bot","tags":["scanner"]},{"pattern":"CXK_Bot","url":"https://datadome.co/bots/cxk_bot/","description":"Web content indexing and SEO analysis crawler","tags":["seo"]},{"pattern":"Dark Visitor Client","url":"https://datadome.co/bots/dark-visitor-client/","description":"Headless bot detection helper client","tags":["scanner"]},{"pattern":"Dark Visitor Server","url":"https://datadome.co/bots/dark-visitor-server/","description":"Developer helper for bot detection service","tags":["scanner"]},{"pattern":"daumoa","url":"http://cs.daum.net/faq/15/4118.html?faqId=28966","description":"South Korean Daum search engine indexing bot","tags":["search-engine"],"rdns":[".daum.net",".daumcorp.com"]},{"pattern":"DaspeedBot","url":"https://datadome.co/bots/dawap-bot/","description":"Web content indexing and analytics crawler","tags":["seo"]},{"pattern":"Dead Link Checker","url":"http://www.dead-link-checker.com/","description":"Broken link detection and website maintenance bot","tags":["seo"]},{"pattern":"Deskyobot","url":"https://www.deskyo.com/bot","description":"Website content indexing and SEO analysis","tags":["seo"]},{"pattern":"Detectify","url":"https://detectify.com/","description":"Vulnerability and attack surface scanning bot","tags":["scanner"]},{"pattern":"Devin","url":"https://docs.devin.ai/get-started/devin-intro","description":"AI-driven browser automation and content extraction","tags":["browser-automation","ai-fetcher"]},{"pattern":"DF Bot","url":"https://datadome.co/bots/df-bot/","description":"Web content indexing and competitive analysis","tags":["seo"]},{"pattern":"DigiCert DCV","url":"https://www.digicert.com/","description":"SSL/TLS domain control validation checker","tags":["scanner"]},{"pattern":"DingTalkBot-LinkService","url":"https://www.dingtalk.com/","description":"Alibaba DingTalk link preview and metadata fetcher","tags":["social-preview"]},{"pattern":"Discourse Forum Onebox","url":"https://discourse.org/","description":"Forum link preview and embed content fetcher","tags":["social-preview"]},{"pattern":"Dmbot","url":"https://datadome.co/bots/dmbot/","description":"Web crawler for data aggregation and SEO","tags":["seo"]},{"pattern":"SustainabilityCrawler","url":"https://datadome.co/bots/ecovadis-bot/","description":"ESG and sustainability rating data collection","tags":["seo"]},{"pattern":"edansbot","url":"https://datadome.co/bots/edansbot/","description":"Web content indexing and analytics crawler","tags":["seo"]},{"pattern":"EdgeWatch","url":"https://about.edgewatch.com/","description":"Performance monitoring and security assessment bot","tags":["monitoring"]},{"pattern":"Do Not Track Verifier","url":"https://datadome.co/bots/eff-crawler/","description":"Privacy practices and security protocol analyzer","tags":["scanner"]},{"pattern":"elmahio-uptimebot","url":"https://elmah.io","description":"Error logging uptime and availability monitoring","tags":["monitoring"]},{"pattern":"eMoneyBot","url":"https://emoneyadvisor.com","description":"Financial account aggregation and data scraping","tags":["seo"]},{"pattern":"EpivozCrawler","url":"https://www.techmeme.com","description":"Content scraping and indexing crawler","tags":["seo"]},{"pattern":"eRepublik\\.tools","url":"https://erepublik.tools","description":"MMO game data and analytics crawler","tags":["seo"]},{"pattern":"EvoUptimeBot","url":"https://evo.agency/","description":"Uptime and performance monitoring service","tags":["monitoring"]},{"pattern":"ExodusMovement","url":"https://www.exodus.com/","description":"Web scraping and data extraction bot","tags":["seo"]},{"pattern":"Ezgif","url":"https://ezgif.com/about","description":"GIF and image fetching and editing bot","tags":["seo"]},{"pattern":"factset_spyderbot","url":"https://www.factset.com/","description":"Financial data and regulatory filing crawler","tags":["seo"]},{"pattern":"FastmailUA","url":"https://www.fastmail.com/policies/bots/","description":"Email link preview and image proxy fetcher","tags":["social-preview"]},{"pattern":"FDL Stats Bot","url":"https://ftwentertainment.com","description":"Firebase dynamic link analytics and validation","tags":["seo"]},{"pattern":"Fedicabot","url":"https://fedica.com/info/fedicabot","description":"Social media analytics and link preview bot","tags":["seo"]},{"pattern":"FedReporterDataBot","url":"https://fedreporter.net/FedReporterBotDocumentation/Readme.txt","description":"Financial and regulatory filing data aggregator","tags":["seo"]},{"pattern":"Feed Image Audit","url":"https://image-validator.com/","description":"Image validation and compliance auditing","tags":["scanner"]},{"pattern":"FeedBurner","url":"http://www.feedburner.com/","description":"RSS feed distribution and subscriber management","tags":["feed-reader"]},{"pattern":"feeder\\.co","url":"https://feeder.co/crawler","description":"RSS feed aggregation and monitoring service","tags":["feed-reader"]},{"pattern":"Feedpresso Content Index Bot","url":"https://datadome.co/bots/feedpresso-crawler/","description":"News aggregation platform content indexer","tags":["feed-reader"]},{"pattern":"Feedwind","url":"http://feed.mikle.com/support/description/","description":"Dynamic content aggregation widget fetcher","tags":["feed-reader"]},{"pattern":"fidget-spinner-bot","url":"https://datadome.co/bots/fidget-spinner-bot/","description":"Content indexing and SEO analysis crawler","tags":["seo"]},{"pattern":"FirmoGraph","url":"https://datadome.co/bots/firmograph/","description":"Business data collection and analytics crawler","tags":["seo"]},{"pattern":"FlipboardRSS","url":"https://about.flipboard.com/proxy-service/","description":"Social magazine content aggregation and curation","tags":["feed-reader"]},{"pattern":"Foregenix","url":"https://www.turacolabs.com/product","description":"Cybersecurity vulnerability and weakness scanner","tags":["scanner"]},{"pattern":"Freespoke\\/","url":"https://docs.freespoke.com/search/bot/","description":"Privacy-focused search engine indexing web content for results","tags":["search-engine"]},{"pattern":"Friendly testing bot","url":"https://datadome.co/bots/friendly-testing-bot/","description":"Testing crawler verifying website functionality and response behavior","tags":["seo"]},{"pattern":"friendly-spider","url":"https://datadome.co/bots/friendly-spider/","description":"Web crawler indexing content for search and data aggregation","tags":["seo"]},{"pattern":"FriendlyCrawler\\/","url":"https://datadome.co/bots/friendlycrawler/","description":"General web crawler indexing content for data collection purposes","tags":["seo"]},{"pattern":"FullStoryBot\\/","url":"https://help.fullstory.com/spp-ref/343521-what-is-the-fullstorybot","description":"Analytics platform crawler collecting session and behavioral data","tags":["seo"]},{"pattern":"Funnelback","url":"https://docs.squiz.net/funnelback/docs/latest/","description":"Enterprise search crawler indexing content for internal search platforms","tags":["search-engine"]},{"pattern":"FuseonBot\\/","url":"https://datadome.co/bots/fuseonbot/","description":"Link affinity crawler indexing web content for SEO analysis","tags":["seo"]},{"pattern":"Gabanzabot\\/","url":"https://datadome.co/bots/gabanzabot/","description":"Search engine crawler indexing websites for Gabanza search results","tags":["search-engine"]},{"pattern":"gdnplus\\.com","url":"https://datadome.co/bots/gdnp-crawler/","description":"Data gathering crawler analyzing and providing web content insights","tags":["seo"]},{"pattern":"getthit\\.com","url":"https://www.getthit.com/bot","description":"Traffic exchange service bot generating visits to listed websites","tags":["seo"]},{"pattern":"GG PeekBot","url":"https://www.gg.pl/","description":"Polish messaging platform crawler generating link previews for users","tags":["social-preview"]},{"pattern":"Ghost Inspector","url":"https://ghostinspector.com/","description":"Browser testing service automating UI tests and monitoring website changes","tags":["browser-automation"],"rdns":[".ghostinspector.com"]},{"pattern":"github-camo","url":"https://github.com/atmos/camo","description":"GitHub image proxy fetching external images for secure rendering","tags":["social-preview"],"rdns":[".github.com",".githubapp.com",".githubusercontent.com"]},{"pattern":"GlobalWebSearch","url":"https://datadome.co/bots/globalwebsearch/","description":"Web crawler indexing content for global search engine results","tags":["search-engine"]},{"pattern":"Golfe\\/","url":"https://datadome.co/bots/6hphjxgx/","description":"UAE-based search engine crawler indexing web content for search","tags":["search-engine"]},{"pattern":"Google-Apps-Script","url":"https://developers.google.com/apps-script/","description":"Google scripting platform fetching URLs for automation workflows","tags":["seo"]},{"pattern":"GoogleStackdriverMonitoring","url":"https://cloud.google.com/monitoring","description":"Google Cloud monitoring service performing uptime checks on endpoints","tags":["monitoring"]},{"pattern":"GoogleAssociationService\\/","url":"https://developers.google.com/identity/credential-sharing/digital-asset-links#:~:text=then%20act%20upon.-,Overview,as%20location%2C%20with%20website%20B.","description":"Google service verifying digital asset links between apps and websites","tags":["seo"]},{"pattern":"GoogleImageProxy","url":"https://support.google.com/webmasters/answer/1061943?hl=en","description":"Google image proxy fetching and caching images for Gmail display","tags":["social-preview"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"GoogleProducer","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Google user-triggered fetcher retrieving content for Google products","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Googlebot-IA\\/","url":"https://scholar.google.com/intl/en/scholar/libraries.html","description":"Google Scholar crawler indexing academic papers and library content","tags":["search-engine","academic"]},{"pattern":"Google-Trust-Services\\/","url":"https://pki.goog/","description":"Google PKI service validating SSL certificates and checking revocation status","tags":["scanner"]},{"pattern":"Google-Area120","url":"https://area120.google.com/","description":"Google Area 120 experimental tool fetching privacy policies from websites","tags":["seo"]},{"pattern":"Google-CloudVertexBot","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers","description":"Google Vertex AI crawler indexing content for enterprise search solutions","tags":["search-engine","ai-crawler"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"GoogleAssociationService$","url":"https://developers.google.com/digital-asset-links","description":"Google service verifying digital asset link associations between platforms","tags":["seo"]},{"pattern":"GoogleDocs","url":"https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fdocs.google.com%2F&dsh=S286587930%3A1778177859467567&emr=1&followup=https%3A%2F%2Fdocs.google.com%2F&osid=1&passive=1209600&flowName=WebLiteSignIn&flowEntry=ServiceLogin&ifkv=AWa2PavW2qpRau4xtdtClrEBgyu9XeRedZwj8tzXCCpkwmnDNN9_7amqw4FzotUIZBRBAaLMYVlZ-w","description":"Google Docs crawler fetching web content for spreadsheet import functions","tags":["seo"]},{"pattern":"GoPay","url":"https://doc.gopay.com/","description":"Payment gateway service verifying merchant websites and payment endpoints","tags":["seo"]},{"pattern":"GotSiteMonitor\\.com","url":"https://datadome.co/bots/gotsitemonitor/","description":"Website monitoring service tracking uptime and alerting on downtime","tags":["monitoring"]},{"pattern":"synthetic-monitoring-agent\\/","url":"https://grafana.com/","description":"Grafana synthetic monitoring agent probing endpoints for availability","tags":["monitoring"],"rdns":[".grafana.com",".grafana.net"]},{"pattern":"Grammarly\\/","url":"https://datadome.co/bots/grammarly/","description":"Writing assistant crawler fetching web content for grammar analysis","tags":["seo"],"rdns":[".grammarly.com"]},{"pattern":"gregcrawler","url":"https://datadome.co/bots/gregcrawler/","description":"Web crawler indexing content for data aggregation and analysis","tags":["seo"]},{"pattern":"GroovinaAdsbot\\/","url":"https://www.groovinads.com/","description":"Advertising platform crawler indexing products and pricing for campaigns","tags":["advertising"]},{"pattern":"Grover\\/","url":"https://datadome.co/bots/grover-bot/","description":"Anonymous web crawler likely used for research or content scraping","tags":["seo"]},{"pattern":"GTmetrix","url":"https://gtmetrix.com/","description":"Performance testing tool analyzing page speed and optimization recommendations","tags":["monitoring"]},{"pattern":"GuestpostsBot\\/","url":"https://guestposts.com.br/","description":"Guest posting platform crawler discovering and indexing blog opportunities","tags":["seo"]},{"pattern":"Gulper Web Bot","url":"https://datadome.co/bots/gulperbot/","description":"Academic web crawler collecting data for research and link analysis","tags":["academic"]},{"pattern":"Verity\\/","url":"https://gumgum.com/contextual","description":"Contextual advertising crawler analyzing page content for brand safety","tags":["advertising"]},{"pattern":"HappyWing","url":"https://datadome.co/bots/happywing/","description":"Web crawler indexing content for data aggregation and analysis","tags":["seo"]},{"pattern":"harsilbot\\/","url":"http://www.harsil.com/bot","description":"Web crawler indexing and collecting data for SEO analysis purposes","tags":["seo"]},{"pattern":"HawaiiBot","url":"https://datadome.co/bots/hawaiibot/","description":"Web crawler gathering content for data aggregation and indexing","tags":["seo"]},{"pattern":"hCardValidator","url":"http://hcard.geekhood.net/","description":"Microformat validator checking hCard contact data on web pages","tags":["seo"]},{"pattern":"Hello World","url":"https://datadome.co/bots/hello-world/","description":"Test crawler sending generic requests to verify web server responses","tags":["seo"]},{"pattern":"HelloworkJobPostingBot\\/","url":"https://www.hellowork-group.com/en/","description":"Job board crawler indexing employment listings and posting data","tags":["seo"]},{"pattern":"HetrixTools","url":"https://hetrixtools.com/uptime-monitor-bot/","description":"Uptime monitoring bot checking server availability and alerting downtime","tags":["monitoring"]},{"pattern":"HIFIBot\\/","url":"https://hi.fi/","description":"Web crawler indexing content for search and information aggregation","tags":["seo"]},{"pattern":"Hlidam\\.to robot","url":"https://hlidam.to/","description":"Czech job listing crawler indexing employment offers and postings","tags":["seo"]},{"pattern":"Honeybadger Uptime Check","url":"https://www.honeybadger.io/","description":"Error monitoring service checking website availability and health","tags":["monitoring"]},{"pattern":"HostTracker\\/","url":"https://datadome.co/bots/hosttracker/","description":"Website monitoring service checking availability from multiple global locations","tags":["monitoring"]},{"pattern":"Hotjar","url":"https://www.hotjar.com/","description":"Analytics platform crawler collecting heatmap and behavior tracking data","tags":["seo"]},{"pattern":"hstspreload-bot","url":"https://hstspreload.org/","description":"HSTS preload list checker verifying HTTPS enforcement across websites","tags":["scanner"]},{"pattern":"Huckabot\\/","url":"https://huckabuy.com/","description":"E-commerce SEO crawler analyzing product pages for optimization insights","tags":["seo"]},{"pattern":"Hype Machine\\/","url":"https://hypem.com/latest","description":"Music blog aggregator crawler indexing tracks and artist content","tags":["seo"]},{"pattern":"Web Screen Service By hyperhost","url":"https://datadome.co/bots/hyperhost-ua-crawler/","description":"Web screenshot service capturing page visuals for hosting platform","tags":["seo"]},{"pattern":"AdsBot-IAB","url":"https://iabtechlab.com/ads-txt/","description":"IAB technology crawler verifying ads.txt files for programmatic advertising","tags":["advertising"]},{"pattern":"iAskBot\\/","url":"https://datadome.co/bots/iaskspider/","description":"AI search engine crawler indexing web content for question answering","tags":["search-engine","ai-crawler"]},{"pattern":"IBM Crawler","url":"https://datadome.co/bots/ibm-crawler/","description":"IBM enterprise crawler indexing content for Watson and cloud services","tags":["seo"]},{"pattern":"IFTTT\\/","url":"https://ifttt.com/feed","description":"Automation platform crawler fetching feeds for conditional workflow triggers","tags":["seo"]},{"pattern":"ImageFetcher\\/","url":"https://wsrv.nl/","description":"Image proxy service fetching and caching images for web applications","tags":["seo"]},{"pattern":"ImageMind","url":"https://datadome.co/bots/imagemind/","description":"Image analysis crawler collecting visual data for AI training purposes","tags":["ai-crawler"]},{"pattern":"img2dataset","url":"https://github.com/rom1504/img2dataset","description":"Open source tool downloading images for machine learning dataset creation","tags":["ai-crawler"]},{"pattern":"imgproxy\\/","url":"https://imgproxy.net/","description":"Image processing proxy resizing and optimizing images for web delivery","tags":["seo"]},{"pattern":"impendoom-bot\\/","url":"https://impendoom.com/","description":"Web crawler indexing content for competitive intelligence and data analysis","tags":["seo"]},{"pattern":"IndeedJobBot","url":"https://www.indeed.com/about/indeed-crawlers","description":"Job search engine crawler indexing employment listings and company pages","tags":["search-engine"]},{"pattern":"Innguma\\/","url":"https://factory.innguma.com/fetcher/","description":"Content fetcher crawler gathering web data for information extraction","tags":["seo"]},{"pattern":"Instapaper\\/","url":"https://www.instapaper.com/publishers","description":"Read-later service crawler fetching articles for offline reading storage","tags":["seo"],"rdns":[".instapaper.com"]},{"pattern":"Integromat\\/","url":"https://developers.make.com/api-documentation","description":"Automation platform crawler fetching web data for workflow integration","tags":["seo"]},{"pattern":"intelx\\.io_bot","url":"https://intelx.io/","description":"Intelligence search engine crawler indexing leaks and dark web content","tags":["search-engine"]},{"pattern":"internetVista monitor","url":"https://datadome.co/bots/internetvista-monitor/","description":"Website monitoring service checking availability performance and SSL status","tags":["monitoring"]},{"pattern":"Irokez\\.cz monitoring","url":"https://datadome.co/bots/irokez-cz-monitoring/","description":"Czech web monitoring service tracking availability and performance metrics","tags":["monitoring"]},{"pattern":"IsDownBot\\/","url":"https://help.isdown.app/website-monitors/isdownbot","description":"Downtime monitoring bot checking website and service availability status","tags":["monitoring"]},{"pattern":"ISSCyberRiskCrawler\\/","url":"https://datadome.co/bots/isscyberriskcrawler/","description":"Cybersecurity risk crawler scanning websites for vulnerabilities and exposures","tags":["scanner"]},{"pattern":"iubenda-radar\\/","url":"https://www.iubenda.com/en/","description":"Compliance platform crawler auditing privacy policies and cookie consent","tags":["scanner"]},{"pattern":"UptimeBot\\/","url":"https://www.jaggedpixel.co/","description":"Uptime monitoring service verifying website availability and response latency","tags":["monitoring"]},{"pattern":"jetmon\\/","url":"https://automattic.com/","description":"WordPress Jetpack site uptime monitor and health checking service","tags":["monitoring"]},{"pattern":"jobswithgptcom-bot","url":"https://jobswithgpt.com/bot.html","description":"Job portal crawler indexing career pages and employment listings","tags":["search-engine"]},{"pattern":"Jumio","url":"https://github.com/Jumio/implementation-guides/blob/master/netverify/callback.md","description":"Identity and KYC verification service web content analyzer","tags":["seo"]},{"pattern":"Kagibot\\/","url":"https://kagi.com/","description":"Privacy-focused search engine crawler indexing content for rankings","tags":["search-engine"],"rdns":[".kagi.com",".kagibot.com"]},{"pattern":"KangarooBot\\/","url":"https://datadome.co/bots/kangaroo-bot/","description":"Data scraping and content aggregation crawler mimicking human behavior","tags":["seo"]},{"pattern":"KargoBot-Artemis","url":"https://www.kargo.com/","description":"Ad-tech crawler analyzing pages for contextual targeting and brand safety","tags":["advertising"]},{"pattern":"kazbtbot\\/","url":"https://kazbt.com/","description":"Web crawler for content indexing and data aggregation purposes","tags":["seo"]},{"pattern":"keycdn-tools\\/","url":"https://tools.keycdn.com/","description":"CDN performance monitoring measuring website load times and metrics","tags":["monitoring"]},{"pattern":"keys-so-bot","url":"https://datadome.co/bots/keys-so-bot/","description":"Automated crawler for indexing and data collection purposes","tags":["seo"]},{"pattern":"kinsta-bot","url":"https://kinsta.com/","description":"Hosting platform crawler for cache warming and SEO audit checks","tags":["seo"]},{"pattern":"Klaviyo\\/","url":"https://www.klaviyo.com/de/","description":"Marketing automation bot fetching product data for email campaigns","tags":["seo"]},{"pattern":"Kukei\\.eu-Bot\\/","url":"https://kukei.eu/","description":"Web crawler indexing and analyzing page content for search purposes","tags":["seo"]},{"pattern":"LAC_IAHarvester","url":"https://www.canada.ca/en/library-archives/services/government/web-social-media-preservation-program.html","description":"Canadian library web archiving bot preserving government websites","tags":["archiver"]},{"pattern":"LastModBot\\/","url":"https://last-modified.com/","description":"Web crawler detecting page freshness by analyzing last modified headers","tags":["seo"]},{"pattern":"LegalMonster","url":"https://openli.com/","description":"Compliance monitoring scanner detecting cookies and tracking consent","tags":["scanner"]},{"pattern":"Let's Encrypt","url":"https://letsencrypt.org/","description":"SSL certificate validation bot verifying domain control for TLS certs","tags":["scanner"]},{"pattern":"Level9SearchBot\\/","url":"https://level9.com/","description":"Search and OSINT crawler indexing public content for search purposes","tags":["search-engine"]},{"pattern":"loc\\.gov\\/programs\\/web-archiving","url":"https://www.loc.gov/programs/web-archiving/for-site-owners","description":"Government web archiver preserving cultural content for long-term access","tags":["archiver"]},{"pattern":"LinerBot\\/","url":"https://docs.getliner.com/docs/linerbot","description":"AI workspace crawler indexing web content for search and discovery","tags":["ai-fetcher"]},{"pattern":"LinkTiger","url":"https://linktiger.com/","description":"Broken link detection crawler auditing website link integrity","tags":["seo"]},{"pattern":"LinkAce\\/","url":"https://www.linkace.org/","description":"Self-hosted bookmark archive crawler preserving web content","tags":["archiver"]},{"pattern":"LinksIndexerBot\\/","url":"https://linksindexer.com/bot","description":"SEO crawler indexing backlinks and submitting URLs for search engines","tags":["seo"]},{"pattern":"LinkWalker\\/","url":"https://datadome.co/bots/linkwalker/","description":"Automated web crawler for content and data collection purposes","tags":["seo"]},{"pattern":"LogicMonitor","url":"https://www.logicmonitor.com/support/about-logicmonitor/overview/logicmonitor-public-ip-addresses-dns-names","description":"Synthetic monitoring bot checking website and API availability","tags":["monitoring"]},{"pattern":"LoomlyBot","url":"https://www.loomly.com/","description":"Social media management metadata fetcher for link previews","tags":["social-preview"]},{"pattern":"Macrobondbot","url":"https://www.macrobond.com/","description":"Financial data crawler gathering macroeconomic time-series data","tags":["seo"]},{"pattern":"MADBbot\\/","url":"https://madb.zapto.org/bot.html","description":"Web crawler gathering data for analytics and research purposes","tags":["seo"]},{"pattern":"Magellan","url":"https://datadome.co/bots/magellan/","description":"Web content indexing and competitive analysis crawler tool","tags":["seo"]},{"pattern":"magicsearchdev\\/","url":"https://datadome.co/bots/magicsearchdev/","description":"Web crawler for content indexing and data aggregation purposes","tags":["seo"]},{"pattern":"Magnet\\.me-web\\/","url":"https://magnet.me/bot.html","description":"Graduate recruitment platform crawling job postings and employer data","tags":["seo"]},{"pattern":"MainWP\\/","url":"https://support.dshost.com.au/","description":"WordPress management platform crawler auditing child site health","tags":["seo"]},{"pattern":"Make\\/","url":"https://www.make.com/en/","description":"Automation platform crawler validating endpoints and collecting metadata","tags":["seo"]},{"pattern":"ManageWP","url":"https://managewp.com/","description":"WordPress dashboard scanner verifying site availability and health","tags":["seo"]},{"pattern":"MarketGoo\\/","url":"https://www.marketgoo.com/","description":"SEO audit spider evaluating technical search engine optimization signals","tags":["seo"]},{"pattern":"MarketingMiner","url":"https://datadome.co/bots/marketingminer-bot/","description":"Digital marketing analytics crawler gathering SEO and performance data","tags":["seo"]},{"pattern":"dbot\\)","url":"https://www.marsflag.com/ja/services/marsfinder/","description":"Link preview and content discovery metadata harvester bot","tags":["social-preview"]},{"pattern":"Mattermost-Bot\\/","url":"https://datadome.co/bots/mattermost-bot/","description":"Team communication platform fetching URLs for preview and automation","tags":["social-preview"]},{"pattern":"Mavifinds","url":"https://brandimi.com/mavifinds-bot/","description":"Shopping discovery crawler indexing product prices and availability","tags":["seo"]},{"pattern":"MB-LinkChecker","url":"https://www.marcobeierer.com/tools","description":"Website crawler for structure analysis and metadata collection","tags":["seo"]},{"pattern":"MedialogiaBot","url":"https://www.mlg.ru/","description":"Media monitoring crawler tracking news and brand mentions online","tags":["seo"]},{"pattern":"MediaMonitoringBot\\/","url":"https://mediamonitoringbot.com/","description":"OSINT crawler harvesting mentions from news blogs and forums","tags":["seo"]},{"pattern":"MediavineMetadataParser\\/","url":"https://radar.cloudflare.com/bots/directory/mediavine-metadata-parser/mediavine.com","description":"Ad network metadata extractor gathering schema and preview data","tags":["advertising"]},{"pattern":"Pywikibot\\/","url":"https://www.mediawiki.org/wiki/Manual:Pywikibot","description":"MediaWiki crawler indexing wiki content for search and analysis","tags":["seo"]},{"pattern":"CentComBot\\/","url":"https://centcom.melonmesa.com/","description":"Content indexing and dataset building crawler tool","tags":["seo"]},{"pattern":"MergadoBot","url":"https://datadome.co/bots/mergadobot/","description":"E-commerce product data feed optimization and aggregation crawler","tags":["seo"]},{"pattern":"Meta-ExternalHit\\/","url":"https://datadome.co/bots/meta-externalagent/","description":"Meta platform bot fetching link previews and metadata for sharing","tags":["social-preview"]},{"pattern":"Metorik","url":"https://metorik.com/","description":"E-commerce analytics platform crawler for WooCommerce and Shopify stores","tags":["seo"]},{"pattern":"MgidBot","url":"https://www.mgid.com","description":"Native advertising platform crawler analyzing pages for targeting","tags":["advertising"]},{"pattern":"Miniature\\.io\\/","url":"https://miniature.io/","description":"Web page archiving and thumbnail generation bot tool","tags":["archiver"]},{"pattern":"mirrorweb\\.com","url":"https://www.mirrorweb.com/","description":"Web archiving platform crawler capturing timestamped website snapshots","tags":["archiver"]},{"pattern":"MissinglettrBot\\/","url":"https://missinglettr.com/","description":"Social media automation platform crawler for blog and feed indexing","tags":["seo"]},{"pattern":"crawler_eb_germany","url":"https://datadome.co/bots/crawler_eb_germany/","description":"General web crawler for indexing and data collection purposes","tags":["seo"]},{"pattern":"ModularConnector\\/","url":"https://uniqoders.com/","description":"Distributed modular crawler framework for content processing","tags":["seo"]},{"pattern":"Mollie HTTP client","url":"https://www.mollie.com/","description":"Payment service provider verifying merchant sites and compliance","tags":["seo"]},{"pattern":"Monibot","url":"https://datadome.co/bots/monibot/","description":"Web crawler for competitive analysis and price monitoring","tags":["seo"]},{"pattern":"monitis -","url":"https://www.teamviewer.com/en/products/add-ons/remote-management/","description":"Synthetic monitoring bot checking website availability and performance","tags":["monitoring"]},{"pattern":"MonitoRSS\\/","url":"https://monitorss.xyz/","description":"Discord bot crawler fetching RSS feeds for channel notifications","tags":["feed-reader"]},{"pattern":"MonSpark\\/","url":"https://monspark.com/","description":"Website monitoring service bot checking uptime and performance","tags":["monitoring"]},{"pattern":"montastic-monitor","url":"https://montastic.com/","description":"Uptime monitoring bot tracking website availability and latency","tags":["monitoring"]},{"pattern":"MonTools\\.com","url":"https://datadome.co/bots/montools/","description":"Web crawler for scanning and indexing website pages","tags":["seo"]},{"pattern":"MotoMinerBot\\/","url":"https://motominer.com/Bot","description":"Automotive marketplace crawler indexing vehicle listings and prices","tags":["seo"]},{"pattern":"MRGbot\\/","url":"https://www.mrg.ro/","description":"Security testing lab crawler for anti-fraud control evaluation","tags":["scanner"]},{"pattern":"MxToolbox","url":"https://datadome.co/bots/mxtoolbox-bot/","description":"Email delivery and reputation monitoring for organizations","tags":["monitoring"]},{"pattern":"my-tiny-bot","url":"https://datadome.co/bots/my-tiny-bot/","description":"Lightweight web crawler for content indexing and analysis","tags":["seo"]},{"pattern":"MyBot\\/","url":"https://datadome.co/bots/mybot/","description":"Web crawler for indexing and data aggregation purposes","tags":["seo"]},{"pattern":"nbertaupete95","url":"https://datadome.co/bots/nbertaupete95/","description":"Web crawler for SEO and competitive analysis purposes","tags":["seo"]},{"pattern":"NetAPI","url":"https://datadome.co/bots/netapi/","description":"Automated crawler interacting with websites for data collection","tags":["seo"]},{"pattern":"NetpeakCheckerBot\\/","url":"https://netpeaksoftware.com/","description":"SEO analysis crawler identifying technical issues and metrics","tags":["seo"]},{"pattern":"NetShelter ContentScan","url":"https://datadome.co/bots/netshelter-contentscan/","description":"Media network crawler categorizing content for ad targeting","tags":["advertising"]},{"pattern":"netumoDisallow","url":"https://docs.netumo.com/","description":"Domain health monitoring bot verifying uptime and DNS","tags":["monitoring"]},{"pattern":"NETVIGIE","url":"https://www.netvigie.com/","description":"European performance and uptime monitoring synthetic crawler","tags":["monitoring"]},{"pattern":"NewRelicbot\\/","url":"https://newrelic.com/","description":"Application monitoring bot for uptime and performance checking","tags":["monitoring"],"rdns":[".newrelic.com"]},{"pattern":"nyt_scraping|scraping@nytimes\\.com","url":"https://int.nyt.com/assets/scraping.json","description":"News organization newsgathering crawler for reporting support","tags":["seo"]},{"pattern":"NewsNow\\/","url":"https://www.newsnow.co.uk/h/","description":"UK news aggregation platform crawler for content discovery","tags":["seo"]},{"pattern":"Nitro-Warmup-Agent","url":"https://nitropack.io/","description":"Website optimization platform cache warming and asset generation","tags":["seo"]},{"pattern":"NLNZ_IAHarvester","url":"https://natlib.govt.nz/publishers-and-authors/web-harvesting","description":"New Zealand library digital preservation and archiving bot","tags":["archiver"]},{"pattern":"NodePing","url":"https://datadome.co/bots/nodeping/","description":"Website monitoring service performing uptime checks globally","tags":["monitoring"]},{"pattern":"Noibu JS Beautifier","url":"https://www.noibu.com/","description":"Error monitoring bot beautifying JavaScript for debugging","tags":["monitoring"]},{"pattern":"nomore404\\.com robot","url":"https://datadome.co/bots/nomore404-com-robot/","description":"Broken link detection crawler identifying HTTP 404 errors","tags":["seo"]},{"pattern":"noorobot","url":"https://noordigital.com/","description":"Web scraper performing high-volume programmatic content collection","tags":["seo"]},{"pattern":"Nooshub\\/","url":"https://www.nooshub.com/","description":"AI feed platform crawler indexing content and RSS feeds","tags":["seo"]},{"pattern":"Notabot","url":"https://corp.helpfeel.com/ja/home","description":"Content aggregation and metadata extraction crawler bot","tags":["seo"]},{"pattern":"Novaact\\/","url":"https://datadome.co/bots/novaact/","description":"Web crawler for data aggregation and analysis purposes","tags":["seo"],"rdns":[".amazon.com",".amazonaws.com"]},{"pattern":"Novellum","url":"https://crawl.corp.novellum.ai/docs","description":"AI data collection crawler for model training and enrichment","tags":["ai-crawler"]},{"pattern":"NsToolsBot\\/","url":"https://ns.tools/","description":"Network and security analysis tool crawler gathering server data","tags":["scanner"]},{"pattern":"nvdorz","url":"https://datadome.co/bots/nvdorz/","description":"Web crawler for content indexing and analysis purposes","tags":["seo"]},{"pattern":"Odin;","url":"https://docs.odin.io/introduction","description":"Automated crawler for data collection and indexing purposes","tags":["seo"]},{"pattern":"Offline Explorer","url":"https://datadome.co/bots/offline-explorer/","description":"Website downloader tool for offline archival and content analysis","tags":["archiver"]},{"pattern":"OhDear\\/","url":"https://ohdear.app/docs/faq/what-is-the-oh-dear-crawler-doing-in-my-logs","description":"Website monitoring bot scanning uptime and TLS certificate validity","tags":["monitoring"]},{"pattern":"Omnisend\\/","url":"https://bots.omnisend.io/cf.txt","description":"Ecommerce marketing automation crawler fetching campaign URLs metadata","tags":["seo"]},{"pattern":"OneTrust","url":"https://www.onetrust.com/","description":"Consent management platform crawler detecting cookies and tracking","tags":["scanner"]},{"pattern":"Online Domain Tools","url":"https://datadome.co/bots/online-domain-tools/","description":"SEO analysis crawler identifying technical issues and metrics","tags":["seo"]},{"pattern":"WebCEO Online\\/","url":"https://datadome.co/bots/online-webceo-bot/","description":"SEO platform crawler analyzing keywords and backlinks","tags":["seo"]},{"pattern":"OnlineOrNot\\.com_bot","url":"https://onlineornot.com","description":"Web crawler for systematic content browsing and indexing","tags":["monitoring"]},{"pattern":"OpenGraph\\.io\\/","url":"https://www.opengraph.io/","description":"Open Graph metadata extractor for link preview generation","tags":["social-preview"]},{"pattern":"OpenRSS","url":"https://openrss.org/","description":"Feed discovery crawler converting websites into RSS feeds","tags":["feed-reader"]},{"pattern":"OpenVAS","url":"https://openvas.org/","description":"Vulnerability scanner crawler identifying security weaknesses","tags":["scanner"],"rdns":[".openvas.org"]},{"pattern":"Owler \\(ows\\.eu","url":"https://openwebsearch.eu/owler/","description":"European open search crawler for decentralized indexing","tags":["search-engine"]},{"pattern":"Operator\\/","url":"https://datadome.co/bots/operator/","description":"Automated crawler performing repetitive internet tasks","tags":["seo"]},{"pattern":"Orbbot\\/","url":"https://datadome.co/bots/orbbot/","description":"Web crawler indexing content for SEO and analysis","tags":["seo"]},{"pattern":"zebra-v2-bot","url":"https://datadome.co/bots/zebra-v2-bot/","description":"Web crawler for indexing and data collection purposes","tags":["seo"]},{"pattern":"Orlo-LinkPreview\\/","url":"https://orlo.tech/","description":"Social platform link preview and metadata fetcher","tags":["social-preview"]},{"pattern":"Cozi-iCalendar-FeedReader","url":"https://www.ourfamilywizard.com/","description":"Co-parenting platform crawler fetching shared content metadata","tags":["feed-reader"]},{"pattern":"OutsellURLValidator","url":"https://impel.ai/outsell/","description":"Digital marketing crawler validating campaign link health","tags":["seo"]},{"pattern":"Overcast\\/","url":"https://overcast.fm/","description":"Podcast platform crawler discovering and refreshing feeds","tags":["feed-reader"]},{"pattern":"PRTGCloudBot\\/","url":"https://www.paessler.com/","description":"Paessler cloud monitoring bot checking uptime and health","tags":["monitoring"]},{"pattern":"Pagespeed\\/","url":"https://www.pagespeed.de/","description":"Performance analysis crawler assessing web page metrics","tags":["monitoring"]},{"pattern":"PanguBot","url":"https://datadome.co/bots/pangubot/","description":"Chinese search engine crawler indexing websites","tags":["search-engine"]},{"pattern":"Panopta","url":"https://www.fortinet.com/products/fortimonitor","description":"Synthetic monitoring bot verifying uptime and transactions","tags":["monitoring"]},{"pattern":"Paqlebot\\/","url":"https://www.paqle.dk/about/paqlebot","description":"Web crawler gathering data for analytics and research","tags":["seo"]},{"pattern":"parse\\.ly scraper\\/","url":"https://www.parse.ly/","description":"Content analytics platform crawler for metadata extraction","tags":["seo"]},{"pattern":"PayPal\\/","url":"https://www.paypal.com/ipn","description":"Payment processor bot validating endpoints and compliance","tags":["seo"]},{"pattern":"PDF24 URL To PDF","url":"https://tools.pdf24.org/en/webpage-to-pdf","description":"PDF conversion crawler capturing web pages as documents","tags":["seo"]},{"pattern":"PingAdmin\\.Ru\\/","url":"https://ping-admin.com/","description":"Uptime monitoring service bot checking website availability","tags":["monitoring"]},{"pattern":"pingping\\.io\\/","url":"https://pingping.io/","description":"Lightweight monitoring bot validating URL availability","tags":["monitoring"]},{"pattern":"PlayStore-Google","url":"https://support.google.com/webmasters/answer/1061943","description":"Google Play Store crawler indexing apps and content","tags":["search-engine"]},{"pattern":"Plesk screenshot bot","url":"https://www.plesk.com/","description":"Hosting platform management crawler for website auditing","tags":["seo"]},{"pattern":"PocketCasts\\/","url":"https://support.pocketcasts.com/knowledge-base/pocket-casts-feed-parser/","description":"Podcast platform crawler parsing feeds and episodes","tags":["feed-reader"]},{"pattern":"Potions\\/","url":"https://get-potions.com/","description":"E-commerce personalization crawler ingesting catalogs","tags":["seo"]},{"pattern":"PressEngineBot","url":"https://pressengine.net/","description":"Press release aggregation crawler for content syndication","tags":["seo"]},{"pattern":"PricedroneShoppingBot\\/","url":"http://pricedrone.com/robot/","description":"Price comparison crawler aggregating product pricing","tags":["seo"]},{"pattern":"PriEcoBot\\/","url":"https://datadome.co/bots/priecobot/","description":"Web crawler for content aggregation and analysis","tags":["seo"]},{"pattern":"PrintFriendly\\.com","url":"https://www.printfriendly.com/","description":"Print optimization crawler converting pages to printable format","tags":["seo"]},{"pattern":"Pro-Sitemaps\\/","url":"https://pro-sitemaps.com/","description":"Sitemap generation crawler discovering and indexing URLs","tags":["seo"]},{"pattern":"ProbelySPDR\\/","url":"https://snyk.io/dast-api-web/sos/","description":"Security testing crawler identifying web vulnerabilities","tags":["scanner"]},{"pattern":"PHPot Verispider","url":"https://radar.cloudflare.com/bots/directory/project-honeypot/www.projecthoneypot.org.","description":"Deception bot detection network identifying malicious crawlers","tags":["scanner"]},{"pattern":"ProjectShield-UrlCheck","url":"https://projectshield.withgoogle.com/","description":"Google DDoS protection bot validating customer origins","tags":["scanner"]},{"pattern":"Blackbox Exporter\\/","url":"https://github.com/prometheus/blackbox_exporter","description":"Prometheus monitoring crawler checking endpoint health","tags":["monitoring"]},{"pattern":"Protopage\\/","url":"https://datadome.co/bots/protopage/","description":"Start page and RSS reader crawler aggregating content","tags":["feed-reader"]},{"pattern":"PS_Daily\\/","url":"https://datadome.co/bots/ps-daily/","description":"Web crawler for data collection and analysis","tags":["seo"]},{"pattern":"pulsetic\\.com","url":"https://pulsetic.com/","description":"Uptime monitoring bot tracking website availability","tags":["monitoring"]},{"pattern":"PWABuilderHttpAgent","url":"https://www.pwabuilder.com/","description":"Microsoft PWA readiness evaluation crawler tool","tags":["seo"]},{"pattern":"QualifiedBot\\/","url":"https://www.qualified.com/legal/qualified-crawler-user-agent","description":"Conversational sales platform crawler inventorying site URLs","tags":["seo"]},{"pattern":"cf-qualys-scanner","url":"https://www.qualys.com/solutions/pci-compliance","description":"Web vulnerability scanner detecting security issues and compliance","tags":["scanner"]},{"pattern":"Quantcastbot\\/","url":"https://www.quantcast.com/bot","description":"Audience analytics crawler for ad measurement and targeting","tags":["advertising"]},{"pattern":"Rackspace Monitoring\\/","url":"https://support.rackspace.com/how-to/about-the-rackspace-monitoring-agent/","description":"Cloud infrastructure monitoring bot checking endpoint availability","tags":["monitoring"]},{"pattern":"rakutenusabot-image\\/","url":"https://product-image.ebates.com/item-gsp/rakutenusabot.html","description":"E-commerce platform crawler extracting product images","tags":["seo"]},{"pattern":"top100\\.rambler\\.ru crawler","url":"https://datadome.co/bots/rambler-bot/","description":"Russian search engine crawler indexing web content","tags":["search-engine"]},{"pattern":"RankurBot\\/","url":"https://rankur.com/technology.html","description":"Social media and reputation monitoring crawler collecting data","tags":["seo"]},{"pattern":"RavenCrawler\\/","url":"https://datadome.co/bots/ravencrawler/","description":"SEO auditing crawler analyzing website optimization signals","tags":["seo"]},{"pattern":"Readable\\/","url":"https://readable.com/","description":"Content analysis crawler computing readability and metrics","tags":["seo"]},{"pattern":"Recurly Webhooks\\/","url":"https://docs.recurly.com/recurly-subscriptions/docs/webhooks","description":"Billing platform webhook validation and monitoring bot","tags":["seo"]},{"pattern":"RED\\/","url":"https://redbot.org/","description":"HTTP response testing and validation analyzer","tags":["monitoring"]},{"pattern":"Reelevant\\/","url":"https://reelevant.com/","description":"Personalization platform fetching URLs for email and content","tags":["seo"]},{"pattern":"remove\\.bg\\/","url":"https://www.remove.bg/","description":"Image editing service crawler accessing and processing images","tags":["seo"]},{"pattern":"Retool\\/","url":"https://docs.tryretool.com/docs/apis","description":"Internal tools platform fetching page metadata and content","tags":["seo"]},{"pattern":"RetroListeCOM\\/","url":"https://retroliste.com/","description":"Directory and price comparison crawler indexing listings","tags":["seo"]},{"pattern":"RevvimGort\\/","url":"https://revvim.com","description":"SEO optimization crawler detecting performance issues","tags":["seo"]},{"pattern":"reward-gateway","url":"https://www.rewardgateway.co","description":"Employee engagement platform crawler validating offers","tags":["seo"]},{"pattern":"Riddler \\(http:\\/\\/riddler\\.io","url":"http://riddler.io/about","description":"Internet scanning crawler gathering domain and IP metadata","tags":["scanner"]},{"pattern":"RobotsChecker\\/","url":"https://www.blocked.org.uk/","description":"Website health monitoring and content indexing crawler","tags":["seo"]},{"pattern":"RSSAPI\\/","url":"https://rssapi.net","description":"RSS and Atom feed crawler for content aggregation","tags":["feed-reader"]},{"pattern":"rss2tg","url":"https://rss2tg.duck.consulting","description":"RSS to Telegram converter fetching feed content","tags":["feed-reader"]},{"pattern":"RssReaderBot","url":"https://datadome.co/bots/rssreaderbot/","description":"RSS feed aggregator indexing content updates","tags":["feed-reader"]},{"pattern":"s4a-probe-bot\\/","url":"https://www.seo4ajax.com/webscraper/","description":"Web scraping and SEO analysis crawler tool","tags":["seo"]},{"pattern":"SFDC-Callout\\/","url":"https://help.salesforce.com/s/articleView?language=en_US&id=000321501&mode=1&type=1","description":"Salesforce platform crawler fetching web resources","tags":["seo"]},{"pattern":"page-preview-tool","url":"https://www.salesviewer.com/en/","description":"B2B intelligence platform identifying visiting companies","tags":["seo"]},{"pattern":"SandobaCrawler\\/","url":"https://www.sandoba.com/en/crawler/","description":"Web crawler for data aggregation and indexing","tags":["seo"]},{"pattern":"Sansec Security Monitor\\/","url":"https://sansec.io/monitor","description":"E-commerce malware detection scanner for skimmers","tags":["scanner"]},{"pattern":"GIFTEDVISITOR SCAN","url":"https://datadome.co/bots/scan/","description":"Web crawler for content indexing and analysis","tags":["seo"]},{"pattern":"Schema-Markup-Validator","url":"https://validator.schema.org/","description":"Structured data validation crawler for schema compliance","tags":["seo"]},{"pattern":"Scoop\\.it\\/","url":"https://datadome.co/bots/scoop-it/","description":"Content curation platform crawler discovering relevant content","tags":["seo"]},{"pattern":"ScourRSSBot\\/","url":"https://scour.ing/bot","description":"RSS feed discovery and monitoring crawler bot","tags":["feed-reader"]},{"pattern":"ScrapeheroBot\\/","url":"https://agentur4www.de/web-scraping","description":"Web scraping service crawler for data extraction","tags":["seo"]},{"pattern":"screeenly-bot","url":"https://docs.3.screeenly.com/guide/faq/why-is-screeenly-3-in-my-logs","description":"Screenshot capture service crawler taking website snapshots","tags":["seo"]},{"pattern":"SEBot-WA","url":"https://help.seranking.com/en/project-tools/website-audit/overview","description":"SEO audit crawler performing technical analysis","tags":["seo"],"rdns":[".seranking.com",".sr-srv.net"]},{"pattern":"Searcherweb","url":"https://datadome.co/bots/searcherweb/","description":"Web crawler indexing content for search engines","tags":["search-engine"]},{"pattern":"Searcherxweb","url":"https://datadome.co/bots/searcherxweb/","description":"Web crawler for indexing and data collection","tags":["search-engine"]},{"pattern":"SearchExpress","url":"https://datadome.co/bots/searchexpress/","description":"Search engine crawler gathering website information","tags":["search-engine"]},{"pattern":"SecurityHeaders","url":"https://securityheaders.com/","description":"HTTP security headers analyzer and evaluator","tags":["scanner"]},{"pattern":"semaltbot\\/","url":"https://datadome.co/bots/semaltbot/","description":"SEO analytics crawler collecting ranking and performance data","tags":["seo"]},{"pattern":"SendGrid Event API","url":"https://www.twilio.com:443/docs/sendgrid/for-developers/tracking-events/event","description":"Email service crawler scanning links in messages","tags":["seo"]},{"pattern":"SentryUptimeBot\\/","url":"https://docs.sentry.io/product/monitors-and-alerts/monitors/uptime-monitoring/troubleshooting/","description":"Application monitoring bot verifying URL availability","tags":["monitoring"]},{"pattern":"seo-audit-check-bot\\/","url":"https://www.webceo.com/webceo-bots.htm","description":"SEO auditing crawler evaluating optimization signals","tags":["seo"]},{"pattern":"s4a\\/|seo4ajax\\.com","url":"https://www.seo4ajax.com/","description":"JavaScript rendering and SEO prerendering service","tags":["seo"]},{"pattern":"ClarityBot\\/","url":"https://www.seoclarity.net/bot.html","description":"SEO analytics crawler providing optimization insights","tags":["seo"]},{"pattern":"SeoSiteCheckup","url":"https://datadome.co/bots/seositecheckup/","description":"SEO site analysis crawler evaluating search performance","tags":["seo"]},{"pattern":"SeoulBot","url":"https://datadome.co/bots/seoulbot/","description":"Web crawler for SEO and competitive analysis","tags":["seo"]},{"pattern":"SequelWP Uptime Monitoring","url":"https://sequelwp.com.au/","description":"WordPress hosting crawler checking site availability","tags":["monitoring"]},{"pattern":"SERPtimizerBot","url":"https://serptimizer.com/","description":"SEO analysis crawler gathering data for optimization","tags":["seo"]},{"pattern":"Server Density Service Monitoring","url":"https://www.stackpath.com/","description":"Cloud monitoring bot checking uptime and performance","tags":["monitoring"]},{"pattern":"ServerHunterSpider\\/","url":"https://www.serverhunter.com/","description":"Hosting and VPS price comparison crawler","tags":["seo"],"rdns":[".hunter.io"]},{"pattern":"SeznamHomepageCrawler\\/","url":"https://o-seznam.cz/napoveda/","description":"Czech search engine crawler indexing web pages","tags":["search-engine"]},{"pattern":"Shopify-Captain-Hook","url":"https://shopify.dev/docs/apps/build/webhooks","description":"E-commerce platform crawler for webhook verification","tags":["seo"]},{"pattern":"Shortwave Image Fetcher","url":"https://www.shortwave.com/","description":"Email client image proxy and link preview fetcher","tags":["social-preview"]},{"pattern":"linkReader\\/","url":"https://sider.ai/","description":"AI assistant crawler for content indexing","tags":["ai-fetcher"]},{"pattern":"Sidetrade indexer bot","url":"https://datadome.co/bots/sidetrade-crawler/","description":"Financial analytics crawler gathering business data","tags":["seo"]},{"pattern":"SilktideBot\\/","url":"https://help.silktide.com/en/articles/4606146-silktidebot-our-site-scanning-bot","description":"Web quality and accessibility auditing crawler bot","tags":["seo"]},{"pattern":"SinceraSyntheticUser\\/","url":"https://datadome.co/bots/sincera-bot/","description":"Web data aggregation and analytics crawler","tags":["seo"]},{"pattern":"Optimizer\\)","url":"https://www.sistrix.com/faq/uptime","description":"SEO analysis and website optimization crawler","tags":["seo"]},{"pattern":"Site24x7","url":"https://www.site24x7.com/","description":"Cloud monitoring service checking website health","tags":["monitoring"],"rdns":[".site24x7.com"]},{"pattern":"SiteAuditBot\\/","url":"https://datadome.co/bots/siteauditbot/","description":"Website SEO auditing and link checking crawler","tags":["seo"]},{"pattern":"SiteLockSpider","url":"https://www.sitelock.com/","description":"Malware and vulnerability scanning bot for websites","tags":["scanner"]},{"pattern":"SiteScoreBot","url":"https://datadome.co/bots/sitescorebot/","description":"Website performance and security evaluation crawler","tags":["seo"]},{"pattern":"SiteSearch360\\/","url":"https://www.sitesearch360.com/","description":"On-site search platform crawler building indices","tags":["search-engine"]},{"pattern":"SiteUptime\\.com","url":"https://datadome.co/bots/siteuptime-com/","description":"Uptime monitoring service checking website availability","tags":["monitoring"]},{"pattern":"Konturbot\\/","url":"https://datadome.co/bots/skb-kontur-bot/","description":"Russian business intelligence and analytics crawler","tags":["seo"]},{"pattern":"SkroutzBot","url":"https://www.skroutz.gr/","description":"Price comparison platform crawler aggregating products","tags":["seo"]},{"pattern":"SkyworkSpider","url":"https://datadome.co/bots/skyworkspider/","description":"Web crawler for content indexing and analysis","tags":["seo"]},{"pattern":"SlickBot\\/","url":"https://www.slickstream.com/","description":"Content recommendation and search indexing crawler","tags":["seo"]},{"pattern":"SmartologyBot\\/","url":"https://smartology.net/smartologybot/","description":"Contextual advertising crawler for brand safety","tags":["advertising"]},{"pattern":"SnapURLPreview\\/","url":"https://business.snapchat.com/legal/snapchat-automated-crawler","description":"Social platform URL preview and metadata fetcher","tags":["social-preview"]},{"pattern":"SnapchatAds\\/","url":"https://businesshelp.snapchat.com/s/article/adsbot-crawler?language=en_US","description":"Advertising platform crawler for ad verification","tags":["advertising"]},{"pattern":"Snipcart\\/","url":"https://snipcart.com/","description":"E-commerce shopping cart crawler validating products","tags":["seo"]},{"pattern":"solarwinds\\/","url":"https://documentation.solarwinds.com/en/success_center/observability/content/get-started/dem_getting_started_guide.htm","description":"Observability platform monitoring website performance","tags":["monitoring"]},{"pattern":"Sora POS\\/","url":"https://www.sora-caisse.com/sora.pdf","description":"Point-of-sale system crawler for transaction data","tags":["seo"]},{"pattern":"SparkShipping","url":"https://www.sparkshipping.com/","description":"E-commerce data crawler normalizing product catalogs","tags":["seo"]},{"pattern":"SparkPost","url":"https://bird.com/?sp=true","description":"Email service crawler validating links and tracking","tags":["seo"]},{"pattern":"Spawning-AI","url":"https://datadome.co/bots/spawning-bot/","description":"Web crawler indexing content for AI models","tags":["ai-crawler"]},{"pattern":"IDG\\/EU","url":"https://datadome.co/bots/spaziodati-bot/","description":"Semantic data and business intelligence crawler","tags":["seo"]},{"pattern":"Specificfeeds","url":"https://follow.it/","description":"Content aggregation crawler for personalized feeds","tags":["feed-reader"]},{"pattern":"Spectate\\/","url":"https://docs.spectate.net/faq/uptime-monitor-bot","description":"Content monitoring and uptime checking crawler","tags":["monitoring"]},{"pattern":"SpiderLing","url":"https://nlp.fi.muni.cz/projects/biwec/","description":"Linguistic research crawler building language corpora","tags":["academic"]},{"pattern":"splash Version\\/","url":"https://www.zyte.com/splash/","description":"Headless browser for web scraping and testing","tags":["browser-automation"]},{"pattern":"Rigor\\)","url":"https://www.splunk.com/","description":"Splunk synthetic monitoring for performance checks","tags":["monitoring"]},{"pattern":"TwinWaveScanner","url":"https://www.splunk.com/en_us/products/attack-analyzer.html","description":"Security threat analysis and phishing detection bot","tags":["scanner"]},{"pattern":"Splunk Synthetics\\)","url":"https://docs.splunk.com/observability/en/synthetics/intro-synthetics.html","description":"Website monitoring crawler measuring availability","tags":["monitoring"]},{"pattern":"SSL Labs \\(https:\\/\\/www\\.ssllabs\\.com","url":"https://www.ssllabs.com/","description":"TLS certificate and encryption configuration analyzer","tags":["scanner"]},{"pattern":"SSSSBot\\/","url":"https://datadome.co/bots/ssssbot/","description":"Web crawler for content indexing and analysis","tags":["seo"]},{"pattern":"Stape\\/","url":"https://stape.io/helpdesk/documentation/stape-scanner","description":"Server-side tagging platform tag verification bot","tags":["seo"]},{"pattern":"StartpagePrivateImageProxy\\/","url":"https://datadome.co/bots/startpage-bot/","description":"Privacy-focused search engine crawler indexing pages","tags":["search-engine"]},{"pattern":"Statabot\\/","url":"https://www.stata.com/support/statabot/","description":"Analytics and market intelligence web crawler","tags":["seo"]},{"pattern":"StatistikAustria\\/","url":"https://www.statistik.at/ueber-uns/innovationen-und-experimentelle-statistik/einsatz-von-kassenscannerdaten-und-webscraping-in-der-preisstatistik","description":"National statistics office crawler for price data","tags":["seo"]},{"pattern":"StatsDroneBot","url":"https://statsdrone.com/statsdrone-bot-documentation/","description":"Affiliate marketing link validation and monitoring bot","tags":["seo"]},{"pattern":"Stripe\\/","url":"https://docs.stripe.com/webhooks","description":"Payment processor crawler verifying merchant sites and collecting metadata signals","tags":["seo"],"rdns":[".stripe.com"]},{"pattern":"Sucuri","url":"https://blog.sucuri.net/2012/10/ask-sucuri-how-does-sitecheck-work.html","description":"Security scanner checking malware blacklist and plugin vulnerabilities","tags":["scanner"]},{"pattern":"Svix-Webhooks\\/","url":"https://docs.svix.com/receiving/source-ips","description":"Webhooks infrastructure provider crawler cataloging vendor sites and delivery practices","tags":["seo"]},{"pattern":"SwifteqLinkChecker","url":"https://swifteq.com/","description":"Crawler verifying link availability and page content","tags":["seo"]},{"pattern":"Swisscows","url":"https://swisscows.com/en","description":"Privacy-focused search engine crawler without storing personal data","tags":["search-engine"]},{"pattern":"Datadog Synthetic","url":"https://docs.datadoghq.com/synthetics/","description":"Synthetic monitoring bot testing APIs and collecting uptime data","tags":["monitoring"]},{"pattern":"TactiScout\\/","url":"http://find-it.world/TempCrawl/Crawltheque.php","description":"Data crawler gathering competitive intelligence and market trends","tags":["seo"]},{"pattern":"tchelebi\\/","url":"https://tchelebi.io/","description":"Web crawler indexing content for SEO data aggregation","tags":["seo"]},{"pattern":"bitdiscovery","url":"https://www.tenable.com/products/attack-surface-management","description":"Security scanner identifying vulnerabilities in web infrastructure","tags":["scanner"],"rdns":[".tenable.com"]},{"pattern":"TermlyBot\\/","url":"https://termly.io/bot/","description":"Compliance scanner gathering legal policies and cookie consent","tags":["scanner"]},{"pattern":"Test Certificate Info","url":"https://datadome.co/bots/test-certificate-info/","description":"SSL/TLS certificate analyzer assessing encryption and expiration","tags":["scanner"]},{"pattern":"Testcrawler","url":"https://datadome.co/bots/testcrawler/","description":"Testing crawler evaluating website performance security vulnerabilities and SEO optimization","tags":["seo"]},{"pattern":"test-bot","url":"https://datadome.co/bots/test-bot/","description":"Automated testing script monitoring uptime and gathering performance analysis data","tags":["seo"]},{"pattern":"TestLocally\\/","url":"https://testlocal.ly/","description":"Testing bot simulating user interactions and verifying website behavior","tags":["browser-automation"]},{"pattern":"TestURI","url":"https://datadome.co/bots/testuri-crawler/","description":"Web crawler testing performance security compliance and assessing website configuration","tags":["seo"]},{"pattern":"TextRazor","url":"https://datadome.co/bots/textrazor-crawler/","description":"NLP crawler collecting web data for text analysis","tags":["seo"]},{"pattern":"The Knowledge AI","url":"https://datadome.co/bots/the-knowledge-ai/","description":"AI data crawler indexing content for machine learning","tags":["ai-crawler"]},{"pattern":"TheInternetSearchx","url":"https://datadome.co/bots/theinternetsearch/","description":"Search engine crawler indexing websites for search results","tags":["search-engine"]},{"pattern":"thesis-research-bot","url":"https://datadome.co/bots/thesis-research-bot/","description":"Academic crawler collecting web data and analyzing website structures","tags":["academic"]},{"pattern":"Trellis-Services","url":"https://www.mediavine.com/","description":"Content scraper for programmatic large-scale web page retrieval","tags":["seo"]},{"pattern":"trentwil\\.es","url":"https://trentwil.es/domains.html","description":"Crawler indexing content for SEO analysis and intelligence","tags":["seo"]},{"pattern":"Trustly\\/","url":"https://www.trustly.com/","description":"Open banking crawler automating sessions for account data","tags":["seo"]},{"pattern":"TTD-Content","url":"https://www.thetradedesk.com/legal/ttd-content","description":"Ad network crawler analyzing pages for contextual advertising","tags":["advertising"]},{"pattern":"Tweakers","url":"https://tweakers.net/","description":"Dutch tech site crawler aggregating product reviews and IT content","tags":["seo"]},{"pattern":"TwilioProxy\\/","url":"https://www.upday.com/de","description":"Communication platform fetcher retrieving media links and previews","tags":["social-preview"]},{"pattern":"UASlinkChecker\\/","url":"https://udger.com/support/UASlinkChecker","description":"Link checker validating URL availability and HTTP status","tags":["seo"]},{"pattern":"hgfAlphaXCrawl\\/","url":"https://datadome.co/bots/uni-passau_bot/","description":"University research crawler collecting data for academic projects","tags":["academic"]},{"pattern":"Unshorten\\.It\\!","url":"https://unshorten.it/","description":"URL expander revealing shortened link destinations for threat intelligence","tags":["scanner"]},{"pattern":"updown\\.io","url":"https://updown.io","description":"Website monitoring crawler checking availability and performance metrics from locations","tags":["monitoring"]},{"pattern":"Uptime\\/","url":"https://uptime.com/","description":"Synthetic monitoring bot verifying site availability and transaction health globally","tags":["monitoring"]},{"pattern":"uptimedoctor","url":"http://uptimestatistics.com/","description":"Uptime monitoring probe verifying website availability and SSL certificate health","tags":["monitoring"]},{"pattern":"Uptimia","url":"https://datadome.co/bots/uptimia/","description":"Platform crawler monitoring website performance uptime and simulating user interactions","tags":["monitoring"]},{"pattern":"uptrends","url":"https://www.uptrends.com/","description":"Synthetic monitoring agent testing transaction availability and performance from locations","tags":["monitoring"]},{"pattern":"Urlcheckr\\/","url":"https://www.urlcheckr.com/","description":"Dead link checker identifying broken hyperlinks and inaccessible pages","tags":["seo"]},{"pattern":"URLSuMaBot","url":"https://www.urlsuma.de/bot.aspx","description":"Crawler systematically indexing and collecting data from websites","tags":["seo"]},{"pattern":"useeBookChecker\\/","url":"https://datadome.co/bots/useebookchecker/","description":"Ebook metadata scanner identifying and cataloging digital publication availability data","tags":["seo"]},{"pattern":"Vagabondo\\/","url":"https://datadome.co/bots/vagabondo-bot/","description":"Legacy Dutch crawler indexing mobile pages for regional search","tags":["seo"]},{"pattern":"VaultPress","url":"https://vaultpress.com/","description":"WordPress backup and security scanner performing malware scanning and integrity","tags":["scanner"]},{"pattern":"videootvBot","url":"https://digitalgreen.es/?ref=videootv","description":"Video aggregator crawler discovering indexing video content and stream endpoints","tags":["seo"]},{"pattern":"VsuSearchSpider\\/","url":"https://datadome.co/bots/vsusearchspider/","description":"Web crawler indexing and gathering information for search","tags":["search-engine"]},{"pattern":"vu-server-health-scanner\\/","url":"http://130.37.198.75/index.html","description":"Academic crawler indexing content for research data collection","tags":["academic"]},{"pattern":"WARDBot\\/","url":"https://ward.ai/robot","description":"Web automation crawler extracting website data for reconnaissance and analysis","tags":["seo"]},{"pattern":"WebsiteOps","url":"https://faqs.watchful.net/technical-support/how-do-i-whitelist-the-watchful-ip-address","description":"Web crawling client monitoring uptime changes and analyzing website data","tags":["monitoring"]},{"pattern":"WatchMouse","url":"https://asm.saas.broadcom.com/","description":"Performance monitoring tool assessing website availability globally","tags":["monitoring"]},{"pattern":"Web Measure\\/","url":"https://webresearch.eecs.umich.edu/overview-of-web-measurements/","description":"Web analytics crawler gathering data for performance optimization","tags":["academic"]},{"pattern":"Webflow","url":"https://datadome.co/bots/webflow-bot/","description":"Platform crawler rendering and indexing Webflow hosted websites for optimization","tags":["seo"]},{"pattern":"webgains-bot","url":"https://datadome.co/bots/webgains-bot/","description":"Affiliate marketing crawler extracting product listings and pricing information","tags":["advertising"]},{"pattern":"webprosbot\\/","url":"https://datadome.co/bots/webprosbot/","description":"SEO crawler analyzing websites for optimization content and competitive research","tags":["seo"]},{"pattern":"websitepulse","url":"https://www.websitepulse.com/kb/websitepulse","description":"Synthetic monitoring agent measuring availability performance and content integrity","tags":["monitoring"]},{"pattern":"WebSniffer\\/","url":"https://websniffer.com/","description":"Web analyzer retrieving HTTP headers and HTML for diagnostics","tags":["seo"]},{"pattern":"WSM\\/|webspidermount","url":"https://webspidermount.com/","description":"Web scraper crawler for content collection and data aggregation purposes","tags":["seo"]},{"pattern":"WTotem","url":"https://wtotem.com/","description":"Website security scanner assessing malware phishing and content integrity monitoring","tags":["scanner"]},{"pattern":"WebwikiBot\\/","url":"https://www.webwiki.com/","description":"Directory crawler collecting website metadata and content for comprehensive indexing","tags":["seo"]},{"pattern":"WEDOS OnLine","url":"https://wedos.online/en/","description":"Website monitoring service checking availability DNS and SSL certificate status","tags":["monitoring"]},{"pattern":"WhatsMyIP\\.org","url":"https://www.whatsmyip.org/ua/","description":"IP address analysis tool gathering network configuration and geolocation data","tags":["seo"]},{"pattern":"WhatWeb\\/","url":"https://www.whatweb.net/","description":"Web technology scanner identifying frameworks servers and CMS configurations","tags":["scanner"]},{"pattern":"Wheregoes\\.com","url":"https://wheregoes.com/","description":"URL redirect analyzer mapping redirect chains and HTTP forwarding paths","tags":["seo"]},{"pattern":"wheresitup\\.com\\/","url":"https://wheresitup.com/","description":"Availability monitor crawler tracking website uptime and performance metrics","tags":["monitoring"]},{"pattern":"Citoid","url":"https://www.mediawiki.org/wiki/Citoid","description":"Wikipedia citation generator extracting metadata for automatic reference creation","tags":["academic"]},{"pattern":"WireReaderBot\\/","url":"https://datadome.co/bots/wirereaderbot/","description":"Web crawler indexing content for data aggregation and competitive analysis","tags":["seo"]},{"pattern":"ZoteroTranslationServer\\/WMF","url":"https://wikitech.wikimedia.org/wiki/Zotero","description":"Citation server extracting metadata from URLs for Wikipedia reference generation","tags":["academic"]},{"pattern":"wmtips\\.com\\/","url":"https://www.wmtips.com/tools/","description":"SEO analysis crawler scanning websites for optimization issues and metrics","tags":["seo"]},{"pattern":"WordCountBot\\/","url":"https://www.weglot.com/","description":"Content analysis tool computing word counts keyword density and readability","tags":["seo"]},{"pattern":"Wordup-1","url":"https://datadome.co/bots/wordup-1/","description":"Crawler indexing and analyzing content for search and analytics","tags":["seo"]},{"pattern":"workona-favicon-service\\/","url":"https://datadome.co/bots/workona-bot/","description":"Productivity platform crawler organizing web content and managing bookmarks","tags":["seo"]},{"pattern":"Indy Library","url":"https://secure.ogone.com/Ncol/Prod/Backoffice/login/index?branding=OGONE&error=login_required&state=25df7c3e-2db0-4cf4-acb5-25cac56e394f&iss=https%3A%2F%2Fmerchant.login.worldline-solutions.com%2Fauth%2Frealms%2Fworldline","description":"Payment processor crawler verifying merchant sites and payment flow transactions","tags":["seo"]},{"pattern":"WJHRO\\/","url":"https://docs.worldpay.com/apis","description":"Payment platform crawler testing integrations and verifying checkout flow functionality","tags":["seo"]},{"pattern":"WormlyBot","url":"https://www.wormly.com/help/server-monitoring/website","description":"Synthetic monitoring crawler testing site APIs and measuring uptime performance","tags":["monitoring"]},{"pattern":"WovnCrawler\\/","url":"https://support.wovn.io/hc/ja/articles/360043165091","description":"Localization platform crawler extracting content and generating translated versions","tags":["seo"]},{"pattern":"wowLink Crawler\\/","url":"https://datadome.co/bots/wowlink-crawler/","description":"Link management crawler indexing and analyzing web content for optimization","tags":["seo"]},{"pattern":"WP Time Capsule","url":"https://docs.wptimecapsule.com/","description":"WordPress backup service crawler detecting changes and performing incremental backups","tags":["seo"]},{"pattern":"WPUmbrella","url":"https://wp-umbrella.com/bot","description":"WordPress monitor scanner assessing availability and tracking plugin updates","tags":["monitoring"]},{"pattern":"wpbot\\/","url":"https://datadome.co/bots/wpbot/","description":"Web crawler indexing websites and analyzing content for search optimization","tags":["seo"]},{"pattern":"WPMU DEV Broken Link Checker","url":"https://wpmudev.com/docs/hub-2-0/link-checker/","description":"Link auditor identifying dead links and HTTP errors","tags":["seo"]},{"pattern":"WPMUDEV Uptime Monitor","url":"https://wpmudev.com/monitor/","description":"HTTP probe checking availability status codes and SSL health","tags":["monitoring"]},{"pattern":"WPSec\\/","url":"https://wpsec.com/","description":"WordPress security scanner identifying vulnerabilities outdated plugins and misconfigurations","tags":["scanner"]},{"pattern":"WRTNBot","url":"https://datadome.co/bots/wrtnbot/","description":"Crawler indexing content for data aggregation and market analysis","tags":["seo"]},{"pattern":"abuse\\.xmco\\.fr","url":"https://abuse.xmco.fr/","description":"Cybersecurity crawler scanning websites for vulnerabilities and threat analysis signals","tags":["scanner"]},{"pattern":"XY-Archive-Compliance","url":"https://xy-archive.helpscoutdocs.com/article/61-does-xy-archive-have-a-dedicated-ip-address","description":"Archiving compliance crawler collecting content for legal retention and eDiscovery","tags":["archiver"]},{"pattern":"Yahoo Ad monitoring","url":"https://developer.yahoo.com/api/","description":"Ad ecosystem crawler auditing quality and policy compliance","tags":["advertising"]},{"pattern":"YahooMailProxy","url":"https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html","description":"Email proxy crawler fetching URLs for security scanning","tags":["social-preview"],"rdns":[".yahoo.co.jp",".yahoo.com",".yahoo.net"]},{"pattern":"YahooCacheSystem","url":"https://developer.yahoo.com/oauth2/guide/","description":"Content fetcher caching web pages and thumbnails for Yahoo","tags":["seo"],"rdns":[".yahoo.co.jp",".yahoo.com",".yahoo.net"]},{"pattern":"YLT Chrome","url":"https://yellowlab.tools/","description":"Performance analyzer evaluating front-end quality and optimization recommendations","tags":["monitoring"]},{"pattern":"YokoyGroupAG\\/","url":"https://www.perk.com/","description":"Expense management platform webhook client delivering events and verifying endpoints","tags":["seo"]},{"pattern":"Yuuperbot","url":"https://datadome.co/bots/yuuperbot/","description":"Crawler indexing content for data aggregation and intelligence","tags":["seo"]},{"pattern":"Zapier","url":"https://zapier.com/","description":"Automation client fetching web pages and content for workflows","tags":["seo"]},{"pattern":"Zendesk Webhook","url":"https://support.zendesk.com/hc/en-us","description":"Support platform service validating webhook endpoints and fetching URL metadata","tags":["seo"]},{"pattern":"Zombiebot\\/","url":"http://www.zombiedomain.net/robot/","description":"Crawler indexing content for SEO analysis and intelligence","tags":["seo"]},{"pattern":"zzhbot","url":"https://datadome.co/bots/zzhbot/","description":"Crawler indexing and analyzing content for search engines","tags":["seo"]},{"pattern":"Penthouse Critical Path CSS Generator","url":"https://criticalcss.com/","description":"CSS above-the-fold extractor generating critical path stylesheets","tags":["seo"]},{"pattern":"Google-AdWords-Express","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Google Ads Express fetching advertiser landing pages for review","tags":["advertising"]},{"pattern":"Notion\\/","url":"https://www.iubenda.com/en/","description":"Notion workspace crawler fetching URLs for link previews","tags":["social-preview"]},{"pattern":"SSL Labs$","url":"https://www.qualys.com/solutions/pci-compliance","description":"Qualys SSL Labs scanner testing TLS configuration and certificate grades","tags":["scanner"]},{"pattern":"Skroutz ImageBot","url":"https://www.skroutz.gr/","description":"Greek price comparison platform crawler fetching product images","tags":["seo"]},{"pattern":"Tumblr\\/","url":"https://automattic.com/","description":"Tumblr social blogging platform crawler fetching link previews","tags":["social-preview"],"rdns":[".tumblr.com"]},{"pattern":"upday\\/","url":"https://www.upday.com/de","description":"Samsung news aggregator crawler fetching articles for daily briefings","tags":["feed-reader"]},{"pattern":"^ArenaUnfurlBot","url":"https://arena.ai/","description":"Arena.ai link preview generator","tags":["social-preview"]},{"pattern":"A360-Search","url":"https://area360.uk/","description":"Area360 real estate data crawler","tags":["advertising"]},{"pattern":"AASA-Bot","url":"https://developer.apple.com/documentation/xcode/allowing-apps-and-websites-to-link-to-your-content","description":"Apple's Universal Links file fetcher","tags":["social-preview"]},{"pattern":"Acquia optimize","url":"https://knownagents.com/agents/acquia-optimize-monsido","description":"Acquia website optimization monitoring","tags":["monitoring"]},{"pattern":"AdkernelTopicCrawler","url":"http://adkernel.com/robot/","description":"Adkernel ad tech solutions crawler","tags":["advertising"]},{"pattern":"AlertSite","url":"https://smartbear.com/product/alertsite/","description":"SmartBear's synthetic monitoring service","tags":["monitoring"]},{"pattern":"Amazing-SearchBot","url":"https://amazing.com/bot.html","description":"Amazing e-commerce platform search crawler","tags":["advertising"]},{"pattern":"Amazon-Bedrock-AgentCore-Browser","url":"https://docs.aws.amazon.com/bedrock-agentcore/","description":"AWS cloud browser for AI agents","tags":["browser-automation"]},{"pattern":"AmazonBuyForMe","url":"https://www.amazon.com/b?node=204815148011","description":"Amazon bot for Buy For Me service purchases","tags":["browser-automation"],"rdns":[".amazon.com",".amazonaws.com"]},{"pattern":"Amzn-SearchBot","url":"https://developer.amazon.com/amazonbot","description":"Amazon's AI search indexer for Alexa","tags":["ai-fetcher"],"rdns":[".crawl.amazonbot.amazon"]},{"pattern":"Amzn-User","url":"https://developer.amazon.com/amazonbot","description":"Amazon AI assistant for Alexa queries","tags":["ai-fetcher"],"rdns":[".crawl.amazonbot.amazon"]},{"pattern":"Anchor Browser","url":"https://knownagents.com/agents/anchor-browser","description":"Anchor's cloud-hosted browser for AI agents","tags":["browser-automation"]},{"pattern":"Anomura","url":"https://docs.direqt-search.com/direqt-bots/direqt-crawlers-and-user-agents","description":"Direqt's AI search indexer","tags":["ai-crawler"]},{"pattern":"ApifyBot","url":"https://knownagents.com/agents/apifybot","description":"Apify's web scraping and data extraction","tags":["ai-crawler"],"rdns":[".apify.com"]},{"pattern":"ApifyWebsiteContentCrawler","url":"https://apify.com/apify/website-content-crawler","description":"Apify's full website content extractor","tags":["ai-crawler"],"rdns":[".apify.com"]},{"pattern":"Archive-It","url":"https://archive-it.org/files/site-owners-special.html","description":"Internet Archive's web preservation crawler","tags":["archiver"],"rdns":[".archive.org",".internetarchive.org"]},{"pattern":"artemis web reader","url":"https://artemis.jamesg.blog/bot","description":"Artemis personal web reader","tags":["feed-reader"]},{"pattern":"atlassian-bot","url":"https://support.atlassian.com/organization-administration/docs/connect-custom-website-to-rovo/","description":"Atlassian Rovo's AI search crawler","tags":["ai-crawler"]},{"pattern":"AudigentAdBot","url":"https://audigent.com/bot.html","description":"Audigent targeted advertising crawler","tags":["advertising"]},{"pattern":"Automaton|Newsify Feed Fetcher","url":"https://knownagents.com/agents/automaton","description":"Automaton feed fetcher monitoring","tags":["monitoring"]},{"pattern":"AwarioRendererBot","url":"https://awario.com/help/","description":"Awario social media monitoring crawler","tags":["advertising"]},{"pattern":"AzureAI-SearchBot","url":"https://azure.microsoft.com/en-us/products/ai-services","description":"Microsoft's Azure AI search indexer","tags":["ai-fetcher"],"rdns":[".azure.com",".azurewebsites.net",".microsoft.com"]},{"pattern":"bigsur\\.ai","url":"https://bigsur.ai/","description":"Big Sur AI's web agent crawler","tags":["ai-crawler"]},{"pattern":"bl\\.uk_lddc_bot","url":"https://www.bl.uk/services/legal-deposit/web-archiving","description":"British Library's legal deposit archiver","tags":["archiver"]},{"pattern":"Bloglines","url":"https://www.bloglines.com/","description":"Bloglines RSS feed reader","tags":["feed-reader"]},{"pattern":"bluesky-domain-status-classifier","url":"https://bsky.social/about/","description":"Bluesky social network link preview bot","tags":["social-preview"]},{"pattern":"Bluesky\\/","url":"https://knownagents.com/agents/bluesky-link-preview-service","description":"Bluesky link preview service","tags":["social-preview"]},{"pattern":"bne\\.es_bot","url":"https://bne.es/es/colecciones/archivo-web-espanola/aviso-webmasters","description":"Spanish National Library web archiver","tags":["archiver"]},{"pattern":"Brightbot","url":"https://brightdata.com/brightbot","description":"Bright Data's AI-ready data collector","tags":["ai-crawler"]},{"pattern":"BrowserBot-Observer","url":"https://obsrvr.net/about","description":"Observer third-party risk security scanner","tags":["scanner"]},{"pattern":"BufferLinkPreviewBot","url":"https://scraper.buffer.com/about/bots/link-preview-bot","description":"Buffer social media link preview bot","tags":["social-preview"]},{"pattern":"Bugsnag","url":"https://knownagents.com/agents/bugsnag-script-fetcher","description":"Bugsnag JavaScript source fetcher","tags":["monitoring"],"rdns":[".bugsnag.com"]},{"pattern":"Buttondown","url":"https://buttondown.com/features","description":"Buttondown RSS to email converter","tags":["feed-reader"]},{"pattern":"CapitalOneBot","url":"https://developer.capitalone.com/","description":"Capital One dealer website crawler","tags":["feed-reader"]},{"pattern":"channable","url":"https://channable.com/","description":"Channable e-commerce marketing automation crawler","tags":["advertising"]},{"pattern":"Channel3Bot","url":"https://trychannel3.com/channel3bot","description":"Channel3's universal product catalog indexer","tags":["ai-crawler"]},{"pattern":"Chirp|gotosocial","url":"https://www.binarycanary.com/","description":"Binary Canary uptime monitoring bot","tags":["monitoring"]},{"pattern":"ClickUpLinkUnfurler","url":"https://clickup.com/","description":"ClickUp link preview generator","tags":["social-preview"]},{"pattern":"Cloudflare-AutoRAG","url":"https://developers.cloudflare.com/ai-search/","description":"Cloudflare's RAG service indexer","tags":["ai-crawler"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Custom-Hostname-Verification","url":"https://knownagents.com/agents/cloudflare-custom-hostname-verification","description":"Cloudflare hostname verification crawler","tags":["seo"],"rdns":[".cloudflare.com"]},{"pattern":"Cloudflare-Stream-Webhook","url":"https://knownagents.com/agents/cloudflare-stream-webhook","description":"Cloudflare Stream webhook crawler","tags":["seo"],"rdns":[".cloudflare.com"]},{"pattern":"CloudflareRadarURLScanner","url":"https://knownagents.com/agents/cloudflare-radar-url-scanner","description":"Cloudflare URL security scanner","tags":["monitoring"],"rdns":[".cloudflare.com"]},{"pattern":"[cC]ludo","url":"https://knownagents.com/agents/cludo","description":"Cludo site search monitoring","tags":["monitoring"]},{"pattern":"Code\\/","url":"https://github.com/features/copilot","description":"GitHub Copilot AI coding agent","tags":["browser-automation"]},{"pattern":"Collapsify","url":"https://developers.cloudflare.com/","description":"Cloudflare error page caching service","tags":["feed-reader"]},{"pattern":"ContextualBot[\\s\\S]*outcomes\\.net","url":"https://www.outcomes.net/","description":"Mobian contextual intelligence crawler","tags":["advertising"]},{"pattern":"CookieHubVerify","url":"https://www.cookiehub.com/","description":"CookieHub consent banner verification scanner","tags":["scanner"]},{"pattern":"CookieYesbot","url":"https://www.cookieyes.com/documentation/cookieyesbot/","description":"CookieYes consent compliance scanner","tags":["scanner"]},{"pattern":"Current[\\s\\S]*RSS Reader","url":"https://www.currentreader.app/","description":"Current RSS reader application","tags":["feed-reader"]},{"pattern":"cypex\\.ai\\/scanning","url":"https://cypex.ai/","description":"Cypex security vulnerability scanner","tags":["scanner"]},{"pattern":"lumar|deepcrawl","url":"https://www.lumar.io/spdr/","description":"Lumar enterprise SEO platform crawler","tags":["seo"],"rdns":[".deepcrawl.co",".deepcrawl.com",".lumar.io"]},{"pattern":"dlvr\\.it","url":"https://dlvrit.com/","description":"dlvr.it social media automation service","tags":["feed-reader"]},{"pattern":"Dotcom-Monitor","url":"https://knownagents.com/agents/doctom-monitor","description":"Doctom website monitoring","tags":["monitoring"]},{"pattern":"DrataAutopilot","url":"https://knownagents.com/agents/drata-autopilot","description":"Drata compliance monitoring","tags":["monitoring"]},{"pattern":"DreamHost Data Team","url":"https://www.dreamhost.com/support/","description":"DreamHost hosting monitoring bot","tags":["monitoring"]},{"pattern":"ds9","url":"https://data.dss.sps.copyright.com/docs/user_agent.html","description":"Copyright Clearance Center licensing crawler","tags":["advertising"]},{"pattern":" DVbot","url":"https://doubleverify.com/","description":"DoubleVerify ad verification crawler","tags":["advertising"]},{"pattern":"EcoVadisSustainabilityBot","url":"https://ecovadis.com/","description":"EcoVadis sustainability ratings crawler","tags":["advertising"]},{"pattern":"elmah\\.io Uptime Monitoring","url":"https://knownagents.com/agents/elmah-io-uptime-monitoring","description":"elmah.io uptime monitoring service","tags":["monitoring"]},{"pattern":"EvernoteRichLinkBot","url":"https://evernote.com/","description":"Evernote rich link preview bot","tags":["social-preview"]},{"pattern":"EzLynx","url":"https://www.ezoic.com:443/bot","description":"Ezoic website optimization crawler","tags":["advertising"]},{"pattern":"EzoicBot","url":"https://www.ezoic.com:443/bot","description":"Ezoic ad monetization platform crawler","tags":["advertising"]},{"pattern":"FacebookBot","url":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","description":"Meta's AI speech recognition training crawler","tags":["ai-crawler"],"rdns":[".facebook.com",".fbcdn.net",".fbsv.net",".tfbnw.net"]},{"pattern":"FastDAST","url":"https://knownagents.com/agents/black-duck-fast-dynamic","description":"Black Duck security testing bot","tags":["monitoring"]},{"pattern":"Feeder \\/","url":"https://knownagents.com/agents/feeder","description":"Feeder RSS reader app","tags":["feed-reader"]},{"pattern":"FeedFlow","url":"https://www.feedflow.dev/","description":"FeedFlow RSS reader application","tags":["feed-reader"]},{"pattern":"FindFiles\\.net","url":"https://findfiles.net/en/bot","description":"FindFiles.net search engine crawler","tags":["search-engine"]},{"pattern":"FirecrawlAgent","url":"https://www.firecrawl.dev/","description":"Firecrawl's LLM data extraction crawler","tags":["ai-fetcher"],"rdns":[".firecrawl.dev"]},{"pattern":"FyndSearchEngine-Crawler","url":"https://fynd.bot/","description":"Fynd search engine indexing crawler","tags":["search-engine"]},{"pattern":"FyndSearchEngine-ReCrawler","url":"https://fynd.bot/","description":"Fynd search engine re-indexing crawler","tags":["search-engine"]},{"pattern":"Goodreads","url":"https://www.goodreads.com/","description":"Goodreads book preview fetcher","tags":["social-preview"]},{"pattern":"Google Trust Services","url":"https://knownagents.com/agents/google-trust-services-dcv-check","description":"Google SSL certificate validation","tags":["monitoring"]},{"pattern":"Google-Agent","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Google's user-triggered agent for web navigation","tags":["browser-automation"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Google-Gemini-CLI","url":"https://geminicli.com/","description":"Google's AI coding agent for terminal","tags":["browser-automation"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Google-NotebookLM","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Google's AI research and note-taking assistant","tags":["ai-fetcher"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"GoogleAgent-Mariner","url":"https://deepmind.google/","description":"Google's AI agent for browser-based tasks","tags":["browser-automation"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Greppr Web Crawler","url":"https://greppr.org/","description":"Greppr unfiltered search engine crawler","tags":["search-engine"]},{"pattern":"Hardenize","url":"https://www.hardenize.com/","description":"Hardenize security configuration scanner","tags":["scanner"]},{"pattern":"HoneybadgerBot","url":"https://knownagents.com/agents/honeybadgerbot","description":"Honeybadger error monitoring bot","tags":["monitoring"]},{"pattern":"IbouBot","url":"https://ibou.io/iboubot.html","description":"Ibou web graph search crawler","tags":["search-engine"]},{"pattern":"imageSpider","url":"https://knownagents.com/agents/imagespider","description":"ByteDance's image collection crawler","tags":["ai-crawler"]},{"pattern":"Innologica","url":"https://knownagents.com/agents/innologica","description":"Innologica content fetcher","tags":["feed-reader"]},{"pattern":"kagi-fetcher","url":"https://help.kagi.com/kagi/ai/kagi-ai.html","description":"Kagi AI assistant for web content","tags":["ai-fetcher"],"rdns":[".kagi.com",".kagibot.com"]},{"pattern":"Kangaroo Bot","url":"https://kangaroollm.com.au/kangaroo-bot/","description":"Australian AI model training crawler","tags":["ai-crawler"]},{"pattern":"Known Agent","url":"https://knownagents.com/","description":"Known Agents test and development bot","tags":["monitoring"]},{"pattern":"KrawlerBot","url":"https://krawler.app/robot","description":"Krawler web scraping service bot","tags":["scanner"]},{"pattern":"laion-huggingface-processor","url":"https://knownagents.com/agents/laion-huggingface-processor","description":"LAION's image dataset builder for AI","tags":["ai-crawler"]},{"pattern":"LinkCheckerBot","url":"https://knownagents.com/agents/linkchecker-bot","description":"LinkChecker broken link bot","tags":["seo"]},{"pattern":"LinkupBot","url":"https://linkup.so/bot","description":"Linkup's enterprise AI search crawler","tags":["ai-fetcher"]},{"pattern":"LMArenaUnfurlBot","url":"https://arena.ai/","description":"LM Arena link preview bot","tags":["social-preview"]},{"pattern":"lyonl-asset-proxy","url":"https://lyonl.com/crawler","description":"Lyonl search engine asset proxy","tags":["search-engine"]},{"pattern":"lyonl-crawler","url":"https://lyonl.com/crawler","description":"Lyonl open web search crawler","tags":["search-engine"]},{"pattern":"MagiBot","url":"https://magi.com/bots","description":"Peak Labs information extraction crawler","tags":["search-engine"]},{"pattern":"MagpieRSS","url":"https://magpierss.sourceforge.net/","description":"MagpieRSS PHP feed parser","tags":["feed-reader"]},{"pattern":"mail\\.ru","url":"https://knownagents.com/agents/mailrubot","description":"Mail.ru content fetcher","tags":["feed-reader"],"rdns":[".mail.ru"]},{"pattern":"MailChimp","url":"https://mailchimp.com/","description":"Mailchimp email campaign content fetcher","tags":["feed-reader"]},{"pattern":"Manus-User","url":"https://knownagents.com/agents/manus-user","description":"Butterfly Effect's AI agent for web tasks","tags":["browser-automation"],"rdns":[".manus.im"]},{"pattern":"McontextualBot","url":"https://mcontextual.net/mcontextual-bot","description":"MContextual advertising targeting crawler","tags":["advertising"]},{"pattern":"Mediumbot-MetaTagFetcher","url":"https://medium.com/","description":"Medium link preview meta tag fetcher","tags":["social-preview"]},{"pattern":"MetaIAB Facebook","url":"https://knownagents.com/agents/facebook","description":"Facebook link preview fetcher","tags":["social-preview"]},{"pattern":"MixrankBot","url":"https://knownagents.com/agents/mixrankbot","description":"MixRank business intelligence crawler","tags":["advertising"]},{"pattern":"ModernizeBot","url":"https://modernizeyourwebsite.com/bot","description":"ModernizeYourWebsite analysis crawler","tags":["seo"]},{"pattern":"MontasticMonitor","url":"https://knownagents.com/agents/montasticmonitor","description":"Montastic uptime monitoring","tags":["monitoring"]},{"pattern":"NanoInteractive","url":"https://nanointeractive.com/crawler-policy/","description":"Nano Interactive ad targeting crawler","tags":["advertising"]},{"pattern":"NestDaddybot","url":"https://nestdaddy.com/bot","description":"NestDaddy search engine crawler","tags":["search-engine"]},{"pattern":"Netcraft SSL Server Survey","url":"https://knownagents.com/agents/netcraft-ssl-server-survey","description":"Netcraft SSL certificate intelligence scanner","tags":["scanner"]},{"pattern":"Netcraft Web Server Survey","url":"https://www.netcraft.com/blog/june-2025-web-server-survey","description":"Netcraft web server survey crawler","tags":["advertising"]},{"pattern":"NetSeer crawler","url":"https://inuvo.com/","description":"NetSeer intelligence gathering crawler","tags":["advertising"]},{"pattern":"Netumo|netumo","url":"https://docs.netumo.com/","description":"Netumo uptime monitoring bot","tags":["monitoring"]},{"pattern":"NewRelicSynthetics","url":"https://knownagents.com/agents/new-relic","description":"New Relic synthetic monitoring","tags":["monitoring"]},{"pattern":"NewsRoom\\.BI","url":"http://newsroom.bi/bot.html","description":"Marfeel publisher analytics crawler","tags":["advertising"]},{"pattern":"Nitro-","url":"https://knownagents.com/agents/nitro","description":"NitroPack speed optimization crawler","tags":["seo"]},{"pattern":"NitroBot","url":"https://knownagents.com/agents/nitrobot","description":"NitroPack SEO optimization bot","tags":["seo"]},{"pattern":"Noibu","url":"https://www.noibu.com/","description":"Noibu ecommerce error monitoring","tags":["monitoring"]},{"pattern":"NostoCrawlerBot","url":"http://my.nosto.com/tagging","description":"Nosto ecommerce personalization crawler","tags":["advertising"]},{"pattern":"opencode-smartfetch","url":"https://opencode.ai/","description":"Open-source AI coding agent","tags":["browser-automation"]},{"pattern":";Owler","url":"https://knownagents.com/agents/owler","description":"Owler competitive intelligence crawler","tags":["advertising"]},{"pattern":"ParselySharesBot","url":"https://docs.parse.ly/","description":"Parse.ly content analytics crawler","tags":["monitoring"]},{"pattern":"PhindBot","url":"https://knownagents.com/agents/phindbot","description":"AI-powered developer answer engine crawler","tags":["ai-fetcher"],"rdns":[".phind.com"]},{"pattern":"PodchaserParser","url":"https://www.podchaser.com/","description":"Podchaser podcast database crawler","tags":["feed-reader"]},{"pattern":"Podimo","url":"https://podimo.com/de","description":"Podimo podcast streaming platform bot","tags":["feed-reader"]},{"pattern":"Poggio-Citations","url":"https://docs.poggio.io/api/robots","description":"Poggio's AI sales enablement data collector","tags":["ai-fetcher"]},{"pattern":"productsup\\.io\\/crawler","url":"https://help.productsup.com/docs/help-center/import-data-feeds/import-data-from-internal-systems/import-web-data-with-website-crawler","description":"Productsup product feed management crawler","tags":["advertising"]},{"pattern":"qcbot","url":"https://docs.quic.cloud/services/qcbot/","description":"QUIC.cloud optimization service crawler","tags":["feed-reader"]},{"pattern":"Qualys","url":"https://knownagents.com/agents/qualys","description":"Qualys security scanning","tags":["monitoring"],"rdns":[".qualys.com"]},{"pattern":"Quora-Bot","url":"http://quora.com/","description":"Quora link preview crawler","tags":["social-preview"]},{"pattern":"Qwantbot","url":"https://help.qwant.com/","description":"Qwant privacy-focused search crawler","tags":["search-engine"],"rdns":[".qwant.com"]},{"pattern":"Qwarrybot","url":"http://qwarry.com/bot.html","description":"Qwarry contextual advertising crawler","tags":["advertising"]},{"pattern":"RSiteAuditor","url":"https://dataforseo.com/apis/on-page-api","description":"DataForSEO on-page audit crawler","tags":["seo"]},{"pattern":"RSS\\.Social","url":"https://rss.social/bot","description":"RSS.Social feed automation bot","tags":["feed-reader"]},{"pattern":"Salesforce\\.com","url":"https://knownagents.com/agents/sfdc-callout","description":"Salesforce HTTP callout user agent","tags":["http-library"]},{"pattern":"Scope3","url":"https://docs.scope3.com/docs/scope3-crawler","description":"Scope3 content classification and brand safety","tags":["advertising"]},{"pattern":"Scrubby","url":"http://scrubtheweb.com/","description":"Scrub the Web business directory crawler","tags":["search-engine"]},{"pattern":"Scrunchbot","url":"https://scrunch.com/bots/","description":"Scrunch AI brand visibility tracker","tags":["monitoring"]},{"pattern":"SequelWP","url":"https://sequelwp.com.au/","description":"SequelWP hosting uptime monitoring","tags":["monitoring"]},{"pattern":"ServerDensity","url":"https://knownagents.com/agents/server-density","description":"Server Density infrastructure monitoring","tags":["monitoring"]},{"pattern":"ShapBot","url":"https://docs.parallel.ai/resources/crawler","description":"Parallel's AI agent web context provider","tags":["ai-crawler"]},{"pattern":"ShortPixel","url":"https://shortpixel.com/","description":"ShortPixel image optimization service","tags":["feed-reader"]},{"pattern":"Silktide","url":"https://silktide.com/","description":"Silktide website quality monitoring","tags":["monitoring"]},{"pattern":"SiteLock","url":"https://knownagents.com/agents/sitelock","description":"SiteLock security monitoring","tags":["monitoring"]},{"pattern":"SmarshBot","url":"https://www.smarsh.com/smb/web-archive/","description":"Smarsh's compliance archiving bot","tags":["archiver"]},{"pattern":"SMTnetPMBot","url":"https://smtnet.com/","description":"SMTnet electronics manufacturing search crawler","tags":["search-engine"]},{"pattern":"Software-Security-Research","url":"https://reverse-proxies-measurements.softsec.ruhr-uni-bochum.de/","description":"Ruhr University CDN security research crawler","tags":["scanner","academic"]},{"pattern":"SottopopNone","url":"https://www.upcontent.com/robots","description":"UpContent content curation crawler","tags":["feed-reader"]},{"pattern":"Spider[\\s\\S]*spider\\.com","url":"https://www.spider.com/solutions/web-crawler","description":"AI project web data crawler","tags":["ai-crawler"]},{"pattern":"Splunk","url":"https://knownagents.com/agents/splunk","description":"Splunk monitoring and analytics","tags":["monitoring"],"rdns":[".splunk.com",".splunkcloud.com"]},{"pattern":"StatusNestBacklinkSpider","url":"https://statusnest.com/bot","description":"StatusNest backlink tracking crawler","tags":["seo"]},{"pattern":"stepstoneCrawlBot","url":"https://www.thestepstonegroup.com/english/crawler/","description":"Stepstone Group job search crawler","tags":["search-engine"]},{"pattern":"TavilyBot","url":"https://knownagents.com/agents/tavilybot","description":"Tavily's real-time AI agent data crawler","tags":["ai-fetcher"],"rdns":[".tavily.com"]},{"pattern":"ThousandEyes","url":"https://knownagents.com/agents/thousand-eyes-cloud-agent","description":"ThousandEyes network monitoring","tags":["monitoring"]},{"pattern":"Trae\\/","url":"https://www.trae.ai/","description":"ByteDance's AI coding agent","tags":["browser-automation"],"rdns":[".bytedance.com",".tiktokv.com"]},{"pattern":"TwinAgent","url":"https://twin.so/","description":"Twin's automated worker for workflow execution","tags":["browser-automation"]},{"pattern":"uipbot","url":"https://knownagents.com/agents/uipbot","description":"Semasio semantic targeting crawler","tags":["advertising"]},{"pattern":"um-FC","url":"https://www.ubermetrics-technologies.com/","description":"Ubermetrics media monitoring crawler","tags":["advertising"]},{"pattern":"um-IC","url":"https://www.ubermetrics-technologies.com/","description":"Ubermetrics communications intelligence crawler","tags":["advertising"]},{"pattern":"UptimeStatistics","url":"https://knownagents.com/agents/uptimestatistics","description":"UptimeStatistics monitoring service","tags":["monitoring"]},{"pattern":"Verispider","url":"https://www.projecthoneypot.org/","description":"Verispider spam and security scanner","tags":["scanner"]},{"pattern":"visionheight\\.com\\/scan","url":"https://knownagents.com/agents/visionheight-comscan","description":"Vision Height security scanner","tags":["scanner"]},{"pattern":"Watchbot monitoring robot","url":"https://watchbot.fflow.net/","description":"Watchbot SSL and uptime monitoring","tags":["monitoring"]},{"pattern":"Watchful","url":"https://knownagents.com/agents/watchful","description":"Watchful website monitoring","tags":["monitoring"]},{"pattern":"weborama-fetcher","url":"https://www.weborama.com/","description":"Weborama advertising optimization crawler","tags":["advertising"]},{"pattern":"WepchSearchEngine","url":"https://www.wepch.com/search-engine","description":"Wepch search engine development crawler","tags":["search-engine"]},{"pattern":"wknd-bot","url":"https://developer.wunderkind.co/docs/server-side-tracking-implementation","description":"Wunderkind marketing automation tracker","tags":["advertising"]},{"pattern":"WPMU DEV Hub","url":"https://wpmudev.com/","description":"WPMU DEV WordPress management bot","tags":["monitoring"]},{"pattern":"XoviOnpageCrawler","url":"https://www.xovi.de/","description":"XOVI SEO suite on-page crawler","tags":["seo"]},{"pattern":"yelpspider","url":"https://yelp.com/","description":"Yelp business information crawler","tags":["feed-reader"]},{"pattern":"ZanistaBot","url":"https://zanista.ai/crawler-info","description":"Zanista's AI search crawler","tags":["ai-crawler"]},{"pattern":"ZoomInfo-","url":"https://knownagents.com/agents/zoominfo","description":"ZoomInfo business intelligence search crawler","tags":["search-engine"]},{"pattern":"7Siters","url":"https://7ooo.ru/siters/","description":"Russian web crawler for site indexing and analysis","tags":["search-engine"]},{"pattern":"Accessible Web Bot","url":"https://accessibleweb.com/bot/","description":"Monitors websites for WCAG accessibility compliance violations","tags":["monitoring"]},{"pattern":"AtVowBot","url":"https://brandeem.com/","description":"Brand monitoring and reputation management web crawler","tags":["monitoring"]},{"pattern":"Bibliotheque Nacional de France Crawler","url":"https://www.bnf.fr/en/web-legal-deposit","description":"French National Library web archiving and preservation bot","tags":["archiver"]},{"pattern":"Bling ERP","url":"https://www.bling.com.br/","description":"Brazilian ERP system data synchronization and integration crawler","tags":["monitoring"]},{"pattern":"CDSCbot","url":"https://wiki.communitydata.science/CommunityData:Fediverse_research","description":"Academic research crawler for Fediverse decentralized social networks","tags":["academic"]},{"pattern":"Critical CSS Bot","url":"https://criticalcss.com/","description":"Extracts critical CSS for web performance optimization tools","tags":["monitoring"]},{"pattern":"CybaaBot","url":"https://cybaa.io/bot-policy","description":"Cybaa advertising and marketing analytics web crawler bot","tags":["advertising"]},{"pattern":"CyberFindCrawler","url":"https://cyberfind.net/bot.html","description":"Web search engine crawler for content discovery indexing","tags":["search-engine"]},{"pattern":"Dark Visitor","url":"https://knownagents.com/","description":"Bot analytics and AI agent tracking service crawler","tags":["monitoring"]},{"pattern":"Determ","url":"https://determ.com/","description":"Media monitoring and brand reputation tracking web crawler","tags":["monitoring"]},{"pattern":"DNSScanner","url":"https://rapef.info/_contacts/","description":"DNS and network infrastructure security scanning crawler bot","tags":["scanner"]},{"pattern":"Drupalbot","url":"https://new.drupal.org/home","description":"Drupal CMS community website crawler and content indexer","tags":["monitoring"]},{"pattern":"eMoney Advisor","url":"https://emoneyadvisor.com/","description":"Financial planning software data integration and sync crawler","tags":["monitoring"]},{"pattern":"everyfeed-spider","url":"http://everyfeed.com/","description":"RSS and Atom feed aggregation and discovery crawler","tags":["feed-reader"]},{"pattern":"ExteContextCrawl","url":"http://crawl001.exte.ai/","description":"AI context extraction and content understanding web crawler","tags":["ai-crawler"]},{"pattern":"FediDB","url":"https://fedidb.com/resources/data-sources","description":"Fediverse instance database and statistics collection crawler bot","tags":["monitoring"]},{"pattern":"FediIndex","url":"https://fedi.wrm.sr/about","description":"Fediverse server indexing and discovery service web crawler","tags":["search-engine"]},{"pattern":"FediList Agent","url":"https://fedilist.com/","description":"Fediverse instance directory and listing service crawler bot","tags":["monitoring"]},{"pattern":"Fedineko","url":"https://fedineko.org/about","description":"Fediverse network mapping and analysis research crawler tool","tags":["academic"]},{"pattern":"FedReporter Bot for FFIEC","url":"https://www.fedreporter.com/","description":"Federal financial institution regulatory compliance data crawler bot","tags":["monitoring"]},{"pattern":"Feedsearch Bot","url":"https://feedearch.dev/","description":"RSS and feed discovery search engine crawler bot","tags":["feed-reader"]},{"pattern":"Feedsearch-Crawler","url":"https://pypi.org/project/feedsearch-crawler","description":"Python library for RSS feed detection and extraction","tags":["feed-reader"]},{"pattern":"fiperbot","url":"https://fiper.net/","description":"Web content indexing and search engine crawler bot","tags":["search-engine"]},{"pattern":"FleebsBot","url":"https://fleebs.com/bot","description":"Web content aggregation and discovery service crawler bot","tags":["search-engine"]},{"pattern":"Flyriverbot","url":"https://flyriver.com/crawler","description":"AI content source verification and attribution checking crawler","tags":["ai-crawler"]},{"pattern":"Freshbot","url":"http://webagent.wise-guys.nl/","description":"European web content monitoring and freshness checking crawler","tags":["monitoring"]},{"pattern":"Gaisbot","url":"http://gais.cs.ccu.edu.tw/robot.php","description":"Academic research web crawler from Taiwan university project","tags":["academic"]},{"pattern":"GenomeCrawlerd","url":"https://nokia.com/genomecrawler","description":"Nokia web content analysis and data collection crawler","tags":["monitoring"]},{"pattern":"HaloBot","url":"https://haloscan.com/","description":"Haloscan comment system integration and content fetching bot","tags":["monitoring"]},{"pattern":"IRLbot","url":"http://irl.cs.tamu.edu/crawler","description":"Texas A&M University academic research web crawler bot","tags":["academic"]},{"pattern":"kaikki\\.org-digital-archive","url":"https://kaikki.org/","description":"Digital archiving and preservation service web crawler bot","tags":["archiver"]},{"pattern":"kb\\.dk_bot","url":"https://www.kb.dk/en","description":"Royal Danish Library web archiving and preservation crawler","tags":["archiver"]},{"pattern":"Library Of Congress Web Archiving","url":"https://www.loc.gov/programs/web-archiving/about-this-program/","description":"US Library of Congress web preservation archiving crawler","tags":["archiver"]},{"pattern":"MagnetmeBot","url":"https://magnet.me/","description":"Job board and recruitment platform web scraping crawler","tags":["monitoring"]},{"pattern":"MatchorySearch","url":"https://matchory.com/en","description":"Search engine and content discovery indexing crawler bot","tags":["search-engine"]},{"pattern":"Minoru's Fediverse Crawler","url":"https://nodes.fediverse.party/","description":"Fediverse network node discovery and mapping crawler tool","tags":["monitoring"]},{"pattern":"MirrorWebCrawler","url":"https://www.mirrorweb.com/","description":"Enterprise web archiving and compliance preservation crawler bot","tags":["archiver"]},{"pattern":"mithril-crawler","url":"https://498-search-engine.github.io/website/","description":"Academic search engine project web indexing crawler bot","tags":["academic"]},{"pattern":"ModatScanner","url":"https://www.modat.io/","description":"Website monitoring and change detection service crawler bot","tags":["monitoring"]},{"pattern":"NapBot","url":"http://napbot.com/","description":"Web content indexing and search engine crawler bot","tags":["search-engine"]},{"pattern":"New York Times Newsgathering","url":"https://www.nytimes.com/","description":"News media content aggregation and research crawler bot","tags":["monitoring"]},{"pattern":"NLUX_IAHarvester","url":"https://crawl.bnl.lu/","description":"Luxembourg National Library web archiving and harvesting crawler","tags":["archiver"]},{"pattern":"NoahBot","url":"https://noahwire.com/bot-info","description":"News aggregation and content discovery service crawler bot","tags":["monitoring"]},{"pattern":"PlagAwareBot","url":"https://www.plagaware.com/bot","description":"Plagiarism detection and content originality checking crawler bot","tags":["monitoring"]},{"pattern":"Rakuten Image extraction bot","url":"https://www.rakuten.com/geogating/sorry?old=/","description":"E-commerce product image extraction and indexing crawler bot","tags":["monitoring"]},{"pattern":"ResearchBot","url":"https://www.kaust.edu.sa/en/error/404?404;https://www.kaust.edu.sa:443/bot","description":"Academic research data collection and analysis crawler bot","tags":["academic"]},{"pattern":"rss-is-dead\\.lol web bot","url":"https://rss-is-dead.lol/","description":"RSS feed monitoring and availability checking service crawler","tags":["feed-reader"]},{"pattern":"seoLyt","url":"https://seolyt.com/","description":"SEO analysis and website optimization tool crawler bot","tags":["seo"]},{"pattern":"SirdataBot","url":"https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction","description":"Contextual advertising and semantic content analysis crawler bot","tags":["advertising"]},{"pattern":"SitesOverPagesBot","url":"https://sitesoverpages.com/bot","description":"Website structure analysis and sitemap generation crawler bot","tags":["seo"]},{"pattern":"SleepBot","url":"http://sleepbot.com/","description":"Web content monitoring and tracking service crawler bot","tags":["monitoring"]},{"pattern":"Sosospider","url":"http://help.soso.com/webspider.htm","description":"Chinese search engine web indexing and crawling bot","tags":["search-engine"]},{"pattern":"Termly","url":"https://termly.io/","description":"Privacy policy and compliance scanning service crawler bot","tags":["monitoring"]},{"pattern":"TLS tester","url":"https://github.com/testssl/testssl.sh","description":"SSL/TLS security testing and vulnerability scanning crawler bot","tags":["scanner"]},{"pattern":"trafilatura","url":"https://github.com/adbar/trafilatura","description":"Python library for web content extraction and scraping","tags":["http-library"]},{"pattern":"UrlBeeBot","url":"https://urlbee.com/","description":"URL analysis and threat detection security crawler bot","tags":["scanner"]},{"pattern":"videootv Bot","url":"https://www.digitalgreen.org/","description":"Video content aggregation and educational media crawler bot","tags":["monitoring"]},{"pattern":"vmcrawl","url":"https://docs.vmst.io/vmcrawl","description":"Fediverse instance monitoring and discovery service crawler bot","tags":["monitoring"]},{"pattern":"WadooBot","url":"https://wadoo.net/wadoobot/","description":"Web content indexing and search engine crawler bot","tags":["search-engine"]},{"pattern":"Website-info\\.net-Robot","url":"https://website-info.net/robot","description":"Website information and analytics data collection crawler bot","tags":["seo"]},{"pattern":"WebZIP","url":"http://spidersoft.com/","description":"Website downloading and offline browsing tool crawler bot","tags":["archiver"]},{"pattern":"WikiDo","url":"http://wikido.com/","description":"Wiki content aggregation and search engine crawler bot","tags":["search-engine"]},{"pattern":"WOVN Crawler","url":"https://wovn.io/","description":"Website translation and localization service crawler bot","tags":["monitoring"]},{"pattern":"YoudaoBot","url":"http://youdao.com/help/webmaster/spider/","description":"Chinese search engine and translation service crawler bot","tags":["search-engine"],"rdns":[".youdao.com"]},{"pattern":"ZyBorg","url":"http://wisenutbot.com/","description":"WISEnut search engine web indexing and crawling bot","tags":["search-engine"]},{"pattern":"Aranet-SearchBot","url":"https://aranet.ai/bot","description":"AI-powered search engine and content indexing crawler bot","tags":["ai-fetcher"]},{"pattern":"crawl4ai","url":"https://github.com/unclecode/crawl4ai","description":"Open-source LLM-friendly web scraping and extraction library","tags":["ai-crawler"]},{"pattern":"DeepSeekBot","url":"http://deepseek.com/bot","description":"DeepSeek AI model training and data collection crawler","tags":["ai-crawler"],"rdns":[".deepseek.com"]},{"pattern":"iaskspider","url":"https://www.iask.com/","description":"Chinese AI search engine and question answering crawler","tags":["ai-crawler"]},{"pattern":"KunatoCrawler","url":"http://kunato.ai/bot.html","description":"AI training data collection and web scraping crawler","tags":["ai-crawler"]},{"pattern":"TerraCotta","url":"https://github.com/CeramicTeam/CeramicTerracotta","description":"Ceramic Network decentralized data indexing and crawling bot","tags":["ai-crawler"]},{"pattern":"ABEvalBot","url":"https://knownagents.com/agents/abevalbot","description":"A/B testing evaluation and website analysis crawler bot","tags":["monitoring"]},{"pattern":"blekkobot","url":"https://knownagents.com/agents/blekkobot","description":"Blekko search engine web indexing and crawling bot","tags":["search-engine"]},{"pattern":"br-crawler","url":"https://knownagents.com/agents/br-crawler","description":"Brazilian web content indexing and discovery crawler bot","tags":["search-engine"]},{"pattern":"BuddyBot","url":"https://knownagents.com/agents/buddybot","description":"Social networking and content aggregation service crawler bot","tags":["monitoring"]},{"pattern":"CapterraBot","url":"https://www.capterra.com","description":"Capterra software review platform data collection crawler bot","tags":["monitoring"]},{"pattern":"carbon-umbrella-bot","url":"https://knownagents.com/agents/carbon-umbrella-bot","description":"Carbon footprint tracking and environmental monitoring crawler bot","tags":["monitoring"]},{"pattern":"caveman-hunter","url":"https://fedi.buzz/","description":"Fediverse content discovery and social network crawler bot","tags":["monitoring"]},{"pattern":"Centro Ads\\.txt Crawler","url":"https://knownagents.com/agents/centro-ads-txt-crawler","description":"Advertising transparency ads.txt file verification crawler bot","tags":["advertising"]},{"pattern":"WISEbot","url":"https://www.cision.com/","description":"Cision media monitoring and PR analytics crawler bot","tags":["monitoring"]},{"pattern":"CodaBot","url":"https://coda.io/","description":"Coda document collaboration platform link preview crawler bot","tags":["social-preview"]},{"pattern":"Corporama matcher","url":"https://www.corporama.fr/","description":"Corporate data matching and business intelligence crawler bot","tags":["monitoring"]},{"pattern":"CyotekWebCopy","url":"https://www.cyotek.com/cyotek-webcopy","description":"Website copying and offline browsing tool crawler bot","tags":["archiver"]},{"pattern":"Datadog Agent","url":"https://www.datadoghq.com/","description":"Datadog infrastructure monitoring and synthetic testing agent bot","tags":["monitoring"]},{"pattern":"Dazzle BlueSky Bot","url":"https://knownagents.com/agents/dazzle-bluesky-bot","description":"BlueSky social network link preview and content crawler","tags":["social-preview"]},{"pattern":"DominicBot","url":"https://vanylla.org/bot","description":"Vanylla social network content indexing and discovery crawler","tags":["monitoring"]},{"pattern":"Dow Jones Searchbot","url":"https://www.dowjones.com/","description":"Dow Jones news and financial content aggregation crawler","tags":["monitoring"]},{"pattern":"Download Ninja","url":"https://knownagents.com/agents/download-ninja","description":"File download manager and web content fetching tool","tags":["http-library"]},{"pattern":"EmailWolf","url":"https://knownagents.com/agents/emailwolf","description":"Email address extraction and contact scraping crawler bot","tags":["scanner"]},{"pattern":"fedistatsCrawler","url":"https://knownagents.com/agents/fedistatscrawler","description":"Fediverse statistics collection and network analysis crawler bot","tags":["monitoring"]},{"pattern":"GoParserBot","url":"https://knownagents.com/agents/goparserbot","description":"Go-based web content parsing and extraction crawler bot","tags":["http-library"]},{"pattern":"gsa-crawler","url":"https://knownagents.com/agents/gsa-crawler","description":"Google Search Appliance enterprise search indexing crawler bot","tags":["search-engine"]},{"pattern":"HanaleiBot","url":"https://knownagents.com/agents/hanaleibot","description":"Testing and integration verification web crawler bot","tags":["monitoring"]},{"pattern":"NicheIndex","url":"https://nicheindex.co","description":"Niche market RSS feed harvesting and indexing crawler","tags":["feed-reader"]},{"pattern":"HeadOnlyScraper","url":"https://knownagents.com/agents/headonlyscraper","description":"HTTP HEAD request metadata collection and validation crawler","tags":["monitoring"]},{"pattern":"HenkBot","url":"https://valyu.ai/crawler","description":"Valyu AI content analysis and data collection crawler","tags":["ai-crawler"]},{"pattern":"Impact\\.com Agent","url":"https://impact.com/","description":"Impact.com affiliate marketing and partnership tracking crawler bot","tags":["advertising"]},{"pattern":"Keydrop\\.io","url":"https://onlyscans.com/about","description":"Keydrop gaming marketplace security scanning and monitoring crawler","tags":["scanner"]},{"pattern":"larbin","url":"https://larbin.sourceforge.net/","description":"Open-source web crawler for large-scale indexing projects","tags":["http-library"]},{"pattern":"SENTINEL-LinkCheck","url":"https://oblivionsearch.com/","description":"Sentinel link validation and broken link detection crawler","tags":["monitoring"]},{"pattern":"linko","url":"https://linko.app/crawler","description":"Linko link management and URL tracking service crawler","tags":["monitoring"],"rdns":[".linkody.com"]},{"pattern":"LinkpadBot","url":"https://linkpad.org/robot/","description":"Linkpad bookmark management and link discovery crawler bot","tags":["monitoring"],"rdns":[".linkpadbot.com"]},{"pattern":"lwp-trivial","url":"https://metacpan.org/pod/LWP","description":"Perl LWP library simple HTTP client and fetcher","tags":["http-library"]},{"pattern":"Magus Bot","url":"https://knownagents.com/agents/magus-bot","description":"Web content analysis and data extraction crawler bot","tags":["monitoring"]},{"pattern":"NaverBot","url":"https://www.naver.com/","description":"Naver Korean search engine web indexing crawler bot","tags":["search-engine"],"rdns":[".naver.com",".naver.jp"]},{"pattern":"loopimprovements\\.com","url":"http://loopimprovements.com/robot.html","description":"Network research and web data collection crawler bot","tags":["monitoring"]},{"pattern":"OpenTheBoxBot","url":"https://knownagents.com/agents/opentheboxbot","description":"Content discovery and web indexing service crawler bot","tags":["search-engine"]},{"pattern":"OWLer-W","url":"https://openwebsearch.eu/","description":"Business intelligence and company data collection crawler bot","tags":["monitoring"]},{"pattern":"peer39_crawler","url":"https://www.peer39.com/","description":"Peer39 contextual advertising and brand safety crawler bot","tags":["advertising"]},{"pattern":"Pixalate\\.com","url":"https://www.pixalate.com/","description":"Pixalate ad fraud detection and brand safety crawler","tags":["advertising"]},{"pattern":"Poduptime","url":"https://fediverse.observer","description":"Fediverse pod uptime monitoring and availability checker crawler","tags":["monitoring"]},{"pattern":"Pomothy-Bot","url":"https://knownagents.com/agents/pomothy-bot","description":"Web content monitoring and change detection crawler bot","tags":["monitoring"]},{"pattern":"PulsePoint-Crawler","url":"https://www.pulsepoint.com/","description":"PulsePoint programmatic advertising and ad verification crawler bot","tags":["advertising"]},{"pattern":"rawweb-bot","url":"https://knownagents.com/agents/rawweb-bot","description":"Raw web content extraction and data scraping crawler","tags":["http-library"]},{"pattern":"semantic-visions","url":"https://www.semantic-visions.com/","description":"Semantic content analysis and AI training data crawler","tags":["ai-crawler"]},{"pattern":"Sindup","url":"https://www.sindup.com/","description":"Sindup competitive intelligence and market monitoring crawler bot","tags":["monitoring"]},{"pattern":"SiteSucker","url":"https://ricks-apps.com/osx/sitesucker/","description":"macOS website downloading and offline browsing tool crawler","tags":["archiver"]},{"pattern":"SpringserveBot","url":"https://www.springserve.com/","description":"Springserve ad server and video advertising platform crawler","tags":["advertising"]},{"pattern":"SQWatcher","url":"http://sqcompliance.com/sqwatcher.html","description":"Software quality compliance monitoring and testing crawler bot","tags":["monitoring"]},{"pattern":"Supabase Paired Crawler","url":"https://supabase.com/","description":"Supabase database platform link preview and metadata crawler","tags":["social-preview"]},{"pattern":"sv-watchagent","url":"https://www.semantic-visions.com/","description":"Semantic Visions content monitoring and analysis crawler agent","tags":["monitoring"]},{"pattern":"Swiftbot","url":"https://swiftype.com/swiftbot","description":"Swiftype site search engine indexing and crawling bot","tags":["search-engine"]},{"pattern":"SynthesiBot","url":"https://knownagents.com/agents/synthesibot","description":"Content synthesis and data aggregation crawler bot","tags":["monitoring"]},{"pattern":"TaraGroup Intelligent Bot","url":"https://knownagents.com/agents/taragroup-intelligent-bot","description":"TaraGroup AI-powered web intelligence and analysis crawler","tags":["ai-crawler"]},{"pattern":"Thinkbot","url":"https://boston.conman.org/2025/08/21.1","description":"Experimental AI thinking and reasoning web crawler bot","tags":["ai-crawler"]},{"pattern":"TSMbot","url":"https://knownagents.com/agents/tsmbot","description":"TSM brand monitoring and content tracking crawler bot","tags":["monitoring"]},{"pattern":"TSM-turingos","url":"https://knownagents.com/agents/turingos","description":"TuringOS AI-powered content analysis and monitoring crawler","tags":["ai-crawler"]},{"pattern":"UGAResearchAgent","url":"https://nislabuga-scan.uga.edu/","description":"University of Georgia academic network research crawler bot","tags":["academic"]},{"pattern":"UrlSuMa\\.de crawler","url":"https://urlsuma.de/","description":"URL summarization and content preview generation crawler bot","tags":["monitoring"]},{"pattern":"WanscannerBot","url":"https://abuse.pend.re","description":"Network security scanning and vulnerability detection crawler bot","tags":["scanner"]},{"pattern":"WebCapture","url":"https://knownagents.com/agents/webcapture-2-0","description":"Website screenshot and page capture archiving tool crawler","tags":["archiver"]},{"pattern":"WebCopier","url":"http://www.maximumsoft.com/","description":"Website copying and offline browsing tool crawler bot","tags":["archiver"]},{"pattern":"cognitiveseo\\.com","url":"https://cognitiveseo.com/bot.html","description":"CognitiveSEO backlink analysis and SEO research crawler bot","tags":["seo"]},{"pattern":"Xing Bot","url":"https://www.xing.com/","description":"Xing professional network link preview and content crawler","tags":["social-preview"]},{"pattern":"XML Sitemaps Generator","url":"https://www.xml-sitemaps.com/","description":"XML sitemap generation and website structure mapping crawler","tags":["seo"]},{"pattern":"YandoriRSSBot","url":"https://knownagents.com/agents/yandorirssbot","description":"RSS feed aggregation and content syndication crawler bot","tags":["feed-reader"]},{"pattern":"Zealbot","url":"https://knownagents.com/agents/zealbot","description":"Web content indexing and search engine crawler bot","tags":["search-engine"]},{"pattern":"DuplexWeb-Google","url":"https://searchengineland.com/google-shut-down-duplex-on-the-web-389999","description":"Google Assistant automated web interaction for reservations and purchases","tags":["browser-automation"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"AppleNewsBot","url":"https://developer.apple.com/documentation/applenews/preparing-image-video-audio-music-and-arkit-assets","description":"Apple News content fetching and article updates","tags":["feed-reader","search-engine"],"rdns":[".applebot.apple.com"]},{"pattern":"Applebot-Extended","url":"https://support.apple.com/en-gb/119829","description":"Apple generative AI model training data collection","tags":["ai-crawler"],"rdns":[".applebot.apple.com"]},{"pattern":"iTMS","url":"https://support.apple.com/en-gb/119829","description":"Apple Podcasts feed retrieval and content updates","tags":["feed-reader"]},{"pattern":"SeznamEmailProxy","url":"https://o-seznam.cz/napoveda/email/mohlo-by-se-hodit/postovni-programy-a-aplikace/","description":"Seznam webmail email image proxy service","tags":["social-preview"],"rdns":[".seznam.cz"]},{"pattern":"Aolbot-News","url":"https://help.aol.com/articles/aolbot","description":"AOL news article crawler for aol.com content","tags":["search-engine"],"rdns":[".aol.com"]},{"pattern":"OnPageBot","url":"https://bot.onpage.org/","description":"OnPage.org legacy crawler now rebranded as Ryte","tags":["seo"],"rdns":[".onpage.org",".ryte.com"]},{"pattern":"Andibot","url":"https://andi.com/","description":"Andi AI search chatbot and answer engine","tags":["ai-fetcher"],"rdns":[".andiai.com",".andisearch.com"]},{"pattern":"ArchivetodayBot","url":"https://archive.ph/","description":"Archive.today web archiving service using headless Chrome","tags":["archiver"],"rdns":[".archive.ph",".archive.today"]},{"pattern":"NewRelicPinger","url":"https://support.newrelic.com/ForumsRedirectPage","description":"New Relic synthetic ping monitor for availability","tags":["monitoring"],"rdns":[".newrelic.com"]},{"pattern":"Spinn3r","url":"http://spinn3r.com/robot","description":"Defunct commercial crawler for blogs and news","tags":["feed-reader"],"rdns":[".spinn3r.com"]},{"pattern":"Prerender","url":"https://docs.prerender.io/docs/33-overview-of-prerender-crawlers","description":"JavaScript rendering service for search engine crawlers","tags":["seo"],"rdns":[".prerender.io"]},{"pattern":"StackRambler","url":"https://useragentstring.com/pages/useragentstring.php?name=StackRambler","description":"Defunct Rambler.ru search engine crawler from Stack Ltd","tags":["search-engine"],"rdns":[".rambler.ru"]},{"pattern":"Sputnik(Image)?Bot","url":"http://corp.sputnik.ru/webmaster","description":"Defunct Russian state search engine crawler shut down 2020","tags":["search-engine"],"rdns":[".sputnik.ru"]},{"pattern":"EasouSpider","url":"http://www.easou.com/search/spider.html","description":"Chinese mobile search engine crawler largely dormant","tags":["search-engine"],"rdns":[".easou.com"]},{"pattern":"EtaoSpider","url":"http://open.etao.com/dev/EtaoSpider","description":"Alibaba Etao shopping search engine crawler","tags":["search-engine"],"rdns":[".etaospider.com",".taobao.com"]},{"pattern":"Alibaba\\.Security\\.Heimdall","url":"https://udger.com/resources/ua-list/bot-detail?bot=Alibaba+crawler","description":"Alibaba Group security scanning bot","tags":["scanner"],"rdns":[".alibaba-inc.com",".alibaba.com"]},{"pattern":"[Jj]ike[Ss]pider","url":"http://shoulu.jike.com/spider.html","description":"Defunct Chinese state search engine crawler merged 2014","tags":["search-engine"],"rdns":[".jike.com"]},{"pattern":"Burp Suite","url":"https://portswigger.net/burp","description":"PortSwigger web application security testing platform","tags":["scanner"],"rdns":[".burpcollaborator.net",".oastify.com",".portswigger.net"]},{"pattern":"Rapid7","url":"https://www.rapid7.com/","description":"Rapid7 vulnerability scanner and security testing platform","tags":["scanner"],"rdns":[".rapid7.com"]},{"pattern":"nuclei","url":"https://docs.projectdiscovery.io/opensource/nuclei/running","description":"ProjectDiscovery fast customizable vulnerability scanner","tags":["scanner"],"rdns":[".projectdiscovery.io"]},{"pattern":"Siteliner","url":"http://www.siteliner.com/bot","description":"Indigo Stream website analysis and duplicate content checker","tags":["seo"],"rdns":[".siteliner.com"]},{"pattern":"mabl","url":"https://help.mabl.com/hc/en-us/articles/18967650734228","description":"mabl cloud test automation platform for web testing","tags":["browser-automation"],"rdns":[".mabl.com"]},{"pattern":"Metasploit","url":"https://www.metasploit.com/","description":"Rapid7 penetration testing and exploitation framework","tags":["scanner"],"rdns":[".metasploit.com"]},{"pattern":"OWASP ZAP","url":"https://www.zaproxy.org/","description":"Zed Attack Proxy web application security scanner","tags":["scanner"],"rdns":[".zap.org",".zaproxy.org"]},{"pattern":"Copyscape","url":"https://www.copyscape.com/","description":"Indigo Stream plagiarism detection and content protection","tags":["seo"],"rdns":[".copyscape.com"]},{"pattern":"BrowserStack","url":"https://www.browserstack.com/","description":"Cloud cross-browser testing platform using real browsers","tags":["browser-automation"],"rdns":[".browserstack.com"]},{"pattern":"Sauce Labs","url":"https://saucelabs.com/","description":"Cloud continuous testing platform for cross-browser testing","tags":["browser-automation"],"rdns":[".saucelabs.com",".saucelabs.net"]},{"pattern":"LambdaTest","url":"https://www.testmuai.com/","description":"Cloud cross-browser testing platform using real browsers","tags":["browser-automation"],"rdns":[".lambdatest.com"]},{"pattern":"Testim","url":"https://www.testim.io/","description":"AI-powered test automation platform using real browsers","tags":["browser-automation"],"rdns":[".testim.io"]},{"pattern":"Katalon","url":"https://www.katalon.com/","description":"Test automation platform using real browser engines","tags":["browser-automation"],"rdns":[".katalon.com"]},{"pattern":"Cypress","url":"https://www.cypress.io/","description":"JavaScript end-to-end testing framework using Chromium","tags":["browser-automation"],"rdns":[".cypress.io"]},{"pattern":"DynatraceSynthetic","url":"https://docs.dynatrace.com/docs/observe/digital-experience/synthetic-monitoring","description":"Dynatrace synthetic monitoring HTTP checks","tags":["monitoring"],"rdns":[".dynatrace.com"]},{"pattern":"GomezAgent","url":"https://www.compuware.com/","description":"Legacy Compuware Gomez APM synthetic monitoring recorder","tags":["monitoring"],"rdns":[".dynatrace.com"]},{"pattern":"Prisync","url":"https://prisync.com/","description":"Price monitoring service intentionally concealing bot identity","tags":["seo"],"rdns":[".prisync.com"]},{"pattern":"Price2Spy","url":"https://www.price2spy.com/","description":"Price monitoring service bypassing bot protection systems","tags":["seo"],"rdns":[".price2spy.com"]},{"pattern":"Langchain","url":"https://www.langchain.com/","description":"AI Agent development and execution platform","tags":["ai-fetcher"]},{"pattern":"[Oo][Aa][Ii]-[Aa]ds[Bb]ot","url":"https://developers.openai.com/api/docs/bots","description":"OpenAI ads bot for validating ad landing pages on ChatGPT","tags":["advertising"]},{"pattern":"GoogleAgent-URLContext","url":"https://ai.google.dev/gemini-api/docs/url-context","description":"Google Gemini agent providing URL context for agentic AI","tags":["ai-fetcher"]},{"pattern":"1001FirmsBot","url":"https://www.1001firms.com/robot.php","description":"Business directory indexer","tags":["seo"]},{"pattern":"200PleaseBot","url":"http://www.200please.com/bot","description":"HTTP status monitoring crawler","tags":["monitoring"]},{"pattern":"AdnormCrawler","url":"http://www.adnorm.com/crawler","description":"Advertising content analyzer","tags":["advertising"]},{"pattern":"AfD-Verbotsverfahren","url":"https://www.webrobots.de/bot/afd-verbotsverfahren/","description":"Political content crawler","tags":["academic"]},{"pattern":"anonymous coward","url":"https://github.com/apache/stormcrawler","description":"Storm-based web crawler","tags":["http-library"]},{"pattern":"archive-tv\\.com","url":"http://archive-tv.com/bot","description":"TV content archiver","tags":["archiver"]},{"pattern":"BPImageWalker","url":"http://www.bdbrandprotect.com","description":"Brand protection image crawler","tags":["monitoring"]},{"pattern":"bulk-retrieval bot","url":"http://varocarbas.com/bulk-retrieval_bot.php","description":"Bulk content retrieval tool","tags":["http-library"]},{"pattern":"ca-crawler","url":"https://www.webrobots.de/bot/ca-crawler/","description":"General purpose web crawler","tags":["http-library"]},{"pattern":"Calypso v","url":"https://www.webrobots.de/bot/calypso-v/","description":"Web content indexer","tags":["http-library"]},{"pattern":"Claritybot","url":"https://www.webrobots.de/bot/claritybot/","description":"Web analytics crawler","tags":["monitoring"]},{"pattern":"CMS Crawler","url":"http://www.cmscrawler.com","description":"CMS detection and analysis","tags":["scanner"]},{"pattern":"CoinCornerBot","url":"https://www.coincorner.com/BitcoinBot","description":"Bitcoin exchange content crawler","tags":["monitoring"]},{"pattern":"contentDetection","url":"https://www.mindup.de/","description":"Content analysis and detection","tags":["monitoring"]},{"pattern":"CRAZYWEBCRAWLER","url":"http://www.crazywebcrawler.com","description":"General web crawler","tags":["http-library"]},{"pattern":"DaaS\\.sh research bot","url":"http://www.daas.sh/delist","description":"Research data collection bot","tags":["academic"]},{"pattern":"Digincore crawler bot","url":"https://www.digincore.com/crawler.html","description":"Digital core content crawler","tags":["seo"]},{"pattern":"DomainCheck\\.io Crawler","url":"https://domaincheck.io","description":"Domain availability checker","tags":["monitoring"]},{"pattern":"DomainSigmaCrawler","url":"http://domainsigma.com/robot","description":"Domain analysis crawler","tags":["seo"]},{"pattern":"elefent","url":"http://crawler.elefent.net","description":"Web content crawler","tags":["http-library"]},{"pattern":"EuripBot","url":"https://www.eurip.com/","description":"European IP content crawler","tags":["seo"]},{"pattern":"Everyonedomainsbot","url":"https://expireddomains.com/domain/everyone.domains","description":"Domain registry crawler","tags":["seo"]},{"pattern":"fastbot\\.de crawler","url":"http://www.fastbot.de","description":"Fast web indexing bot","tags":["search-engine"]},{"pattern":"Fetchbot","url":"https://github.com/PuerkitoBio/fetchbot","description":"Go-based web crawler library","tags":["http-library"]},{"pattern":"GDPR-Bot","url":"https://www.webrobots.de/bot/gdpr-bot/","description":"GDPR compliance checker","tags":["scanner"]},{"pattern":"GetIntent Crawler","url":"http://getintent.com/bot.html","description":"Intent analysis crawler","tags":["advertising"]},{"pattern":"Gimme60bot","url":"http://gimme60.com","description":"Content aggregation bot","tags":["seo"]},{"pattern":"Gosign-Security-Crawler","url":"https://www.gosign.de/de/","description":"TYPO3 security monitoring","tags":["scanner"]},{"pattern":"HRCrawler","url":"https://www.webrobots.de/bot/hrcrawler/","description":"HR content crawler","tags":["seo"]},{"pattern":"HyperCrawl","url":"http://www.seograph.net/bot.html","description":"SEO graph analysis crawler","tags":["seo"]},{"pattern":"iCjobs","url":"http://www.icjobs.de","description":"Job listing crawler","tags":["seo"]},{"pattern":"HybridBot","url":"https://hybrid.ru:80/about/","description":"Russian hybrid search bot","tags":["search-engine"]},{"pattern":"ImplisenseBot","url":"https://www.webrobots.de/bot/implisensebot/","description":"Business intelligence crawler","tags":["seo"]},{"pattern":"Insitesbot","url":"https://www.webrobots.de/bot/insitesbot/","description":"Site analysis bot","tags":["seo"]},{"pattern":"Keybot Translation-Search-Machine","url":"https://www.webrobots.de/bot/keybot-translation-search-machin/","description":"Translation search engine","tags":["search-engine"]},{"pattern":"LinqiaCrawlerBot","url":"https://www.linqia.com","description":"Influencer marketing crawler","tags":["advertising"]},{"pattern":"LoadTimeBot","url":"http://www.loadtime.net/bot.html","description":"Page load time monitoring","tags":["monitoring"]},{"pattern":"meanpathbot","url":"http://www.meanpath.com/meanpathbot.html","description":"Content path analysis","tags":["seo"]},{"pattern":"Meta_Bot","url":"https://www.webrobots.de/bot/meta_bot/","description":"Metadata extraction bot","tags":["seo"]},{"pattern":"SearchmetricsBot","url":"https://www.conductor.com:443/","description":"SEO metrics analysis","tags":["seo"]},{"pattern":"MnoGoSearch","url":"https://www.webrobots.de/bot/mnogosearch/","description":"Open source search engine","tags":["search-engine"]},{"pattern":"MSIECrawler","url":"https://www.webrobots.de/bot/msiecrawler/","description":"IE-based web crawler","tags":["browser-automation"]},{"pattern":"NetzCheckBot","url":"http://www.netzcheck.com","description":"Security seal verification","tags":["scanner"]},{"pattern":"pmoz\\.info ODP link checker","url":"http://pmoz.info/doc/botinfo.htm","description":"ODP directory link checker","tags":["monitoring"]},{"pattern":"OFM search_robot","url":"https://www.webrobots.de/bot/ofm-search_robot/","description":"Search engine robot","tags":["search-engine"]},{"pattern":"OpenWebSpider","url":"http://www.openwebspider.org/","description":"Open source web spider","tags":["http-library"]},{"pattern":"PagesInventory","url":"https://sedo.com/search/?keyword=pagesinventory.com","description":"Page inventory crawler","tags":["seo"]},{"pattern":"PathDefender","url":"http://www.pathdefender.com/help/crawler","description":"Security path analysis","tags":["scanner"]},{"pattern":"Plukkie","url":"http://www.botje.com/plukkie.htm","description":"Dutch web crawler","tags":["search-engine"]},{"pattern":"PPBot","url":"https://pagepeeker.com/robots/","description":"Page preview screenshot bot","tags":["monitoring"]},{"pattern":"publiclibraryarchive\\.org","url":"http://publiclibraryarchive.org","description":"Public library archiving","tags":["archiver"]},{"pattern":"PycURL","url":"http://pycurl.io/","description":"Python cURL library","tags":["http-library"]},{"pattern":"R6_CommentReader","url":"http://www.radian6.com/crawler","description":"Social media comment reader","tags":["monitoring"]},{"pattern":"R6_FeedFetcher","url":"http://www.radian6.com/crawler","description":"Social media feed fetcher","tags":["feed-reader"]},{"pattern":"RankFlex\\.com Webspider","url":"https://www.sedoparking.com/rankflex.com","description":"SEO ranking analysis","tags":["seo"]},{"pattern":"rpm-bot","url":"https://www.webrobots.de/bot/rpm-bot/","description":"RPM package crawler","tags":["http-library"]},{"pattern":"ScreenerBot Crawler","url":"http://www.ScreenerBot.com","description":"Screen capture crawler","tags":["monitoring"]},{"pattern":"seebot","url":"http://www.seegnify.com/bot","description":"Visual search crawler","tags":["search-engine"]},{"pattern":"seedingup/status_crawler","url":"https://www.webrobots.de/bot/seedingup/","description":"Seeding status monitoring","tags":["monitoring"]},{"pattern":"SEOdiver","url":"http://www.seodiver.com/bot","description":"SEO deep analysis","tags":["seo"]},{"pattern":"SpuhexBot","url":"https://www.webrobots.de/bot/spuhexbot/","description":"Web content crawler","tags":["http-library"]},{"pattern":"ssearch_bot","url":"http://www.semantissimo.de","description":"Semantic search crawler","tags":["search-engine"]},{"pattern":"Stasebot","url":"http://www.stase.eu/about-the-stasebot","description":"Web archiving bot","tags":["archiver"]},{"pattern":"stq_bot","url":"https://www.searchteq.de/","description":"Search technology bot","tags":["search-engine"]},{"pattern":"TagVisit","url":"http://tag-visit.com","description":"Tag analysis crawler","tags":["advertising"]},{"pattern":"ThumbShotsBot","url":"https://www.webrobots.de/bot/thumbshotsbot/","description":"Thumbnail screenshot generator","tags":["monitoring"]},{"pattern":"TouTrix crawler","url":"https://www.webrobots.de/bot/toutrix-crawler/","description":"Web content crawler","tags":["http-library"]},{"pattern":"TwitterCrawler","url":"https://www.webrobots.de/bot/twittercrawler/","description":"Twitter content crawler","tags":["social-preview"]},{"pattern":"uCrawler","url":"https://blog.ucoz.ru/upolicy","description":"Ucoz platform crawler","tags":["http-library"]},{"pattern":"uMBot","url":"https://www.ubermetrics-technologies.com/","description":"Ubermetrics social monitoring","tags":["monitoring"]},{"pattern":"Allmystery UrlBot","url":"https://www.webrobots.de/bot/urlbot/","description":"URL verification bot","tags":["monitoring"]},{"pattern":"URLfilterDB-crawler","url":"https://www.webrobots.de/bot/urlfilterdb-crawler/","description":"URL filter database","tags":["scanner"]},{"pattern":"Virusdie crawler","url":"https://www.webrobots.de/bot/virusdie-crawler/","description":"Malware detection crawler","tags":["scanner"]},{"pattern":"WASALive-Bot","url":"http://blog.wasalive.com/wasalive-bots/","description":"Live content monitoring","tags":["monitoring"]},{"pattern":"WeSEE:Ads/PageBot","url":"http://www.wesee.com/bot/","description":"Ad verification bot","tags":["advertising"]},{"pattern":"WeViKaBot","url":"http://www.wevika.de/","description":"German web crawler","tags":["search-engine"]},{"pattern":"WSB-BOT","url":"http://www.webseitenbewertung.com/","description":"Website evaluation bot","tags":["seo"]},{"pattern":"XenzuuBot","url":"https://www.xenzuu.com","description":"Social network crawler","tags":["social-preview"]},{"pattern":"crawler-complaints@gmx\\.de","url":"https://www.webrobots.de/bot/compatible-crawler/","description":"Generic compatible web crawler","tags":["http-library"]},{"pattern":"IABTechLab[^/]*Ads\\.txt","url":"https://iabtechlab.com/ads-txt/ads-txt-aggregator/","description":"IAB Tech Lab ads.txt aggregator crawler","tags":["advertising"]},{"pattern":"coccocbot-image","url":"https://coccoc.com/search/console/en/coc-coc-robots","description":"Vietnamese search engine image crawler","tags":["search-engine"],"rdns":[".coccoc.com"]},{"pattern":"coccocbot-web","url":"https://coccoc.com/search/console/en/coc-coc-robots","description":"Vietnamese search engine web crawler","tags":["search-engine"],"rdns":[".coccoc.com"]},{"pattern":"PubMatic[^/]*Crawler","url":"https://pubmatic.com/blog/introducing-crawler-2-0-the-next-generation-ads-txt-crawler-at-pubmatic/","description":"PubMatic ads.txt verification crawler","tags":["advertising"]},{"pattern":"Xovi(bot|OnpageCrawler)","url":"https://knownagents.com/agents/xovibot","description":"XOVI SEO analysis crawler","tags":["seo"]},{"pattern":"awin\\.com","url":"https://datadome.co/bots/awin-com-crawler/","description":"Awin affiliate marketing network crawler","tags":["advertising"]},{"pattern":"thumbshots-de","url":"https://udger.com/resources/ua-list/bot-detail?bot=thumbshots-de-Bot","description":"Website thumbnail screenshot generation service","tags":["monitoring"]},{"pattern":"\"echocrawl ","url":"http://search.thunderstone.com/texis/websearch/about.html","description":"Thunderstone search engine crawler","tags":["search-engine"]},{"pattern":"\\/n software IPWorks HTTP\\/S Component","url":"https://www.nsoftware.com","description":"HTTP/S component library by /n software","tags":["http-library"]},{"pattern":"13TABS","url":"https://www.13tabs.com/aboutUs.jsp","description":"13TABS web crawler","tags":["search-engine"]},{"pattern":"192\\.comAgent","description":"192.com web crawler","tags":["search-engine"]},{"pattern":"1Pilot Monitoring","url":"http://1pilot.io/","description":"1Pilot website monitoring service","tags":["monitoring"]},{"pattern":"404 Checker","url":"http://www.404checker.com/user-agent","description":"404 link checker service","tags":["monitoring"]},{"pattern":"50\\.nu","url":"http://50.nu/bot.html","description":"50.nu web crawler","tags":["search-engine"]},{"pattern":"UnChaos","url":"http://www.unchaos.com/","description":"UnChaos hybrid web search engine crawler","tags":["search-engine"]},{"pattern":"AFB\\/","url":"http://allloadin.com","description":"AllLoadIn web crawler","tags":["search-engine"]},{"pattern":"ANAM","url":"http://internetsecure.today/","description":"Security scanner for exposed .git directories using raw TCP","tags":["scanner"]},{"pattern":"AWS Security Scanner","description":"AWS security vulnerability scanner","tags":["scanner"]},{"pattern":"search\\.thunderstone\\.com","url":"http://search.thunderstone.com/texis/websearch/about.html","description":"Thunderstone search engine crawler","tags":["search-engine"]},{"pattern":"AboutUsBot","url":"http://www.aboutus.org/Aboutus:Bot","description":"AboutUs.org website analysis crawler","tags":["seo"]},{"pattern":"Abrave","url":"http://robot.abrave.com","description":"Abrave search engine web crawler","tags":["search-engine"]},{"pattern":"Accoona-AI-Agent","description":"Accoona AI-powered search engine crawler","tags":["search-engine"]},{"pattern":"Acoon","url":"http://www.acoon.de","description":"Acoon German search engine crawler","tags":["search-engine"]},{"pattern":"AdAuth","url":"https://www.adauth.com","description":"AdAuth advertising authentication and verification bot","tags":["advertising"]},{"pattern":"Adstxtaggregator","url":"http://adstxtaggregator.com","description":"Adstxt aggregator for advertising verification","tags":["advertising"]},{"pattern":"Adyen HttpClient","description":"Adyen payment platform HTTP client","tags":["http-library"]},{"pattern":"Aether","description":"Aether HTTP client library","tags":["http-library"]},{"pattern":"Airmail","description":"Airmail email client for macOS","tags":["http-library"]},{"pattern":"Amazon Simple Notification Service Agent","description":"AWS SNS notification service agent","tags":["monitoring"]},{"pattern":"Amazon-Route53-Health-Check-Service","url":"http://amzn.to/1vsZADi","description":"AWS Route53 health check monitoring service","tags":["monitoring"]},{"pattern":"AndroidDownloadManager","description":"Android system download manager","tags":["http-library"]},{"pattern":"Anemone","description":"Anemone web crawler library","tags":["http-library"]},{"pattern":"AngleSharp","description":"AngleSharp .NET HTML parser library","tags":["http-library"]},{"pattern":"Anthill","description":"Anthill web crawler","tags":["search-engine"]},{"pattern":"Apache Ant","description":"Apache Ant build automation tool","tags":["http-library"]},{"pattern":"Apache Droid","description":"Apache Droid HTTP client","tags":["http-library"]},{"pattern":"Apache OpenOffice","description":"Apache OpenOffice document suite HTTP client","tags":["http-library"]},{"pattern":"Apache-HttpAsyncClient","description":"Apache async HTTP client library for Java","tags":["http-library"]},{"pattern":"ApacheBench","description":"Apache HTTP server benchmarking tool","tags":["monitoring"]},{"pattern":"AppManager RPT-HTTPClient","description":"AppManager HTTP client for monitoring","tags":["monitoring"]},{"pattern":"AppStoreScraperZ","description":"App Store data scraper tool","tags":["seo"]},{"pattern":"AppleSyndication","description":"Apple RSS feed syndication client","tags":["feed-reader"]},{"pattern":"ArielisBot","url":"http://arielis.com/","description":"Arielis search engine crawler","tags":["search-engine"]},{"pattern":"Arukereso","description":"Arukereso price comparison crawler","tags":["seo"]},{"pattern":"AskQuickly","url":"http://askquickly.org/","description":"AskQuickly search engine crawler","tags":["search-engine"]},{"pattern":"Astute SRM","description":"Astute supplier relationship management bot","tags":["seo"]},{"pattern":"AutomaticWPTester","description":"Automatic WordPress testing tool","tags":["monitoring"]},{"pattern":"Automattic Analytics Crawler","url":"http://wordpress.com/crawler/","description":"WordPress.com analytics crawler by Automattic","tags":["seo"]},{"pattern":"Autonomy","description":"Autonomy enterprise search crawler","tags":["search-engine"]},{"pattern":"BCKLINKS","description":"Backlinks checker and analyzer","tags":["seo"]},{"pattern":"BKCTwitterUnshortener","description":"Harvard Berkman Klein Center Twitter URL unshortener","tags":["academic"]},{"pattern":"BLEXBot","description":"BLEXBot web crawler for SEO analysis","tags":["seo"]},{"pattern":"BabalooSpider","url":"http://www.babaloo.si","description":"Babaloo search engine crawler","tags":["search-engine"]},{"pattern":"Backlink-Ceck\\.de","url":"http://www.backlink-check.de/bot.html","description":"Backlink checker and analysis tool","tags":["seo"]},{"pattern":"BacklinkCrawler","url":"http://www.backlinktest.com/crawler.html","description":"Backlink testing and analysis crawler","tags":["seo"]},{"pattern":"Bad-Neighborhood Link Analyzer","url":"http://www.bad-neighborhood.com/","description":"Bad neighborhood link quality analyzer","tags":["seo"]},{"pattern":"Barracuda Sentinel","description":"Barracuda email security scanner","tags":["scanner"]},{"pattern":"Bewica-security-scan","description":"Bewica security vulnerability scanner","tags":["scanner"]},{"pattern":"Bidtellect","description":"Bidtellect advertising platform crawler","tags":["advertising"]},{"pattern":"Bitrix link preview","description":"Bitrix CMS link preview generator","tags":["social-preview"]},{"pattern":"BizwikiBot","url":"http://www.bizwiki.com/bizwikibot.htm","description":"Bizwiki business directory crawler","tags":["search-engine"]},{"pattern":"BlockNote\\.Net","description":"BlockNote note-taking application HTTP client","tags":["http-library"]},{"pattern":"BlogBridge","url":"http://www.blogbridge.com/","description":"BlogBridge RSS feed reader","tags":["feed-reader"]},{"pattern":"BlogPulseLive","description":"BlogPulse blog search and analytics crawler","tags":["search-engine"]},{"pattern":"BlogSearch","url":"http://www.icerocket.com/","description":"IceRocket blog search engine crawler","tags":["search-engine"]},{"pattern":"BoardReader Favicon Fetcher","description":"BoardReader forum search favicon fetcher","tags":["seo"]},{"pattern":"Boost\\.Beast","description":"Boost.Beast C++ HTTP library","tags":["http-library"]},{"pattern":"BotOnParade","url":"http://www.bots-on-para.de/bot.html","description":"BotOnParade web crawler","tags":["search-engine"]},{"pattern":"BrainbruBot","url":"http://www.brainbru.com/","description":"Brainbru search engine crawler","tags":["search-engine"]},{"pattern":"Braintree-Webhooks","description":"Braintree payment platform webhook client","tags":["http-library"]},{"pattern":"Branch Metrics API","description":"Branch mobile deep linking API client","tags":["http-library"]},{"pattern":"Branch-Passthrough","description":"Branch deep linking passthrough client","tags":["http-library"]},{"pattern":"Brodie","url":"http://www.15miles.com/","description":"15miles local business search crawler","tags":["search-engine"]},{"pattern":"Browsershots","description":"Browsershots cross-browser testing service","tags":["monitoring"]},{"pattern":"Burf Search Engine","description":"Burf search engine crawler","tags":["search-engine"]},{"pattern":"CATExplorador","url":"http://domini.cat/catexplorador.html","description":"Catalan domain registry crawler","tags":["search-engine"]},{"pattern":"CDR[\\S ]* Simulator","url":"http://timewe.net","description":"CDR mobile simulator crawler","tags":["http-library"]},{"pattern":"CERT\\.at-Statistics-Survey","url":"http://www.cert.at/about/consec/content.html","description":"CERT.at security statistics survey bot","tags":["scanner"]},{"pattern":"CISPA","url":"https://notify.cispa.saarland","description":"CISPA security vulnerability notification scanner","tags":["scanner"]},{"pattern":"CJNetworkQuality","url":"http://www.cj.com/networkquality","description":"CJ network quality monitoring bot","tags":["monitoring"]},{"pattern":"COMODO DCV","description":"Comodo domain control validation bot","tags":["scanner"]},{"pattern":"CSS Certificate Spider","url":"http://www.css-security.com/certificatespider/","description":"CSS Security certificate validation spider","tags":["scanner"]},{"pattern":"CSSCheck","description":"CSS validation and checking tool","tags":["monitoring"]},{"pattern":"CSmartCrawler","description":"CSmart advertising crawler","tags":["advertising"]},{"pattern":"CakePHP","description":"CakePHP framework HTTP client","tags":["http-library"]},{"pattern":"CamontSpider","url":"http://epweb2.ph.bham.ac.uk/user/slater/camont/info.html","description":"Camont academic research crawler","tags":["academic"]},{"pattern":"Canary%20Mail","description":"Canary Mail email client for macOS","tags":["http-library"]},{"pattern":"Castabot","url":"http://topixtream.com/","description":"Castabot content aggregation crawler","tags":["search-engine"]},{"pattern":"CatchBot","url":"http://www.catchbot.com","description":"CatchBot search engine crawler","tags":["search-engine"]},{"pattern":"Chromaxa","description":"Chromaxa web crawler with Sonar","tags":["search-engine"]},{"pattern":"Cityreview Robot","url":"http://www.cityreview.org/crawler/","description":"Cityreview local business crawler","tags":["search-engine"]},{"pattern":"Cliqz Bot","url":"http://www.cliqz.com","description":"Cliqz privacy-focused search engine crawler","tags":["search-engine"]},{"pattern":"Cloud mapping experiment","description":"PDR Labs cloud mapping research crawler","tags":["academic"]},{"pattern":"CloudEndure Scanner","description":"CloudEndure disaster recovery scanner","tags":["scanner"]},{"pattern":"ColdFusion","description":"Adobe ColdFusion HTTP client","tags":["http-library"]},{"pattern":"Comodo[- ](?:SSL|Spider|Certificates)","description":"Comodo SSL certificate validation spider","tags":["scanner"]},{"pattern":"Comodo-Webinspector-Crawler","url":"http://www.comodorobot.com","description":"Comodo web security inspection crawler","tags":["scanner"]},{"pattern":"Contacts Crawler","url":"http://www.scrapinghub.com","description":"ScrapingHub contacts data crawler","tags":["seo"]},{"pattern":"Contextual Code Sites Explorer","description":"Contextual code exploration crawler","tags":["academic"]},{"pattern":"ConveraCrawler\\/","url":"http://www.authoritativeweb.com/crawl","description":"Convera authoritative web crawler","tags":["search-engine"]},{"pattern":"CookieReports","url":"http://cookiereports.com","description":"CookieReports cookie compliance scanner","tags":["scanner"]},{"pattern":"Corax","description":"Corax cybersecurity scanner","tags":["scanner"]},{"pattern":"CorpusCrawler","url":"http://corpora.fi.muni.cz/crawler/","description":"Czech corpus linguistic research crawler","tags":["academic"]},{"pattern":"Covario","url":"http://www.covario.com/ids","description":"Covario SEO and marketing intelligence crawler","tags":["seo"]},{"pattern":"Craw\\/","description":"Craw crawler","tags":["seo"]},{"pattern":"CronDroid","url":"http://www.crondroid.com","description":"CronDroid automated crawler","tags":["seo"]},{"pattern":"Cronjob\\.de","url":"https://www.cronjob.de","description":"Cronjob.de monitoring service","tags":["monitoring"]},{"pattern":"Crowsnest\\/","url":"http://www.crowsnest.tv/","description":"Crowsnest crawler","tags":["seo"]},{"pattern":"Curious George","url":"https://www.analyticsseo.com/crawler","description":"AnalyticsSEO crawler","tags":["seo"]},{"pattern":"DAP\\/NetHTTP","description":"DAP NetHTTP client","tags":["http-library"]},{"pattern":"DER-bot","url":"https://goo.gl/9ZibHG","description":"DER bot crawler","tags":["seo"]},{"pattern":"DHSH","description":"DHSH security and statistics checker","tags":["scanner"]},{"pattern":"DNS-Tools Header-Analyzer","description":"DNS-Tools header analysis tool","tags":["monitoring"]},{"pattern":"DNSPod-reporting","url":"http://www.dnspod.cn/reporting","description":"DNSPod reporting bot","tags":["monitoring"]},{"pattern":"DWDS-Crawler","url":"http://odo.dwds.de/dwds-crawler.html","description":"DWDS dictionary crawler","tags":["academic"]},{"pattern":"DangDang","description":"DangDang crawler","tags":["seo"]},{"pattern":"DataparkSearch\\/","url":"http://www.dataparksearch.org/","description":"DataparkSearch engine crawler","tags":["search-engine"]},{"pattern":"DavClnt","description":"WebDAV client","tags":["http-library"]},{"pattern":"DealGates Bot\\/","url":"http://spider.dealgates.com/bot.html","description":"DealGates spider","tags":["seo"]},{"pattern":"DemandbasePublisherAnalyzer\\/","url":"http://www.demandbase.com","description":"Demandbase publisher analytics bot","tags":["advertising"]},{"pattern":"Dex Social Bot","description":"Dex social media bot","tags":["social-preview"]},{"pattern":"DialogSearch\\.com Bot","url":"http://dialogsearch.com/webmasters","description":"DialogSearch search engine bot","tags":["search-engine"]},{"pattern":"Dispatch\\/","description":"Dispatch HTTP client library","tags":["http-library"]},{"pattern":"DoCoMo[^\\n]*moba-crawler","url":"http://crawler.dena.jp/","description":"DeNA mobile crawler","tags":["seo"]},{"pattern":"Docoloc","description":"Docoloc crawler","tags":["seo"]},{"pattern":"Dolphin http client\\/","description":"Dolphin Android HTTP client","tags":["http-library"]},{"pattern":"DomainLabz","description":"DomainLabz domain research crawler","tags":["seo"]},{"pattern":"DomainsDB\\.net MetaCrawler","url":"http://domainsdb.net/","description":"DomainsDB domain database crawler","tags":["seo"]},{"pattern":"Donuts Content Explorer","url":"https://www.donuts.domains","description":"Donuts domain registry content explorer","tags":["seo"]},{"pattern":"Dragonfly File Reader","description":"Dragonfly file reading bot","tags":["seo"]},{"pattern":"Drupal","url":"http://drupal.org/","description":"Drupal CMS HTTP client","tags":["http-library"]},{"pattern":"DwnldBot","url":"http://dwnld.me","description":"Dwnld download bot","tags":["seo"]},{"pattern":"ECCP\\/","description":"Eniro product listing crawler","tags":["seo"]},{"pattern":"EDS EXIF\\/","url":"https://www.exif-search.com","description":"EXIF search engine crawler","tags":["search-engine"]},{"pattern":"Easy-Thumb","url":"https://www.easy-thumb.net/","description":"Easy-Thumb thumbnail generator","tags":["seo"]},{"pattern":"EdisterBot","url":"http://www.edister.com/bot.html","description":"Edister search engine bot","tags":["search-engine"]},{"pattern":"Elmer[^/]*Thinglink ImageBot","url":"http://www.thinglink.com/help/ThinglinkImageBot","description":"Thinglink image crawler bot","tags":["seo"]},{"pattern":"Email%20Extractor","description":"Email extractor tool","tags":["scanner"]},{"pattern":"Embarcadero URI Client\\/","description":"Embarcadero Delphi URI client library","tags":["http-library"]},{"pattern":"Embed PHP Library","description":"PHP embedding library","tags":["http-library"]},{"pattern":"Eurobot\\/","url":"http://eurobot.ayell.eu","description":"Eurobot web crawler","tags":["seo"]},{"pattern":"EventMachine HttpClient","description":"Ruby EventMachine HTTP client library","tags":["http-library"]},{"pattern":"Everwall Link Expander","url":"http://everwall.com","description":"Everwall URL expansion service","tags":["seo"]},{"pattern":"ExB Language Crawler","url":"http://www.exb.de/crawler","description":"ExB language processing crawler","tags":["academic"]},{"pattern":"ExactSearch","description":"ExactSearch search engine crawler","tags":["search-engine"]},{"pattern":"ExactSeekCrawler\\/","description":"ExactSeek search engine crawler","tags":["search-engine"]},{"pattern":"ExoRank","description":"ExoRank SEO crawler","tags":["seo"]},{"pattern":"ExperianCrawlUK","description":"Experian UK web crawler","tags":["seo"]},{"pattern":"Express WebPictures","url":"http://www.express-soft.com","description":"Express WebPictures image downloader","tags":["seo"]},{"pattern":"ExtractorPro","description":"ExtractorPro data extraction tool","tags":["scanner"]},{"pattern":"FAST Enteprise Crawler\\/","url":"http://www.fastsearch.com","description":"FAST Enterprise Crawler version 6","tags":["search-engine"]},{"pattern":"FDM ","description":"Free Download Manager","tags":["http-library"]},{"pattern":"FFbot\\/","url":"http://feedfury.com","description":"FeedFury RSS feed bot","tags":["feed-reader"]},{"pattern":"Factbot ","url":"http://www.factbites.com/webmasters.php","description":"Factbites fact extraction bot","tags":["seo"]},{"pattern":"Faraday v","description":"Faraday Ruby HTTP client library","tags":["http-library"]},{"pattern":"FavOrg","description":"FavOrg favicon crawler","tags":["seo"]},{"pattern":"Favicon downloader[^)]*favico\\.be","url":"https://favico.be/bot.html","description":"Favico.be favicon downloading service","tags":["seo"]},{"pattern":"Favicon downloader[^)]*favicon\\.netk6\\.com","url":"http://favicon.netk6.com/","description":"Netk6 favicon downloading service","tags":["seo"]},{"pattern":"FeedBucket\\/","url":"http://www.feedbucket.com","description":"FeedBucket RSS feed reader","tags":["feed-reader"]},{"pattern":"FeedBunch\\/","url":"http://www.feedbunch.com","description":"FeedBunch RSS feed reader","tags":["feed-reader"]},{"pattern":"FeedCatBot\\/","url":"http://www.feedcat.net/","description":"FeedCat RSS feed crawler","tags":["feed-reader"]},{"pattern":"FeedChecker-Zocle\\/","url":"https://zocle.com/zoclechecker","description":"Zocle feed checker bot","tags":["feed-reader"]},{"pattern":"FeedZcollector","url":"http://www.feeds4all.com/feedzcollector","description":"Feeds4All feed collector","tags":["feed-reader"]},{"pattern":"Feedable\\/","description":"Feedable RSS feed reader","tags":["feed-reader"]},{"pattern":"Feedfetcher-Nuesbyte","url":"http://www.nuesbyte.com","description":"Nuesbyte feed fetcher","tags":["feed-reader"]},{"pattern":"Feedreader ","description":"Feedreader RSS client","tags":["feed-reader"]},{"pattern":"Feedshow\\/","url":"http://www.feedshow.com","description":"Feedshow RSS feed reader","tags":["feed-reader"]},{"pattern":"FeedshowOnline","url":"http://www.feedshow.com","description":"FeedshowOnline RSS feed reader","tags":["feed-reader"]},{"pattern":"Fiery%20Feeds\\/","description":"Fiery Feeds iOS RSS reader","tags":["feed-reader"]},{"pattern":"Filestack","description":"Filestack file processing service","tags":["seo"]},{"pattern":"FlightDeckReportsBot\\/","url":"http://www.flightdeckreports.com/pages/bot","description":"FlightDeckReports aviation bot","tags":["seo"]},{"pattern":"Florienzh\\/","description":"Florienzh crawler","tags":["seo"]},{"pattern":"FollowSite Bot","url":"http://www.followsite.com/bot.html","description":"FollowSite website monitoring bot","tags":["monitoring"]},{"pattern":"Frontman v","description":"Frontman HTTP client","tags":["http-library"]},{"pattern":"Fuzz Faster U Fool","description":"FFUF web fuzzing tool","tags":["scanner"]},{"pattern":"FyberSpider\\/","url":"http://www.fybersearch.com/fyberspider.php","description":"Fyber search engine spider","tags":["search-engine"]},{"pattern":"G-i-g-a-b-o-t","description":"Gigabot variant crawler","tags":["search-engine"]},{"pattern":"Google-Ads-Overview","description":"Google Ads overview crawler","tags":["advertising"]},{"pattern":"Google-Calendar-Importer","description":"Google Calendar import service","tags":["search-engine"]},{"pattern":"Google-HTTP-Java-Client\\/","url":"https://github.com/googleapis/google-http-java-client","description":"Google HTTP Java client library","tags":["http-library"]},{"pattern":"Google-SMTP-STS","description":"Google SMTP STS policy fetcher","tags":["monitoring"]},{"pattern":"Google-Transparency-Report","description":"Google Transparency Report crawler","tags":["monitoring"]},{"pattern":"Google-speakr","description":"Google Speakr audio service","tags":["search-engine"]},{"pattern":"GoogleCloudMonitoring","description":"Google Cloud monitoring service","tags":["monitoring"]},{"pattern":"GoogleHC\\/","description":"Google health check service","tags":["monitoring"]},{"pattern":"GoogleProber","description":"Google network prober","tags":["monitoring"]},{"pattern":"Google-CWS","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Chrome Web Store fetcher for extension metadata","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"GoogleMessages","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Google Messages link preview fetcher","tags":["social-preview"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Google-Pinpoint","url":"https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers","description":"Google Pinpoint personal document collection fetcher","tags":["search-engine"],"rdns":[".1e100.net",".googlebot.com",".googleusercontent.com"]},{"pattern":"Feedspot","url":"http://www.feedspot.com","description":"Feedspot RSS feed reader and blog aggregator","tags":["feed-reader"]},{"pattern":"GAChecker","url":"http://www.gachecker.com","description":"Google Analytics checker tool","tags":["monitoring"]},{"pattern":"GC3pro\\+dir","url":"http://chkme.com/","description":"GC3pro SEO directory tool","tags":["seo"]},{"pattern":"GIDBot\\/","url":"http://www.gidnetwork.com/tools/gzip-test.php","description":"GID Network gzip test bot","tags":["monitoring"]},{"pattern":"GIS-LABS:CertMon","description":"GIS-LABS certificate monitoring bot","tags":["monitoring"]},{"pattern":"GOFORDITBOT","url":"http://www.goforit.com/about/","description":"GoForIt web crawler","tags":["seo"]},{"pattern":"GRequests\\/","description":"GRequests Python HTTP library","tags":["http-library"]},{"pattern":"Genderanalyzer\\/","description":"Gender analyzer tool","tags":["seo"]},{"pattern":"GentleSource Short URL Checker","url":"http://www.gentlesource.com/","description":"GentleSource URL shortener checker","tags":["monitoring"]},{"pattern":"GetCode","description":"GetCode web scraper","tags":["seo"]},{"pattern":"GetFoundBot","url":"http://www.getfound.cz/getfoundbot/","description":"GetFound SEO crawler","tags":["seo"]},{"pattern":"GetProxi\\.es-bot\\/","url":"http://getproxi.es/spiderinfo/","description":"GetProxies proxy testing bot","tags":["monitoring"]},{"pattern":"GetURLInfo\\/","description":"GetURLInfo URL information fetcher","tags":["monitoring"]},{"pattern":"GetintentCrawler","url":"http://getintent.com","description":"Getintent web crawler","tags":["seo"]},{"pattern":"Geziyor","description":"Geziyor web scraping framework","tags":["browser-automation"]},{"pattern":"GitHub-Hookshot\\/","description":"GitHub webhook delivery service","tags":["monitoring"]},{"pattern":"GitLab\\/","description":"GitLab webhook and integration service","tags":["monitoring"]},{"pattern":"GloomarBot\\/","url":"https://www.gloomar.com/bot","description":"Gloomar search engine bot","tags":["search-engine"]},{"pattern":"Go 1\\.1 package http","description":"Go programming language HTTP client library","tags":["http-library"]},{"pattern":"Go http package","description":"Go programming language HTTP package","tags":["http-library"]},{"pattern":"GoScraper","description":"GoScraper web scraping tool","tags":["browser-automation"]},{"pattern":"GoSpotCheck","description":"GoSpotCheck mobile inspection app","tags":["monitoring"]},{"pattern":"GoSquared-Status-Checker\\/","description":"GoSquared website status monitoring bot","tags":["monitoring"]},{"pattern":"Gofeed\\/","description":"Gofeed RSS/Atom feed parser library","tags":["feed-reader"]},{"pattern":"Goldfire Server","description":"Goldfire Server web crawler","tags":["seo"]},{"pattern":"Google_Analytics_Snippet_Validator","description":"Google Analytics snippet validation tool","tags":["monitoring"]},{"pattern":"Googlebot \\(gocrawl","description":"Gocrawl library using Googlebot user agent","tags":["http-library"]},{"pattern":"Goose\\/","description":"Goose article extraction library","tags":["http-library"]},{"pattern":"GozaikBot","url":"http://www.gozaik.com/gozaikbot.html","description":"Gozaik content aggregation bot","tags":["seo"]},{"pattern":"Grabber","url":"https://sites.google.com/3ziele.de/grabber/","description":"Grabber web scraping tool","tags":["browser-automation"]},{"pattern":"Gradle\\/","description":"Gradle build automation tool HTTP client","tags":["http-library"]},{"pattern":"Grahambot\\/","url":"http://www.sunaga-lab.com/graham-bot","description":"Grahambot web crawler","tags":["seo"]},{"pattern":"GreatNews\\/","description":"GreatNews RSS feed reader","tags":["feed-reader"]},{"pattern":"Gregarius\\/","url":"http://devlog.gregarius.net/docs/ua","description":"Gregarius RSS feed aggregator","tags":["feed-reader"]},{"pattern":"GuzzleHttp\\/","description":"Guzzle PHP HTTP client library","tags":["http-library"]},{"pattern":"HAAARTLAND http client","description":"HAAARTLAND HTTP client library","tags":["http-library"]},{"pattern":"HEADMasterSEO","description":"HEADMasterSEO tool for SEO analysis","tags":["seo"]},{"pattern":"HTMLParser\\/","description":"HTMLParser library for parsing HTML","tags":["http-library"]},{"pattern":"HTTP-Header-Abfrage\\/","url":"http://www.internalscripts.de/werkzeuge/http-header-abfrage.php","description":"HTTP Header query tool","tags":["monitoring"]},{"pattern":"HTTP-Tiny\\/","description":"HTTP::Tiny Perl HTTP client library","tags":["http-library"]},{"pattern":"HTTPClient\\/","description":"Ruby HTTPClient library","tags":["http-library"]},{"pattern":"HTTP_Request2\\/","url":"http://pear.php.net/package/http_request2","description":"PEAR HTTP_Request2 PHP library","tags":["http-library"]},{"pattern":"HTTPing","description":"HTTPing network monitoring tool","tags":["monitoring"]},{"pattern":"Hadi Agent","description":"Hadi Agent web crawler","tags":["seo"]},{"pattern":"HappyApps-WebCheck\\/","description":"HappyApps website checking tool","tags":["monitoring"]},{"pattern":"HaxerMen","description":"HaxerMen web crawler","tags":["scanner"]},{"pattern":"Headless Chrome","description":"Headless Chrome browser automation","tags":["browser-automation"]},{"pattern":"HeartRails Robot\\/","url":"http://www.heartrails.com","description":"HeartRails web crawler bot","tags":["seo"]},{"pattern":"Heurekabot-Feed\\/","url":"https://sluzby.heureka.cz/napoveda/heurekabot/","description":"Heureka shopping comparison feed bot","tags":["feed-reader"]},{"pattern":"Heurekabot-ImageFullText\\/","url":"https://sluzby.heureka.cz/napoveda/heurekabot/","description":"Heureka image and full-text indexing bot","tags":["search-engine"]},{"pattern":"Hexometer","description":"Hexometer website monitoring and analytics bot","tags":["monitoring"]},{"pattern":"HolmesBot","url":"http://holmes.ge","description":"HolmesBot web crawler","tags":["seo"]},{"pattern":"HomePage Rss Reader","description":"HomePage RSS feed reader","tags":["feed-reader"]},{"pattern":"HonesoSearchEngine\\/","description":"Honeso search engine crawler","tags":["search-engine"]},{"pattern":"Hong Kong Education City\\/","url":"http://www.hkedcity.net/","description":"Hong Kong Education City web crawler","tags":["search-engine"]},{"pattern":"HootSuite Image proxy","description":"HootSuite image proxy service","tags":["social-preview"]},{"pattern":"HttpComponents\\/","description":"Apache HttpComponents HTTP client library","tags":["http-library"]},{"pattern":"HuaweiSymantecSpider\\/","url":"http://www.huaweisymantec.com/en/IRL/spider","description":"Huawei Symantec security spider","tags":["scanner"]},{"pattern":"Huaweisymantecspider","description":"Huawei Symantec security spider variant","tags":["scanner"]},{"pattern":"HubPages","url":"http://hubpages.com/help/crawlingpolicy","description":"HubPages content crawler","tags":["seo"]},{"pattern":"HyperZbozi\\.cz Feeder\\/","description":"Zbozi.cz shopping comparison feed crawler","tags":["feed-reader"]},{"pattern":"IDG Twitter Links Resolver","description":"IDG Twitter link resolver","tags":["social-preview"]},{"pattern":"IDG\\/IT","url":"http://spaziodati.eu/","description":"IDG IT web crawler","tags":["seo"]},{"pattern":"IPS\\/","description":"IPS web crawler","tags":["seo"]},{"pattern":"IZaBEE\\/","url":"https://izabee.com","description":"IZaBEE web crawler","tags":["search-engine"]},{"pattern":"IdeelaborPlagiaat\\/","description":"Ideelabor plagiarism detection crawler","tags":["seo"]},{"pattern":"IlTrovatore-Setaccio\\/","url":"http://www.iltrovatore.it/aiuto/faq.html","description":"IlTrovatore Italian search engine crawler","tags":["search-engine"]},{"pattern":"ImageEngine\\/","description":"ImageEngine image optimization service","tags":["monitoring"]},{"pattern":"ImageVisu\\/","description":"ImageVisu image crawler","tags":["seo"]},{"pattern":"Imagga Downloader\\/","description":"Imagga image recognition API downloader","tags":["seo"]},{"pattern":"InAGist URL Resolver","url":"http://inagist.com","description":"InAGist URL resolver service","tags":["social-preview"]},{"pattern":"InDesign%20CC\\/","description":"Adobe InDesign CC HTTP client","tags":["http-library"]},{"pattern":"Inboxb0t-Image\\/","description":"Inboxbot image crawler","tags":["seo"]},{"pattern":"GOFORITBOT","url":"http://www.goforit.com/about/","description":"GoForIt web crawler","tags":["seo"]},{"pattern":"HAARTLAND http client","description":"HAARTLAND HTTP client library","tags":["http-library"]},{"pattern":"InetURL:","description":"InetURL HTTP client","tags":["http-library"]},{"pattern":"Influencebot\\/","url":"http://www.influencebox.com/","description":"Influencebox website classification bot","tags":["seo"]},{"pattern":"InfoWizards Reciprocal Link System","url":"http://www.infowizards.com","description":"InfoWizards reciprocal link checker","tags":["seo"]},{"pattern":"Inspingbot\\/","url":"https://www.insping.com/","description":"Insping website monitoring bot","tags":["monitoring"]},{"pattern":"Integrity\\/","description":"Integrity website link checker","tags":["monitoring"]},{"pattern":"Interdose AntiSpamBot\\/","url":"http://www.piep.net","description":"Interdose anti-spam bot","tags":["scanner"]},{"pattern":"InternetSeer\\.com","description":"InternetSeer website monitoring service","tags":["monitoring"]},{"pattern":"Is is up\\?","url":"http://isitup.org","description":"IsItUp website availability checker","tags":["monitoring"]},{"pattern":"JAHHO check","description":"JAHHO website checker","tags":["monitoring"]},{"pattern":"JS-Kit URL Resolver","url":"http://js-kit.com/","description":"JS-Kit URL resolver service","tags":["social-preview"]},{"pattern":"JUST-CRAWLER","url":"http://www.justsystems.com/jp/tech/crawler/","description":"JustSystems web crawler","tags":["search-engine"]},{"pattern":"Jabse\\.com Crawler","url":"http://www.jabse.com/crawler.php","description":"Jabse search engine crawler","tags":["search-engine"]},{"pattern":"Jaunt\\/","description":"Jaunt web scraping library","tags":["http-library"]},{"pattern":"Java\\/[0-9]","description":"Java HTTP client library","tags":["http-library"]},{"pattern":"Jeffrey's Exif Viewer","url":"http://regex.info/exif","description":"Jeffrey's Exif metadata viewer","tags":["monitoring"]},{"pattern":"Jeode\\/","description":"Jeode Java runtime environment","tags":["http-library"]},{"pattern":"Jersey\\/","description":"Jersey RESTful web services framework","tags":["http-library"]},{"pattern":"JetBrains Omea Reader","url":"http://www.jetbrains.com/omea_reader/","description":"JetBrains Omea RSS feed reader","tags":["feed-reader"]},{"pattern":"JoBo\\/","url":"http://www.matuschek.net/jobo.html","description":"JoBo web crawler","tags":["seo"]},{"pattern":"JobFeed discovery script","url":"http://www.jobfeed.com","description":"JobFeed job listing discovery crawler","tags":["seo"]},{"pattern":"Jobg8 URL Monitor","description":"Jobg8 URL monitoring service","tags":["monitoring"]},{"pattern":"Jobsearch","description":"Jobsearch crawler","tags":["seo"]},{"pattern":"JoinVision Generic Site Loader","description":"JoinVision site loading bot","tags":["seo"]},{"pattern":"JolokiaPwn","url":"https://matmannion.com/jolokiapwn/","description":"JolokiaPwn security testing tool","tags":["scanner"]},{"pattern":"Jyxobot\\/","description":"Jyxo search engine bot","tags":["search-engine"]},{"pattern":"JyxobotRSS\\/","description":"Jyxo RSS feed crawler","tags":["feed-reader"]},{"pattern":"KD Bot","description":"KD Bot web crawler","tags":["seo"]},{"pattern":"KOCMOHAVT","url":"https://kozmonavt.tk/","description":"Kozmonavt web crawler","tags":["seo"]},{"pattern":"Karneval-Bot","url":"http://www.karnevalsuchmaschine.de/bot.html","description":"Karnevalsuchmaschine carnival search engine bot","tags":["search-engine"]},{"pattern":"Kaspersky Lab CFR link resolver","description":"Kaspersky Lab link resolver","tags":["scanner"]},{"pattern":"Kerrigan\\/","description":"Mashable Kerrigan crawler","tags":["seo"]},{"pattern":"KeyCDN Perf Test","description":"KeyCDN performance testing tool","tags":["monitoring"]},{"pattern":"KeywordDensityRobot\\/","url":"http://www.seocentro.com/tools/search-engines/keyword-density.html","description":"SEOCentro keyword density analyzer","tags":["seo"]},{"pattern":"KiNShooboT","url":"http://www.kinshoo.com/bot.html","description":"KiNShoo search engine bot","tags":["search-engine"]},{"pattern":"KickFire","description":"KickFire IP intelligence crawler","tags":["seo"]},{"pattern":"Kimengi\\/","description":"Kimengi web crawler","tags":["seo"]},{"pattern":"Kml-Google","url":"http://code.google.com/apis/kml","description":"Google KML file fetcher","tags":["monitoring"]},{"pattern":"KrOWLer\\/","description":"KrOWLer semantic web crawler","tags":["academic"]},{"pattern":"Kyoto-(?:Tohoku-)?Crawler\\/","url":"http://nlp.ist.i.kyoto-u.ac.jp/","description":"Kyoto University web crawler","tags":["academic"]},{"pattern":"L\\.webis\\/","url":"http://webalgo.iit.cnr.it/index.php?pg=lwebis","description":"L.webis web crawler","tags":["academic"]},{"pattern":"LSSRocketCrawler\\/","description":"Lightspeed Systems web filter crawler","tags":["scanner"]},{"pattern":"LWP::Simple\\/","description":"Perl LWP::Simple HTTP library","tags":["http-library"]},{"pattern":"LYT\\.SR","url":"http://lyt.sr/","description":"LYT.SR web crawler","tags":["seo"]},{"pattern":"LapozzBot\\/","url":"http://robot.lapozz.com","description":"Lapozz search engine bot","tags":["search-engine"]},{"pattern":"Lavf\\/","description":"FFmpeg Lavf library","tags":["http-library"]},{"pattern":"LexxeBot\\/","description":"Lexxe search engine bot","tags":["search-engine"]},{"pattern":"Liferea\\/","url":"http://liferea.sf.net/","description":"Liferea RSS feed reader","tags":["feed-reader"]},{"pattern":"Lijit Crawler","url":"http://www.lijit.com/robot/crawler","description":"Lijit web crawler","tags":["seo"]},{"pattern":"LinguaBot\\/","description":"LinguaSeek multilingual search engine bot","tags":["search-engine"]},{"pattern":"Link Valet Online","description":"Link Valet link checker","tags":["monitoring"]},{"pattern":"Link Validity Check","url":"http://www.w3dir.com/","description":"W3Dir link validity checker","tags":["monitoring"]},{"pattern":"LinkAider","url":"http://linkaider.com/crawler/","description":"LinkAider link analysis crawler","tags":["seo"]},{"pattern":"LinkAlarm\\/","description":"LinkAlarm link monitoring tool","tags":["monitoring"]},{"pattern":"LinkAnalyser\\/","description":"LinkAnalyser link analysis tool","tags":["seo"]},{"pattern":"LinkExaminer\\/","description":"LinkExaminer link checking tool","tags":["monitoring"]},{"pattern":"LinkPreview\\/","url":"http://linkpreview.net","description":"LinkPreview link preview generator","tags":["social-preview"]},{"pattern":"LinkStats Bot","description":"LinkStats link statistics bot","tags":["seo"]},{"pattern":"LinqiaMetadataDownloaderBot\\/","description":"Linqia metadata downloader bot","tags":["seo"]},{"pattern":"LinqiaRSSBot\\/","description":"Linqia RSS feed bot","tags":["feed-reader"]},{"pattern":"LinqiaScrapeBot\\/","description":"Linqia scraping bot","tags":["seo"]},{"pattern":"LoadImpactPageAnalyzer\\/","url":"http://loadimpact.com/","description":"Load Impact page analyzer","tags":["monitoring"]},{"pattern":"LoadImpactRload\\/","url":"http://loadimpact.com/","description":"Load Impact load testing tool","tags":["monitoring"]},{"pattern":"LongURL API","description":"LongURL URL expansion API","tags":["social-preview"]},{"pattern":"Lucee","description":"Lucee CFML engine HTTP client","tags":["http-library"]},{"pattern":"LumpImageSearch\\/","url":"http://lump.co/about/bot","description":"Lump image search bot","tags":["seo"]},{"pattern":"MB-SiteCrawler","description":"MB SiteCrawler tool","tags":["seo"]},{"pattern":"MB-SitemapGenerator","description":"MB Sitemap Generator tool","tags":["seo"]},{"pattern":"MIA DEV\\/search:robot\\/","url":"http://spider.neofonie.de","description":"MIA research project bot","tags":["academic"]},{"pattern":"MSFrontPage\\/","description":"Microsoft FrontPage HTTP client","tags":["http-library"]},{"pattern":"MSRBOT","description":"MSR Bot web crawler","tags":["seo"]},{"pattern":"MVAClient","description":"MVA Client HTTP client","tags":["http-library"]},{"pattern":"MacOutlook\\/","description":"Microsoft Outlook for Mac HTTP client","tags":["http-library"]},{"pattern":"Mail\\.R[Uu]\\/","description":"Mail.RU email client","tags":["http-library"]},{"pattern":"Mail\\/","description":"Apple Mail HTTP client","tags":["http-library"]},{"pattern":"Mail::STS","description":"Mail::STS Perl module","tags":["http-library"]},{"pattern":"KOCMOHABT","url":"https://kozmonavt.tk/","description":"Kozmonabt web crawler","tags":["http-library"]},{"pattern":"Mailpit\\/","description":"Mailpit email testing tool","tags":["monitoring"]},{"pattern":"Mandrill-Webhook\\/","description":"Mandrill email service webhook","tags":["monitoring"]},{"pattern":"MarketBrewBot\\/","url":"http://www.marketbrew.com/","description":"MarketBrew SEO analysis bot","tags":["seo"]},{"pattern":"MaxPoint Bot","url":"http://www.maxpoint.com","description":"MaxPoint advertising bot","tags":["advertising"]},{"pattern":"MeMoNewsBot\\/","url":"http://www.memonews.com/en/crawler","description":"MeMoNews crawler","tags":["feed-reader"]},{"pattern":"Mediametric","description":"Mediametric monitoring bot","tags":["monitoring"]},{"pattern":"Mediumbot-ProxyFetcher\\/","url":"https://medium.com/","description":"Medium proxy fetcher bot","tags":["social-preview"]},{"pattern":"Melvil Rawi\\/","description":"Melvil Rawi crawler","tags":["http-library"]},{"pattern":"MemGator:","description":"MemGator web archive aggregator","tags":["archiver"]},{"pattern":"Mergadobot\\/","url":"http://mergado.cz","description":"Mergado e-commerce crawler","tags":["seo"]},{"pattern":"MergeFlow-PageReader\\/","url":"http://mergeflow.net/info/pagereader","description":"MergeFlow page reader bot","tags":["http-library"]},{"pattern":"MetaCommentBot","url":"http://metacomment.io/about","description":"MetaComment discussion platform bot","tags":["social-preview"]},{"pattern":"MetaGeneratorCrawler\\/","url":"http://www.metagenerator.info","description":"MetaGenerator SEO crawler","tags":["seo"]},{"pattern":"MetaHeadersBot","url":"http://www.metaheaders.com/bot.html","description":"MetaHeaders SEO analysis bot","tags":["seo"]},{"pattern":"MetaTagRobot\\/","url":"http://www.seocentro.com/tools/search-engines/metatag-analyzer.html","description":"MetaTag analyzer bot","tags":["seo"]},{"pattern":"Metaspinner\\/","url":"http://www.meta-spinner.de/","description":"Metaspinner search engine crawler","tags":["search-engine"]},{"pattern":"Mget\\/","description":"Mget HTTP client","tags":["http-library"]},{"pattern":"MiaDev\\/","url":"http://www.mia-marktplatz.de/spider","description":"MIA marketplace research bot","tags":["seo"]},{"pattern":"Microsearch\\.ru","url":"http://microsearch.ru/webmasters","description":"Microsearch.ru search engine bot","tags":["search-engine"]},{"pattern":"Microsoft Data Access Internet Publishing Provider","description":"Microsoft WebDAV client","tags":["http-library"]},{"pattern":"Microsoft Office","description":"Microsoft Office applications","tags":["http-library"]},{"pattern":"Microsoft URL Control","description":"Microsoft URL Control library","tags":["http-library"]},{"pattern":"Microsoft Windows Network Diagnostics","description":"Windows network diagnostic tool","tags":["monitoring"]},{"pattern":"Microsoft-WebDAV-MiniRedir\\/","description":"Microsoft WebDAV client","tags":["http-library"]},{"pattern":"Microsoft\\.Data\\.Mashup","url":"https://go.microsoft.com/fwlink/?LinkID=304225","description":"Microsoft Power Query data connector","tags":["http-library"]},{"pattern":"Mikrotik\\/","description":"Mikrotik RouterOS fetch tool","tags":["http-library"]},{"pattern":"Mindjet","description":"Mindjet MindManager application","tags":["http-library"]},{"pattern":"Miniflux","url":"https://miniflux.net","description":"Miniflux RSS feed reader","tags":["feed-reader"]},{"pattern":"Miro-HttpClient\\/","description":"Miro HTTP client","tags":["http-library"]},{"pattern":"MixBot","url":"http://t.co/GSRLLKex24","description":"Mix social media bot","tags":["social-preview"]},{"pattern":"Mixmax-LinkPreview","url":"https://sdk.mixmax.com","description":"Mixmax link preview bot","tags":["social-preview"]},{"pattern":"Mixnode\\/","url":"https://www.mixnode.com/","description":"Mixnode speed analysis bot","tags":["monitoring"]},{"pattern":"Mnogosearch","description":"Mnogosearch search engine","tags":["search-engine"]},{"pattern":"Mojolicious","description":"Mojolicious Perl web framework","tags":["http-library"]},{"pattern":"Monit\\/","description":"Monit monitoring tool","tags":["monitoring"]},{"pattern":"More-Testing","url":"http://www.theinternetrules.com","description":"More-Testing crawler","tags":["seo"]},{"pattern":"Morning Paper","description":"Morning Paper crawler","tags":["search-engine"]},{"pattern":"Motoricerca-Robots\\.txt-Checker\\/","url":"http://tool.motoricerca.info/robots-checker.phtml","description":"Motoricerca robots.txt checker","tags":["seo"]},{"pattern":"MovableType\\/","description":"MovableType blogging platform","tags":["http-library"]},{"pattern":"StudioFACA Search","url":"http://www.studiofaca.com/","description":"StudioFACA search engine","tags":["search-engine"]},{"pattern":"web\\.nli\\.org\\.il","url":"http://web.nli.org.il/sites/NLI/English/collections/israel-collection/internet_archive/Pages/default.aspx","description":"National Library of Israel web archive","tags":["archiver"]},{"pattern":"Fluffy the spider","url":"http://www.searchhippo.com/","description":"SearchHippo Fluffy spider","tags":["search-engine"]},{"pattern":"WebMon","description":"WebMon monitoring tool","tags":["monitoring"]},{"pattern":"Toread-Crawler\\/","url":"http://news.toread.cc/crawler.php","description":"Toread news crawler","tags":["feed-reader"]},{"pattern":"Advanced Email Extractor","description":"Advanced Email Extractor tool","tags":["scanner"]},{"pattern":"Arachmo","description":"Arachmo web crawler","tags":["search-engine"]},{"pattern":"B-l-i-t-z-B-O-T","description":"Blitz BOT crawler","tags":["search-engine"]},{"pattern":"BOTW Spider","url":"http://botw.org","description":"Best of the Web spider","tags":["search-engine"]},{"pattern":"Cerberian Drtrs","description":"Cerberian Drtrs crawler","tags":["search-engine"]},{"pattern":"DAWINCI ANTIPLAG SPIDER","description":"Dawinci antiplagiarism spider","tags":["seo"]},{"pattern":"Fooooo_Web_Video_Crawl","url":"http://fooooo.com/bot.html","description":"Fooooo video crawler","tags":["search-engine"]},{"pattern":"HostTracker\\.com","url":"http://host-tracker.com/","description":"HostTracker monitoring service","tags":["monitoring"]},{"pattern":"IDwhois","url":"http://idwhois.info/about","description":"IDwhois domain information tool","tags":["seo"]},{"pattern":"LinkMarket-Bot","description":"LinkMarket bot","tags":["seo"]},{"pattern":"DomainDB","url":"http://domaindb.com/crawler/","description":"DomainDB domain crawler","tags":["seo"]},{"pattern":"FeedFinder","url":"http://bloggz.se/crawler","description":"FeedFinder RSS feed discovery tool","tags":["feed-reader"]},{"pattern":"DTS Agent","description":"DTS Agent crawler","tags":["http-library"]},{"pattern":"Girafabot","url":"http://www.girafa.com","description":"Girafa search engine bot","tags":["search-engine"]},{"pattern":"obot","description":"Obot web crawler","tags":["search-engine"]},{"pattern":"Fetch API Request","description":"Fetch API Request client","tags":["http-library"]},{"pattern":"mowser","url":"http://www.mowser.com","description":"Mowser mobile web browser","tags":["http-library"]},{"pattern":"MS Web Services Client Protocol","description":"Microsoft Web Services Client","tags":["http-library"]},{"pattern":"basicstate\\.com","url":"http://basicstate.com/","description":"BasicState crawler","tags":["seo"]},{"pattern":"a3logics\\.in","description":"A3Logics crawler","tags":["seo"]},{"pattern":"FairShare","url":"http://fairshare.cc","description":"FairShare crawler","tags":["seo"]},{"pattern":"webauth@cmcm\\.com","description":"CMCM web authentication crawler","tags":["seo"]},{"pattern":"PeoplePal","description":"PeoplePal toolbar","tags":["http-library"]},{"pattern":"QQDownload","description":"QQ Download Manager","tags":["http-library"]},{"pattern":"CrawlerProcess","url":"http://www.PowerMapper.com","description":"PowerMapper CrawlerProcess","tags":["seo"]},{"pattern":"BrokenLinkCheck\\.com","url":"http://BrokenLinkCheck.com","description":"BrokenLinkCheck link checker","tags":["seo"]},{"pattern":"MDDS","description":"MDDS crawler","tags":["http-library"]},{"pattern":"PTST","description":"PTST monitoring tool","tags":["monitoring"]},{"pattern":"alertra\\.com","url":"http://www.alertra.com","description":"Alertra monitoring service","tags":["monitoring"]},{"pattern":"BacklinkHttpStatus","description":"Backlink HTTP status checker","tags":["seo"]},{"pattern":"WinHttp\\.WinHttpRequest","description":"Windows HTTP Request library","tags":["http-library"]},{"pattern":"vBSEO","url":"http://www.vbseo.com","description":"vBSEO forum SEO tool","tags":["seo"]},{"pattern":"xcm@huaweisymantec\\.com","description":"Huawei Symantec crawler","tags":["http-library"]},{"pattern":"Webster Pro","description":"Webster Pro web crawler","tags":["http-library"]},{"pattern":"CloudServerMarketSpider","url":"http://cloudservermarket.com/spider.html","description":"CloudServerMarket spider","tags":["seo"]},{"pattern":"SETOOZBOT","url":"http://www.setooz.com/bot.html","description":"Setooz search engine bot","tags":["search-engine"]},{"pattern":"Mammoth","url":"http://www.sli-systems.com/","description":"SLI Systems Mammoth crawler","tags":["search-engine"]},{"pattern":"Anturis Agent","description":"Anturis monitoring agent","tags":["monitoring"]},{"pattern":"Vedma","url":"http://www.vedma.ru/bot.htm","description":"Vedma search engine bot","tags":["search-engine"]},{"pattern":"FauBot","url":"http://buzzvolume.com/fau/","description":"BuzzVolume FauBot crawler","tags":["seo"]},{"pattern":"GetLinkInfo\\.com","url":"http://www.getlinkinfo.com","description":"GetLinkInfo link information service","tags":["seo"]},{"pattern":"Grabify\\.link","url":"https://grabify.link/expander","description":"Grabify link expander service","tags":["seo"]},{"pattern":"KeepRight","url":"http://keepright.at","description":"KeepRight OpenStreetMap checker","tags":["monitoring"]},{"pattern":"Nodemeter","description":"Nodemeter monitoring tool","tags":["monitoring"]},{"pattern":"Statically-Screenshot","url":"https://statically.io/screenshot","description":"Statically screenshot service","tags":["seo"]},{"pattern":"DebugBear","description":"DebugBear performance monitoring","tags":["monitoring"]},{"pattern":"Google-Publisher-Plugin","description":"Google Publisher Plugin","tags":["advertising"]},{"pattern":"Neustar WPM","description":"Neustar Web Performance Management","tags":["monitoring"]},{"pattern":"Google Page Speed Insights","description":"Google PageSpeed Insights bot","tags":["monitoring"]},{"pattern":"Seznam screenshot-generator","description":"Seznam screenshot generator","tags":["seo"]},{"pattern":"glindahl-cocrawler","url":"http://www.pbm.com/~lindahl/glindahl-cocrawler.html","description":"Glindahl CoCrawler","tags":["search-engine"]},{"pattern":"Observatory","description":"Mozilla Observatory security scanner","tags":["scanner"]},{"pattern":"Project-Resonance","url":"http://project-resonance.com/","description":"Project Resonance crawler","tags":["seo"]},{"pattern":"NameFo","url":"https://namefo.com/","description":"NameFo domain information service","tags":["seo"]},{"pattern":"backend@getprismatic\\.com","description":"Prismatic content discovery","tags":["seo"]},{"pattern":"Shoppimon","url":"http://www.shoppimon.com/","description":"Shoppimon e-commerce analyzer","tags":["seo"]},{"pattern":"KimonoLabs","description":"Kimono Labs web scraper","tags":["http-library"]},{"pattern":"Rival IQ","url":"http://rivaliq.com","description":"Rival IQ competitive intelligence","tags":["seo"]},{"pattern":"Princetonbot","url":"http://tigress-web.princeton.edu/~fy/bot.html","description":"Princeton University bot","tags":["academic"]},{"pattern":"ReederForMac","description":"Reeder RSS reader for Mac","tags":["feed-reader"]},{"pattern":"FlipboardBrowserProxy","url":"http://flipboard.com/browserproxy","description":"Flipboard browser proxy","tags":["social-preview"]},{"pattern":"Nekstbot","url":"http://nekst.ipipan.waw.pl/nekstbot/","description":"Nekst research bot","tags":["academic"]},{"pattern":"DMBrowser-UV","description":"DMBrowser user verification","tags":["http-library"]},{"pattern":"TryJsoup","url":"http://try.jsoup.org/","description":"Try Jsoup HTML parser tester","tags":["http-library"]},{"pattern":"The National Archives UK","url":"http://www.nationalarchives.gov.uk/webarchive/","description":"UK National Archives web archive","tags":["archiver"]},{"pattern":"ForusPM","description":"ForusPM monitoring tool","tags":["monitoring"]},{"pattern":"google_partner_monitoring","description":"Google partner monitoring","tags":["monitoring"]},{"pattern":"YottaaMonitor","description":"Yottaa performance monitoring","tags":["monitoring"]},{"pattern":"spyonweb\\.com","url":"http://spyonweb.com","description":"SpyOnWeb domain research tool","tags":["seo"]},{"pattern":"shrinktheweb\\.com","url":"https://shrinktheweb.com","description":"ShrinkTheWeb screenshot service","tags":["seo"]},{"pattern":"QuiteRSS","description":"QuiteRSS feed reader","tags":["feed-reader"]},{"pattern":"StumbleUpon","description":"StumbleUpon content discovery","tags":["social-preview"]},{"pattern":"sniptracker\\.com","url":"http://sniptracker.com","description":"SnipTracker tracking service","tags":["seo"]},{"pattern":"XaxisSemanticsClassifier","url":"http://crystalsemantics.com","description":"Crystal Semantics classifier","tags":["seo"]},{"pattern":"Yandex\\.Translate","description":"Yandex Translate service","tags":["http-library"]},{"pattern":"topster\\.de Linkchecker","url":"http://topster.de","description":"Topster.de link checker","tags":["seo"]},{"pattern":"Evidon","description":"Evidon privacy compliance","tags":["scanner"]},{"pattern":"Pix[^C]*Criteo","description":"Criteo Pix advertising bot","tags":["advertising"]},{"pattern":"app\\.limber\\.io","url":"https://app.limber.io","description":"Limber app crawler","tags":["seo"]},{"pattern":"caam dot crwlr","description":"CAAM crawler","tags":["search-engine"]},{"pattern":"Diffbot","url":"http://www.diffbot.com","description":"Diffbot web data extraction","tags":["seo"]},{"pattern":"Runet-Research-Crawler","url":"http://itrack.ru/research/cmsrate","description":"Runet Research CMS rating crawler","tags":["seo"]},{"pattern":"Websnapr","description":"Websnapr screenshot service","tags":["seo"]},{"pattern":"Webthumb","description":"Webthumb screenshot service","tags":["seo"]},{"pattern":"BingLocalSearch","description":"Bing Local Search bot","tags":["search-engine"]},{"pattern":"Google-SearchByImage","description":"Google Search by Image","tags":["search-engine"]},{"pattern":"AraBot","url":"http://ara.com.tr","description":"Ara.com.tr search bot","tags":["search-engine"]},{"pattern":"BrowserSpyBot","url":"http://browserspy.dk/","description":"BrowserSpy bot","tags":["scanner"]},{"pattern":"SPEng","description":"SPEng search engine","tags":["search-engine"]},{"pattern":"VoilaBotCollector","url":"http://www.voila.com/","description":"Voila bot collector","tags":["search-engine"]},{"pattern":"ow\\.ly","url":"http://ow.ly","description":"Ow.ly URL shortener crawler","tags":["social-preview"]},{"pattern":"MsnBot-Media","url":"https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0","description":"Microsoft Bing media search bot","tags":["search-engine"]},{"pattern":"webkit2png","description":"Webkit-based screenshot tool","tags":["browser-automation"]},{"pattern":"SSL-Crawler","url":"http://crawler.dcsec.uni-hannover.de","description":"SSL security research crawler from Leibniz University Hannover","tags":["academic","scanner"]},{"pattern":"UXCrawlerBot","url":"http://zium.co","description":"UX crawler bot from Zium","tags":["seo"]},{"pattern":"Site-Shot","url":"http://www.site-shot.com/","description":"Website screenshot service","tags":["browser-automation"]},{"pattern":"linkpeek","url":"https://linkpeek.com","description":"Link preview and screenshot service","tags":["social-preview"]},{"pattern":"wkhtmltoimage","url":"https://wkhtmltopdf.org/","description":"HTML to image conversion tool using WebKit","tags":["browser-automation"]},{"pattern":"wkhtmltopdf","url":"https://wkhtmltopdf.org/","description":"HTML to PDF conversion tool using WebKit","tags":["browser-automation"]},{"pattern":"immediatenet\\.com/thumbnail_api","url":"http://immediatenet.com/thumbnail_api.html","description":"ImmediateNet free thumbnail generation service","tags":["social-preview"]},{"pattern":"imgsizer","description":"Image sizing and optimization tool","tags":["browser-automation"]},{"pattern":"web-capture\\.net","url":"http://web-capture.net","description":"Web page capture and screenshot service","tags":["browser-automation"]},{"pattern":"webscreenie","description":"Website screenshot tool","tags":["browser-automation"]},{"pattern":"wpif","description":"WordPress information fetcher","tags":["scanner"]},{"pattern":"HomepageScreenshot\\.de","url":"http://www.HomepageScreenshot.de","description":"Free screenshot service from HomepageScreenshot.de","tags":["browser-automation"]},{"pattern":"Google-HotelAdsVerifier","url":"https://support.google.com/hotelprices/","description":"Google Hotel Ads verification bot","tags":["advertising"]},{"pattern":"notifyninja\\.com","url":"http://notifyninja.com/monitoring","description":"NotifyNinja website monitoring service","tags":["monitoring"]},{"pattern":"aylienbot","url":"http://www.aylien.com/bot.html","description":"AYLIEN text analysis and NLP crawler","tags":["ai-crawler"]},{"pattern":"Lush Http Client","description":"Lush HTTP client library","tags":["http-library"]},{"pattern":"Slack_SSB","url":"https://api.slack.com/robots","description":"Slack link preview bot","tags":["social-preview"]},{"pattern":"GeedoShopProductFinder","description":"Geedo shopping product finder bot","tags":["seo"]},{"pattern":"Google Search Console","url":"https://search.google.com/search-console","description":"Google Search Console verification and testing tool","tags":["seo"]},{"pattern":"Jobboerse\\.com","url":"http://www.xn--jobbrse-d1a.com","description":"German job board crawler","tags":["seo"]},{"pattern":"HeartRails_Capture","url":"http://capture.heartrails.com/","description":"HeartRails website capture and thumbnail service","tags":["browser-automation"]},{"pattern":"WebThumb","description":"Website thumbnail generation service","tags":["browser-automation"]},{"pattern":"wprecon\\.com","url":"http://wprecon.com","description":"WordPress reconnaissance and security scanner","tags":["scanner"]},{"pattern":"Dot TK - spider","description":"Dot TK domain spider","tags":["seo"]},{"pattern":"Yahoo-MMCrawler","url":"https://help.yahoo.com/kb/SLN22600.html","description":"Yahoo multimedia crawler","tags":["search-engine"]},{"pattern":"YahooYSMcm","url":"http://help.yahoo.com","description":"Yahoo search marketing crawler","tags":["search-engine"]},{"pattern":"Zoom\\.Mac","description":"Zoom Mac application web preview","tags":["social-preview"]},{"pattern":"Google Keyword Suggestion","description":"Google keyword suggestion tool","tags":["seo"]},{"pattern":"servernfo\\.com","url":"http://servernfo.com/","description":"ServerNFO server information crawler","tags":["seo"]},{"pattern":"hosterstats\\.com","url":"http://www.hosterstats.com/newgtld-websurvey.php","description":"HosterStats new gTLD web survey crawler","tags":["seo"]},{"pattern":"2GDPR","url":"https://2gdpr.com","description":"2GDPR GDPR compliance crawler","tags":["scanner"]},{"pattern":"4SeoHuntBot","url":"http://4seohunt.biz/about.html","description":"4SeoHunt SEO analysis bot","tags":["seo"]},{"pattern":"80bot","url":"http://www.80legs.com/spider.html","description":"80legs web crawler service","tags":["seo"]},{"pattern":"ACHE","url":"https://github.com/ViDA-NYU/ache","description":"ACHE focused web crawler from NYU","tags":["academic"]},{"pattern":"AMZNKAssocBot","url":"http://affiliate-program.amazon.com","description":"Amazon Associates crawler bot","tags":["advertising"]},{"pattern":"Abonti","url":"http://www.abonti.com","description":"Abonti web crawler","tags":["seo"]},{"pattern":"AffiliateLabz","url":"https://affiliatelabz.com/","description":"AffiliateLabz affiliate marketing crawler","tags":["advertising"]},{"pattern":"AportWorm","url":"http://www.aport.ru/help","description":"Aport search engine crawler","tags":["search-engine"]},{"pattern":"AppBeat","url":"http://www.appbeat.io/","description":"AppBeat website monitoring service","tags":["monitoring"]},{"pattern":"Aprc","url":"https://aprc.it/","description":"Aprc web crawler","tags":["seo"]},{"pattern":"Arachnophilia","url":"http://arachnys.com/","description":"Arachnys web crawler for financial data","tags":["seo"]},{"pattern":"BIXOCRAWLER","url":"http://wiki.github.com/bixo/bixo/bixocrawler","description":"Bixo open-source web crawler","tags":["seo"]},{"pattern":"BegunAdvertising","url":"http://begun.ru/begun/technology/indexer/","description":"Begun advertising network crawler","tags":["advertising"]},{"pattern":"BigBozz","url":"http://www.bigbozz.com/","description":"BigBozz web crawler","tags":["seo"]},{"pattern":"BlinkaCrawler","url":"http://www.blinka.jp/crawler/","description":"Blinka web crawler from Japan","tags":["seo"]},{"pattern":"BusinessSeek\\.biz","url":"http://www.businessseek.biz/","description":"BusinessSeek business directory crawler","tags":["seo"]},{"pattern":"BuzzSumo","url":"http://www.buzzsumo.com/bot.html","description":"BuzzSumo content research and monitoring bot","tags":["seo"]},{"pattern":"CareerBot","url":"http://www.career-x.de/bot.html","description":"Career-X job search crawler","tags":["seo"]},{"pattern":"ChangesMeter","url":"http://intuiware.com/apps/changes-meter","description":"ChangesMeter website change monitoring service","tags":["monitoring"]},{"pattern":"Cloudinary","description":"Cloudinary media management service","tags":["social-preview"]},{"pattern":"CompSpyBot","url":"http://www.compspy.com/spider.html","description":"CompSpy competitor analysis crawler","tags":["seo"]},{"pattern":"CrawlForMe","description":"CrawlForMe web crawling service","tags":["seo"]},{"pattern":"linkfluence\\.net","url":"http://linkfluence.net/","description":"Linkfluence social media monitoring crawler","tags":["seo"]},{"pattern":"Crawlera","description":"Crawlera proxy service for web scraping","tags":["http-library"]},{"pattern":"CukBot","url":"http://www.companiesintheuk.co.uk/bot.html","description":"Companies in the UK directory crawler","tags":["seo"]},{"pattern":"DBLBot","url":"http://www.dontbuylists.com/","description":"DontBuyLists email verification crawler","tags":["scanner"]},{"pattern":"DCPbot","url":"http://domains.checkparams.com/","description":"CheckParams domain parameter checker","tags":["seo"]},{"pattern":"DIY-SEOBot","url":"http://www.upcity.com/bot.html","description":"UpCity DIY SEO analysis bot","tags":["seo"]},{"pattern":"DKIMRepBot","url":"http://www.dkim-reputation.org","description":"DKIM reputation checker bot","tags":["scanner"]},{"pattern":"htmlyse\\.com","url":"https://www.htmlyse.com/","description":"Htmlyse website security scanner","tags":["scanner"]},{"pattern":"DNS-Digger","url":"http://www.dnsdigger.com","description":"DNS Digger domain exploration tool","tags":["scanner"]},{"pattern":"DataXu","url":"http://dataxu.com","description":"DataXu advertising technology crawler","tags":["advertising"]},{"pattern":"Dataprovider","url":"https://www.dataprovider.com/","description":"Dataprovider web data collection service","tags":["seo"]},{"pattern":"DomainAppender","url":"http://www.profound.net/domainappender","description":"Profound Networks domain appender tool","tags":["seo"]},{"pattern":"DomainMacroCrawler","url":"http://domainmacro.com","description":"DomainMacro domain analysis crawler","tags":["seo"]},{"pattern":"DomainSONOCrawler","url":"http://domainsono.com","description":"DomainSono domain analysis crawler","tags":["seo"]},{"pattern":"DotBot","url":"http://www.dotnetdotcom.org/","description":"DotNetDotCom web crawler","tags":["seo"]},{"pattern":"DripfeedBot","url":"http://dripfeedbookmark.com/bot.html","description":"Dripfeed bookmark service crawler","tags":["seo"]},{"pattern":"EARTHCOM","url":"http://enter4u.eu","description":"Earthcom web crawler","tags":["seo"]},{"pattern":"Esribot","url":"http://www.esrihu.hu/","description":"Esri Hungary web crawler","tags":["seo"]},{"pattern":"EventGuruBot","url":"http://www.eventguru.com/spider.html","description":"EventGuru event discovery crawler","tags":["seo"]},{"pattern":"Evrinid","url":"http://www.evri.com/evrinid","description":"Evri semantic web crawler","tags":["seo"]},{"pattern":"ExaleadCloudView","description":"Exalead cloud search crawler","tags":["search-engine"]},{"pattern":"ExchangleBot","url":"https://www.exchangle.com/exchangling","description":"Exchangle web exchange crawler","tags":["seo"]},{"pattern":"ExpertSearchSpider","url":"http://www.expertsearch.nl/spider","description":"ExpertSearch Netherlands crawler","tags":["seo"]},{"pattern":"Exploratodo","url":"http://www.exploratodo.com","description":"Exploratodo web exploration crawler","tags":["seo"]},{"pattern":"Ezooms","description":"Ezooms SEO crawler (Moz related)","tags":["seo"]},{"pattern":"Falconsbot","url":"http://ws.nju.edu.cn/falcons/","description":"Falcons semantic web search crawler","tags":["academic"]},{"pattern":"FatBot","url":"http://www.thefind.com/crawler","description":"TheFind shopping search crawler","tags":["seo"]},{"pattern":"Faveeo","url":"http://www.faveeo.com","description":"Faveeo web crawler","tags":["seo"]},{"pattern":"FeedBooster","url":"http://feeds.qsensei.com","description":"QSensei feed booster crawler","tags":["feed-reader"]},{"pattern":"Feedage","url":"http://www.feedage.com/bot.php","description":"Feedage RSS feed directory crawler","tags":["feed-reader"]},{"pattern":"Finderbots","url":"http://wiki.github.com/bixo/bixo/bixocrawler","description":"Bixo Finderbots crawler","tags":["seo"]},{"pattern":"FoundSeoTool","url":"http://www.found.co.uk/seo-tool/","description":"Found SEO analysis tool","tags":["seo"]},{"pattern":"GeliyooBot","url":"http://www.geliyoo.com/","description":"Geliyoo search engine crawler","tags":["search-engine"]},{"pattern":"GigaBot","url":"http://www.gigablast.com/","description":"Gigablast search engine bot (variant spelling)","tags":["search-engine"]},{"pattern":"GimmeUSAbot","url":"https://gimmeusa.com/pages/crawler","description":"GimmeUSA business directory crawler","tags":["seo"]},{"pattern":"Google AppsViewer","url":"http://drive.google.com","description":"Google Apps document viewer","tags":["social-preview"]},{"pattern":"Google Desktop","url":"http://desktop.google.com/","description":"Google Desktop search application","tags":["search-engine"]},{"pattern":"Google Keyword Tool","url":"http://adwords.google.com/select/KeywordToolExternal","description":"Google AdWords Keyword Tool","tags":["seo"]},{"pattern":"Google-Podcast","description":"Google Podcast crawler","tags":["feed-reader"]},{"pattern":"Google-Youtube-Links","description":"Google YouTube link crawler","tags":["social-preview"]},{"pattern":"GoogleSites","description":"Google Sites web crawler","tags":["social-preview"]},{"pattern":"Gookey\\.co","url":"http://gookey.co/","description":"Gookey search and discovery crawler","tags":["seo"]},{"pattern":"GurujiBot","url":"http://www.guruji.com/en/WebmasterFAQ.html","description":"Guruji Indian search engine crawler","tags":["search-engine"]},{"pattern":"Heurekabot","url":"http://sluzby.heureka.cz/","description":"Heureka shopping comparison crawler","tags":["seo"]},{"pattern":"HomeTags","url":"http://www.hometags.nl/bot","description":"HomeTags real estate crawler","tags":["seo"]},{"pattern":"IODC","url":"https://iodc.co.uk","description":"IODC Internet Engineering Odysseus web survey crawler","tags":["seo"]},{"pattern":"IXEbot","url":"http://medialab.di.unipi.it/IXEbot.html","description":"IXE academic web crawler from University of Pisa","tags":["academic"]},{"pattern":"InbyBot","url":"http://inbyapp.com","description":"Inby app crawler","tags":["seo"]},{"pattern":"Infohelfer","url":"http://www.infohelfer.de/crawler.php","description":"Infohelfer German information crawler","tags":["seo"]},{"pattern":"Javelin","url":"https://about.javelin.io/","description":"Javelin web crawler","tags":["seo"]},{"pattern":"JobKeresoBot","url":"https://www.kozvetlen-allasok.hu/help.jsp","description":"Hungarian job search crawler","tags":["seo"]},{"pattern":"JobdiggerSpider","url":"http://www.jobdigger.nl/spider","description":"Jobdigger Netherlands job search crawler","tags":["seo"]},{"pattern":"Jobrapido","url":"http://www.jobrapido.com","description":"Jobrapido job search aggregator crawler","tags":["seo"]},{"pattern":"JungleKeyThumbnail","url":"http://www.junglekey.fr/","description":"JungleKey thumbnail generation crawler","tags":["social-preview"]},{"pattern":"KAZ\\.KZ_Bot","description":"KAZ.KZ Kazakh search engine crawler","tags":["search-engine"]},{"pattern":"KaloogaBot","url":"http://www.kalooga.com/info.html?page=crawler","description":"Kalooga video search crawler","tags":["search-engine"]},{"pattern":"Kelny","url":"http://www.kelny.com/","description":"Kelny web crawler","tags":["seo"]},{"pattern":"Keywords Research","url":"http://www.check-domains.com/keyword-research/keywords-adwords.php","description":"Check-Domains keyword research tool","tags":["seo"]},{"pattern":"Exabot-Thumbnails","description":"Exabot thumbnail generation crawler","tags":["social-preview"]},{"pattern":"zeerch\\.com","url":"http://www.zeerch.com/bot.php","description":"Zeerch search engine crawler","tags":["search-engine"]},{"pattern":"LXRbot","url":"http://lxrseo.com/","description":"LXR SEO and marketplace crawler","tags":["seo"]},{"pattern":"Licorne Image Snapshot","url":"http://www.jednorozec.cz","description":"Licorne (Jednorozec) image snapshot crawler","tags":["social-preview"]},{"pattern":"LinkMarketbot","url":"http://www.linkmarket.com/","description":"LinkMarket link analysis crawler","tags":["seo"]},{"pattern":"LuminateBot","url":"http://www.luminate.com/bot/","description":"Luminate web crawler","tags":["seo"]},{"pattern":"METASpider","url":"http://meta.ua/spider","description":"Meta.ua Ukrainian search engine crawler","tags":["search-engine"]},{"pattern":"MFGPagesBot","url":"http://www.mfgpages.com","description":"MFGPages manufacturing directory crawler","tags":["seo"]},{"pattern":"MapperCmd","description":"Microsoft Mapper command-line tool","tags":["browser-automation"]},{"pattern":"Clarsentia","description":"Clarsentia web crawler","tags":["seo"]},{"pattern":"Akamai_Site_Analyzer","description":"Akamai site analysis tool","tags":["monitoring"]},{"pattern":"WAC-OFU","description":"WAC-OFU web crawler","tags":["seo"]},{"pattern":"SiteMonitor","description":"Site monitoring crawler","tags":["monitoring"]},{"pattern":"Daumoa","url":"http://tab.search.daum.net/aboutWebSearch.html","description":"Daum Korean search engine crawler","tags":["search-engine"]},{"pattern":"Daum","url":"http://tab.search.daum.net/aboutWebSearch.html","description":"Daum Korean search engine crawler (variant)","tags":["search-engine"]},{"pattern":"MetamojiCrawler","url":"http://www.metamoji.com/jp/crawler.html","description":"Metamoji note-taking app crawler","tags":["seo"]},{"pattern":"bixo","url":"http://wiki.github.com/bixo/bixo/bixocrawler","description":"Bixo open-source web crawler toolkit","tags":["seo"]},{"pattern":"Mp3Bot","url":"http://mp3realm.org/mp3bot/","description":"Mp3Realm music search crawler","tags":["seo"]},{"pattern":"Najdi\\.si","description":"Najdi.si Slovenian search engine crawler","tags":["search-engine"]},{"pattern":"Netseer","url":"http://www.netseer.com/crawler.html","description":"Netseer contextual advertising crawler","tags":["advertising"]},{"pattern":"Nigma\\.ru","description":"Nigma Russian search engine crawler","tags":["search-engine"]},{"pattern":"OpenfosBot","url":"http://www.openfos.com","description":"Openfos web crawler","tags":["seo"]},{"pattern":"OpenindexSpider","url":"http://www.openindex.io/en/webmasters/spider.html","description":"OpenIndex web crawler","tags":["search-engine"]},{"pattern":"OptimizationCrawler","url":"http://www.domainoptima.com/bot.html","description":"DomainOptima optimization crawler","tags":["seo"]},{"pattern":"OrangeBot","description":"Orange telecom web crawler","tags":["seo"]},{"pattern":"OsO","url":"http://oso.octopodus.com/abot.html","description":"Octopodus OsO search crawler","tags":["search-engine"]},{"pattern":"Owler","description":"Owler business information crawler","tags":["seo"]},{"pattern":"PRTG Network Monitor","url":"https://www.paessler.com/","description":"Paessler PRTG network monitoring tool","tags":["monitoring"]},{"pattern":"Pandeo","url":"http://pandeo.de/bot.php","description":"Pandeo web crawler","tags":["seo"]},{"pattern":"ParsijooBot","url":"http://www.parsijoo.ir/","description":"Parsijoo Iranian search engine crawler","tags":["search-engine"]},{"pattern":"Peepowbot","url":"http://www.peepow.com/bot.php","description":"Peepow web crawler","tags":["seo"]},{"pattern":"Peew","url":"http://www.peew.de/crawler/","description":"Peew German web crawler","tags":["seo"]},{"pattern":"Pi-Monster","url":"https://pricepi.com/","description":"PricePi price comparison crawler","tags":["seo"]},{"pattern":"Prlog","url":"http://prlog.ru/","description":"Prlog Russian web crawler","tags":["seo"]},{"pattern":"ProCog(?:SEO)?Bot","url":"http://www.procog.com/spider.html","description":"ProCog SEO and web crawler","tags":["seo"]},{"pattern":"ProductoDownloadUrlBot","url":"http://www.producto.de/","description":"Producto download URL crawler","tags":["seo"]},{"pattern":"ProspectB2B","url":"http://prospectb2b.com","description":"ProspectB2B business data crawler","tags":["seo"]},{"pattern":"Qseero","url":"http://www.q0.com","description":"Q0 Qseero search crawler","tags":["search-engine"]},{"pattern":"Qualidator\\.com","description":"Qualidator website quality analyzer","tags":["seo"]},{"pattern":"RankvalBot","url":"https://rankval.com","description":"Rankval SEO ranking crawler","tags":["seo"]},{"pattern":"RelateIQ","url":"http://www.relateiq.com","description":"RelateIQ CRM data crawler","tags":["seo"]},{"pattern":"RukiCrawler","url":"http://ruki.rezko.net","description":"Ruki web crawler","tags":["seo"]},{"pattern":"RyowlEngine","url":"https://ryowl.org","description":"Ryowl search engine crawler","tags":["search-engine"]},{"pattern":"SEODat","url":"http://crawler.seodat.com","description":"SEODat SEO analysis crawler","tags":["seo"]},{"pattern":"SEOlyticsCrawler","url":"http://crawler.seolytics.net/","description":"SEOlytics SEO analytics crawler","tags":["seo"]},{"pattern":"SOLOFIELD","url":"http://solofield.net/bot.html","description":"Solofield web crawler","tags":["seo"]},{"pattern":"SSLBot","url":"http://www.sslstats.com/sslbot","description":"SSL statistics crawler","tags":["scanner"]},{"pattern":"STINGbot","url":"http://136.186.231.16","description":"STING web crawler","tags":["seo"]},{"pattern":"SWEBot","url":"http://swebot.net","description":"Swedish web crawler","tags":["seo"]},{"pattern":"Scarlett","url":"http://www.ellerdale.com/crawler.html","description":"Ellerdale Scarlett crawler","tags":["seo"]},{"pattern":"Scopia Crawler","url":"http://www.scopia.co","description":"Scopia web crawler","tags":["seo"]},{"pattern":"Scrutiny","url":"https://peacockmedia.software/mac/scrutiny/","description":"Peacock Media Scrutiny SEO tool","tags":["seo"]},{"pattern":"Search17Bot","url":"http://www.search17.com/bot.php","description":"Search17 search engine crawler","tags":["search-engine"]},{"pattern":"SecretSerachEngineLabs","url":"http://www.secretsearchenginelabs.com/secret-web-crawler.php","description":"Secret Search Engine Labs crawler","tags":["search-engine"]},{"pattern":"SecurityResearch\\.bot","url":"http://besome1.info/securityresearchbot.html","description":"Security research bot","tags":["scanner"]},{"pattern":"Semager","url":"http://www.semager.de/blog/semager-bots/","description":"Semager semantic search crawler","tags":["search-engine"]},{"pattern":"Semiocast","url":"http://semiocast.com/","description":"Semiocast social media analytics crawler","tags":["seo"]},{"pattern":"Shareaholicbot","url":"http://www.shareaholic.com/bot","description":"Shareaholic content sharing crawler","tags":["social-preview"]},{"pattern":"SiteAnalyzerBot","url":"https://www.site-analyzer.com/","description":"Site-Analyzer SEO analysis bot","tags":["seo"]},{"pattern":"SiteCondor","url":"http://www.sitecondor.com","description":"SiteCondor web crawler","tags":["seo"]},{"pattern":"SiteIndexed","url":"http://www.siteindexed.com","description":"SiteIndexed web indexing crawler","tags":["seo"]},{"pattern":"Siteluxbot","url":"https://sitelux.co.uk/","description":"Sitelux UK web crawler","tags":["seo"]},{"pattern":"Sitemap Generator","url":"http://www.check-domains.com/sitemap/index.php","description":"Check-Domains sitemap generator","tags":["seo"]},{"pattern":"SputnikFaviconBot","url":"http://corp.sputnik.ru/webmaster","description":"Sputnik favicon crawler","tags":["social-preview"]},{"pattern":"Steeler","url":"http://www.tkl.iis.u-tokyo.ac.jp/~crawler/","description":"University of Tokyo Steeler crawler","tags":["academic"]},{"pattern":"SwiteScraper","description":"Swite scraper tool","tags":["http-library"]},{"pattern":"TWMBot","url":"http://thewebminer.com","description":"TheWebMiner crawler","tags":["seo"]},{"pattern":"TeeRaidBot","url":"https://teeraid.com/bot/","description":"TeeRaid web crawler","tags":["seo"]},{"pattern":"Tetrahedron","url":"https://compute.info/tet","description":"Compute.info Tetrahedron crawler","tags":["seo"]},{"pattern":"The Drop Reaper","url":"http://tag1consulting.com/dropreaper.html","description":"Tag1 Consulting Drop Reaper crawler","tags":["seo"]},{"pattern":"Thumbshots\\.ru","url":"http://thumbshots.ru/bot","description":"Thumbshots Russian thumbnail service","tags":["social-preview"]},{"pattern":"Topicbot","url":"http://topicbot.awardspace.us/","description":"Topicbot web crawler","tags":["search-engine"]},{"pattern":"Tweezler","url":"http://tweezler.com","description":"Tweezler web crawler","tags":["seo"]},{"pattern":"Hootsuite-WebFeed","description":"Hootsuite social media feed crawler","tags":["social-preview"]},{"pattern":"UnisterBot","url":"http://www.bluekiwi.de/misc/imprint","description":"Unister web crawler","tags":["seo"]},{"pattern":"Urlfilebot","url":"http://urlfile.com/bot.html","description":"Urlfile web crawler","tags":["http-library"]},{"pattern":"VSAgent","description":"VSAgent crawler","tags":["http-library"]},{"pattern":"VideoSurf_bot","url":"http://www.videosurf.com/bot.html","description":"VideoSurf video search crawler","tags":["search-engine"]},{"pattern":"WBSearchBot","url":"http://www.warebay.com/bot.html","description":"WareBay search bot","tags":["search-engine"]},{"pattern":"Wappalyzer","url":"https://www.wappalyzer.com","description":"Technology profiler that identifies software on websites","tags":["seo"]},{"pattern":"WbSrch","url":"https://wbsrch.com","description":"WbSrch web search crawler","tags":["search-engine"]},{"pattern":"Web-Monitoring","url":"http://monoid.nic.ru/","description":"Russian web monitoring service","tags":["monitoring"]},{"pattern":"WebCookies","url":"https://webcookies.org/faq/#agent","description":"WebCookies crawler","tags":["seo"]},{"pattern":"WebCorp","url":"http://www.webcorp.org.uk","description":"WebCorp linguistic search engine","tags":["academic"]},{"pattern":"WebNL","url":"http://www.web.nl/webmasters/spider.html","description":"Web.nl search crawler","tags":["search-engine"]},{"pattern":"Webauskunft","url":"http://www.webauskunft.org","description":"Webauskunft.org crawler","tags":["seo"]},{"pattern":"Website Analyzer","url":"http://www.check-domains.com/website-analysis/website-analyzer.php","description":"Website analysis tool","tags":["seo"]},{"pattern":"Whynder Magnet","url":"http://whynder.com/magnet","description":"Whynder content discovery bot","tags":["http-library"]},{"pattern":"WoTBoT","url":"https://www.wslta.com/WoTBoT.html","description":"WSLTA web crawler","tags":["http-library"]},{"pattern":"YYSpider","url":"http://www.yunyun.com/spider.html","description":"YunYun search spider","tags":["search-engine"]},{"pattern":"YandexCatalog","url":"http://yandex.com/bots","description":"Yandex catalog crawler","tags":["search-engine"]},{"pattern":"YandexZakladki","url":"http://yandex.com/bots","description":"Yandex bookmarks crawler","tags":["search-engine"]},{"pattern":"YesupBot","url":"http://www.yesup.net/bot.html","description":"Yesup search bot","tags":["search-engine"]},{"pattern":"YioopBot","url":"http://www.yioop.com/bot.php","description":"Yioop search engine crawler","tags":["search-engine"]},{"pattern":"YodaoBot","url":"http://www.yodao.com/help/webmaster/spider/","description":"Yodao search engine bot","tags":["search-engine"]},{"pattern":"Your-Website-Sucks","url":"http://www.Your-Website-Sucks.net/","description":"Website quality analysis bot","tags":["seo"]},{"pattern":"ZnHTTP","description":"ZnHTTP library crawler","tags":["http-library"]},{"pattern":"abby","url":"http://www.ellerdale.com/crawler.html","description":"Ellerdale crawler","tags":["http-library"]},{"pattern":"adbeat-publisher-description-fetcher","description":"Adbeat publisher description fetcher","tags":["advertising"]},{"pattern":"akula","url":"http://k311.fd.cvut.cz/","description":"Akula academic crawler","tags":["academic"]},{"pattern":"archive-com.com","url":"http://archive-com.com/bot","description":"Archive.com archiving bot","tags":["archiver"]},{"pattern":"discoverybot","url":"http://discoveryengine.com/discoverybot.html","description":"Discovery Engine crawler","tags":["search-engine"]},{"pattern":"dlcbot","url":"http://www.drlinkcheck.com/","description":"Dr. Link Check bot","tags":["monitoring"]},{"pattern":"dotMailer","description":"dotMailer content retrieval service","tags":["http-library"]},{"pattern":"dotSemantic","url":"http://www.dotsemantic.org","description":"dotSemantic crawler","tags":["seo"]},{"pattern":"emefgebot","url":"http://emefge.de/bot.html","description":"Emefge crawler","tags":["http-library"]},{"pattern":"etoolsbot","url":"https://etools.io/bots/","description":"eTools crawler","tags":["seo"]},{"pattern":"firmilybot","url":"http://www.firmily.com/bot.php","description":"Firmily business directory crawler","tags":["seo"]},{"pattern":"forensiq","url":"http://www.forensiq.com","description":"Forensiq ad fraud detection bot","tags":["advertising"]},{"pattern":"gimme60bot","url":"http://gimme60.com","description":"Gimme60 crawler","tags":["http-library"]},{"pattern":"Microsoft Outlook","description":"Microsoft Outlook email client","tags":["http-library"]},{"pattern":"Mac OS X\\) (?:Word|Excel|PowerPoint)\\/","description":"Microsoft Office for Mac","tags":["http-library"]},{"pattern":"ForusP","description":"Forus crawler","tags":["http-library"]},{"pattern":"stumbleupon","url":"http://stumbleupon.com","description":"StumbleUpon discovery bot","tags":["social-preview"]},{"pattern":"diffbot","url":"http://www.diffbot.com","description":"Diffbot structured data extraction bot","tags":["ai-fetcher"]},{"pattern":"ExaleadCloudview","description":"Exalead Cloudview crawler","tags":["search-engine"]},{"pattern":"sitexy","url":"http://sitexy.com/","description":"Sitexy webmaster tools crawler","tags":["seo"]},{"pattern":"gofind","url":"http://govid.mobi/bot.php","description":"GoFind mobile search bot","tags":["search-engine"]},{"pattern":"grapeFX","url":"http://www.grapeshot.co.uk/","description":"Grapeshot contextual targeting crawler","tags":["advertising"]},{"pattern":"historious","url":"http://historio.us/","description":"Historious bookmarks history crawler","tags":["social-preview"]},{"pattern":"houzzbot","url":"http://www.houzz.com/","description":"Houzz image and design crawler","tags":["ai-fetcher"]},{"pattern":"alyze","url":"http://alyze.info","description":"Alyze SEO analysis crawler","tags":["seo"]},{"pattern":"iThemes Sync","url":"http://ithemes.com/sync","description":"iThemes WordPress sync service","tags":["monitoring"]},{"pattern":"idmarch Automatic","url":"http://www.idmarch.org/bot.html","description":"IDMarch Automatic web monitoring bot","tags":["monitoring"]},{"pattern":"imbot","url":"http://internetmemory.org/en/","description":"Internet Memory archiving bot","tags":["archiver"]},{"pattern":"informatique at mixdata","description":"Mixdata information extraction crawler","tags":["http-library"]},{"pattern":"janforman","url":"http://janforman.com/","description":"Janforman web crawler","tags":["seo"]},{"pattern":"kulturarw3","url":"http://www.kb.se/om/projekt/Svenska-webbsidor---Kulturarw3/","description":"Kulturarw3 Swedish web archive crawler","tags":["archiver"]},{"pattern":"localsearch-web","url":"http://www.localsearch.ch/en/legal-notice","description":"Local Search crawler","tags":["search-engine"]},{"pattern":"ltbot","url":"http://www.kdsl.tu-darmstadt.de/de/kdsl/research-program/crawling-and-semantic-structuring/","description":"TU Darmstadt semantic web crawler","tags":["academic"]},{"pattern":"memoryBot","url":"http://internetmemory.org/en/","description":"Internet Memory archive bot","tags":["archiver"]},{"pattern":"metager2-verification-bot","url":"http://metager2.de/technology.php","description":"MetaGer2 search engine bot","tags":["search-engine"]},{"pattern":"oBot","url":"http://www-935.ibm.com/services/us/index.wss/detail/iss/a1029077","description":"IBM OpenView operations bot","tags":["monitoring"]},{"pattern":"openstat","url":"http://openstat.ru/","description":"OpenStat Russian analytics bot","tags":["monitoring"]},{"pattern":"parsijoo-bot","url":"http://www.parsijoo.ir/","description":"Parsijoo Persian search engine bot","tags":["search-engine"]},{"pattern":"parsijoo-update-crawler","url":"http://www.parsijoo.ir/","description":"Parsijoo update crawler","tags":["search-engine"]},{"pattern":"parsijoo","url":"http://www.parsijoo.ir/","description":"Parsijoo Persian search engine","tags":["search-engine"]},{"pattern":"pdffillerbot","url":"http://www.pdffiller.com/en/support.htm","description":"PDFFiller content extraction bot","tags":["ai-fetcher"]},{"pattern":"scanbot","url":"http://dazzlepod.com/ip/","description":"Dazzlepod IP scanner bot","tags":["scanner"]},{"pattern":"seocompany","description":"SEO Company crawler","tags":["seo"]},{"pattern":"seplinkbot","description":"SepLink crawler","tags":["seo"]},{"pattern":"socialbm_bot","url":"http://spider.socialbm.net","description":"SocialBM bookmark crawler","tags":["social-preview"]},{"pattern":"socketcrawler","url":"http://nlp.fi.muni.cz/projects/biwec/","description":"Masaryk University BiWec corpus crawler","tags":["academic"]},{"pattern":"special_archiver","url":"http://www.loc.gov/webarchiving/","description":"Library of Congress web archiver","tags":["archiver"]},{"pattern":"suggybot","url":"http://blog.suggy.com/was-ist-suggy/suggy-webcrawler/","description":"Suggy content discovery bot","tags":["seo"]},{"pattern":"tldstat","url":"http://tldstat.com/bot.html","description":"TLD Statistics crawler","tags":["seo"]},{"pattern":"uCrawlr","url":"https://blog.ucoz.ru/upolicy","description":"UcozU platform crawler","tags":["http-library"]},{"pattern":"value\\/","url":"http://www.aihit.com/","description":"AIHit value crawler","tags":["seo"]},{"pattern":"monitor.us","url":"http://www.monitor.us","description":"Monitor.us uptime monitoring bot","tags":["monitoring"]},{"pattern":"zitebot","url":"http://zite.com","description":"Zite content discovery bot","tags":["ai-fetcher"]},{"pattern":"ParchBot","url":"http://www.parchmenthill.com/search.htm","description":"Parchment Hill search bot","tags":["search-engine"]},{"pattern":"Google PP Default","description":"Google Publisher Profiles bot","tags":["advertising"]},{"pattern":"Google-Adwords-DisplayAds-WebRender","description":"Google Ads display crawler","tags":["advertising"]},{"pattern":"mixed-content-scan","description":"Mixed content security scanner","tags":["scanner"]},{"pattern":"iplabel","description":"IP Label analysis bot","tags":["monitoring"]},{"pattern":"Siege","url":"https://www.joedog.org/siege-home/","description":"Siege HTTP load testing tool","tags":["scanner"]},{"pattern":"Zombie.js","url":"http://zombie.js.org/","description":"Zombie.js headless browser testing","tags":["browser-automation"]},{"pattern":"WhoRunsCoinHive","description":"Coin mining detection bot","tags":["scanner"]},{"pattern":"SnapSearch","url":"https://www.snapsearch.io/","description":"SnapSearch SEO crawler","tags":["seo"]},{"pattern":"Project 25499","url":"http://project25499.com","description":"Project 25499 crawler","tags":["http-library"]},{"pattern":"Tweetminster","description":"Tweetminster social analytics bot","tags":["social-preview"]},{"pattern":"LieBaoFast","description":"LieBao Fast mobile browser","tags":["http-library"]},{"pattern":"Cula","url":"https://cula.io/","description":"Cula visual analytics crawler","tags":["ai-fetcher"]},{"pattern":"Monitority","url":"http://www.monitority.com/","description":"Monitority website monitoring bot","tags":["monitoring"]},{"pattern":"Mr.4x3","description":"Mr.4x3 web utility","tags":["http-library"]},{"pattern":"MrakyPraceCzBot","url":"http://www.mrakyprace.cz/about-crawler","description":"MrakyPrace Czech job search crawler","tags":["search-engine"]},{"pattern":"NETCRAFT","url":"http://www.netcraft.com/","description":"Netcraft web server survey","tags":["monitoring"]},{"pattern":"NG-Search","url":"http://www.ng-search.com","description":"NG-Search engine crawler","tags":["search-engine"]},{"pattern":"^NG/2\\.","description":"NG generic crawler","tags":["http-library"]},{"pattern":"NalezenCzBot","url":"http://www.nalezen.cz","description":"Nalezen Czech search engine bot","tags":["search-engine"]},{"pattern":"Needle","url":"https://github.com/needleio/needle","description":"Needle Node.js HTTP client","tags":["http-library"]},{"pattern":"NerdyBot","description":"Nerdy Bot crawler","tags":["http-library"]},{"pattern":"NetLyzer FastProbe","url":"http://netlyzer.com/","description":"NetLyzer speed and network probe","tags":["monitoring"]},{"pattern":"NetResearchServer","url":"http://www.look.com","description":"Net Research Server crawler","tags":["search-engine"]},{"pattern":"NetTrack","url":"https://nettrack.info/support.php","description":"NetTrack analytics bot","tags":["monitoring"]},{"pattern":"Netlab360","description":"Netlab 360 security crawler","tags":["scanner"]},{"pattern":"NetpeakCheckerBot","description":"Netpeak SEO checker bot","tags":["seo"]},{"pattern":"Netpursual","description":"Netpursual crawler","tags":["http-library"]},{"pattern":"Netsparker","url":"https://www.netsparker.com/","description":"Netsparker web application security scanner","tags":["scanner"]},{"pattern":"NeumobBot","url":"http://www.neumob.com/bot","description":"Neumob mobile optimization bot","tags":["monitoring"]},{"pattern":"NeutrinoAPI","url":"https://www.neutrinoapi.com/","description":"NeutrinoAPI utility API client","tags":["http-library"]},{"pattern":"NewsGator","url":"http://www.newsgator.com","description":"NewsGator feed aggregator","tags":["feed-reader"]},{"pattern":"Nexgate Ruby Client","description":"Nexgate Ruby HTTP client","tags":["http-library"]},{"pattern":"NextGenSearchBot","url":"http://www.zoominfo.com/","description":"ZoomInfo business data crawler","tags":["seo"]},{"pattern":"NihilScio","url":"https://www.nihilscio.it/NihilScio.htm","description":"NihilScio educational search engine","tags":["search-engine"]},{"pattern":"botmobi","url":"http://find.mobi/bot.html","description":"Botmobi mobile content crawler","tags":["http-library"]},{"pattern":"Norton-Safeweb","url":"https://safeweb.norton.com/","description":"Norton SafeWeb security checker","tags":["scanner"]},{"pattern":"Notifixious","url":"http://notifixio.us","description":"Notifixious link checking service","tags":["monitoring"]},{"pattern":"NotionEmbedder","url":"https://www.notion.so/","description":"Notion content embedder","tags":["ai-fetcher"]},{"pattern":"Nuclei","url":"https://projectdiscovery.io/nuclei/","description":"Project Discovery vulnerability scanner","tags":["scanner"]},{"pattern":"Nuhk","url":"http://www.neti.ee/cgi-bin/abi/otsing.html","description":"Neti Estonian search engine bot","tags":["search-engine"]},{"pattern":"Nusearch Spider","url":"http://www.nusearch.com","description":"Nusearch web crawler","tags":["search-engine"]},{"pattern":"Nymesis","url":"http://nymesis.com","description":"Nymesis web analytics bot","tags":["monitoring"]},{"pattern":"OMSC","description":"OMSC crawler","tags":["http-library"]},{"pattern":"OOZBOT","url":"http://www.setooz.com/oozbot.html","description":"Setooz web crawler","tags":["search-engine"]},{"pattern":"OSPScraper","url":"https://www.opensyllabusproject.org","description":"Open Syllabus Project crawler","tags":["academic"]},{"pattern":"Ocelli","url":"http://www.globalspec.com/Ocelli","description":"GlobalSpec technical search crawler","tags":["search-engine"]},{"pattern":"OgScrper","description":"Open Graph scraper","tags":["social-preview"]},{"pattern":"OhDear","url":"https://ohdear.app/docs/checks/uptime","description":"Oh Dear uptime monitoring bot","tags":["monitoring"]},{"pattern":"OmniExplorer_Bot","url":"http://www.omni-explorer.com","description":"Omni Explorer world indexing bot","tags":["search-engine"]},{"pattern":"OnalyticaBot","url":"https://www.onalytica.com/","description":"Onalytica influence mapping bot","tags":["seo"]},{"pattern":"Online Virus Scanner","url":"http://tools.geek-tools.org","description":"Geek Tools virus scanner","tags":["scanner"]},{"pattern":"Open Web Analytics","url":"http://www.openwebanalytics.com/","description":"Open Web Analytics bot","tags":["monitoring"]},{"pattern":"OpenCalaisSemanticProxy","url":"http://www.opencalais.com/","description":"OpenCalais semantic analysis proxy","tags":["ai-fetcher"]},{"pattern":"Opendi Screenshot Bot","url":"http://www.opendi.com","description":"Opendi screenshot crawler","tags":["ai-fetcher"]},{"pattern":"Openstat","description":"Openstat analytics bot","tags":["monitoring"]},{"pattern":"Orbiter","url":"http://dailyorbit.com/","description":"Daily Orbit news aggregator bot","tags":["feed-reader"]},{"pattern":"contactbigdatafr","description":"Contact Big Data crawler","tags":["http-library"]},{"pattern":"OrgProbe","url":"http://www.blocked.org.uk","description":"Blocked.org.uk probe bot","tags":["monitoring"]},{"pattern":"Orgbybot","url":"http://www.orgby.com/","description":"Orgby search engine crawler","tags":["search-engine"]},{"pattern":"Outlook-Express","description":"Outlook Express email client","tags":["http-library"]},{"pattern":"Outlook-iOS","description":"Outlook iOS mail client","tags":["http-library"]},{"pattern":"OxfordCloudService","description":"Oxford cloud service client","tags":["http-library"]},{"pattern":"PEAR HTTPRequest","url":"http://pear.php.net/","description":"PEAR PHP HTTP library","tags":["http-library"]},{"pattern":"PHP-AYMAPI","description":"PHP AYMAPI client","tags":["http-library"]},{"pattern":"^PHP/","description":"PHP runtime environment","tags":["http-library"]},{"pattern":"PINGOMETER_BOT","url":"https://pingometer.com","description":"Pingometer uptime monitoring bot","tags":["monitoring"]},{"pattern":"POE-Component-Client-HTTP","description":"POE Perl HTTP client library","tags":["http-library"]},{"pattern":"Page Valet","description":"Page Valet web validator","tags":["seo"]},{"pattern":"PageFreezer","url":"https://www.pagefreezer.com/","description":"PageFreezer web archiver","tags":["archiver"]},{"pattern":"PagePeeker","url":"http://pagepeeker.com","description":"PagePeeker screenshot service","tags":["ai-fetcher"]},{"pattern":"Pattern","url":"http://www.clips.ua.ac.be/pattern","description":"Pattern web mining library","tags":["http-library"]},{"pattern":"PayPal IPN","url":"https://www.paypal.com/ipn","description":"PayPal instant payment notification","tags":["http-library"]},{"pattern":"Pearltrees","url":"https://www.pearltrees.com/","description":"Pearltrees curation platform bot","tags":["social-preview"]},{"pattern":"Peeplo Screenshot Bot","description":"Peeplo screenshot crawler","tags":["ai-fetcher"]},{"pattern":"PercolateCrawler","url":"https://www.percolate.com/","description":"Percolate content marketing crawler","tags":["ai-fetcher"]},{"pattern":"Photon/[0-9]","description":"Photon image crawler","tags":["http-library"]},{"pattern":"PingSpot","url":"http://www.pingspot.org/","description":"PingSpot monitoring bot","tags":["monitoring"]},{"pattern":"Pingability","url":"https://pingability.com/","description":"Pingability monitoring service","tags":["monitoring"]},{"pattern":"Pingoscope","description":"Pingoscope monitoring service","tags":["monitoring"]},{"pattern":"Pinterest/[0-9]","url":"http://pinterest.com/","description":"Pinterest content discovery bot","tags":["social-preview"]},{"pattern":"Pixray-Seeker","url":"http://www.pixray.com/pixraybot","description":"Pixray image search crawler","tags":["seo"]},{"pattern":"Pizilla","description":"Pizilla web crawler","tags":["http-library"]},{"pattern":"Plagger","url":"http://plagger.org/","description":"Plagger feed aggregator","tags":["feed-reader"]},{"pattern":"Pleroma","url":"https://search.fedi.app","description":"Pleroma federated social network indexer","tags":["social-preview"]},{"pattern":"Ploetz \\+ Zeller","url":"http://www.ploetz-zeller.de","description":"Ploetz Zeller link validator","tags":["monitoring"]},{"pattern":"PocketImageCache","description":"Pocket image cache service","tags":["ai-fetcher"]},{"pattern":"PodcastAddict","url":"https://podcastaddict.com/","description":"Podcast Addict RSS reader","tags":["feed-reader"]},{"pattern":"Polymail","description":"Polymail email client","tags":["http-library"]},{"pattern":"Pompos","url":"http://dir.com/pompos.html","description":"Pompos web directory crawler","tags":["search-engine"]},{"pattern":"Porkbun","url":"http://porkbun.com","description":"Porkbun website analyzer","tags":["seo"]},{"pattern":"Port Monitor check service","url":"http://www.port-monitor.com","description":"Port Monitor service checker","tags":["monitoring"]},{"pattern":"PostPost","url":"http://postpost.com/crawlers","description":"PostPost social discovery bot","tags":["social-preview"]},{"pattern":"PostmanRuntime","url":"https://www.getpostman.com/","description":"Postman API testing tool","tags":["http-library"]},{"pattern":"Prebid.js Scraper","url":"https://prebid.org/","description":"Prebid.js ad auction crawler","tags":["advertising"]},{"pattern":"PritTorrent","description":"PritTorrent client","tags":["http-library"]},{"pattern":"Project25499 Scanner","url":"http://project25499.com","description":"Project 25499 security scanner","tags":["scanner"]},{"pattern":"Promotion_Tools_www.searchenginepromotionhelp.com","url":"http://www.searchenginepromotionhelp.com","description":"Search Engine Promotion Help tool","tags":["seo"]},{"pattern":"Pu_iN Crawler","url":"http://semanticjuice.com/","description":"SemanticJuice semantic analysis crawler","tags":["seo"]},{"pattern":"Pulp","url":"http://www.acrylicapps.com/pulp/","description":"Pulp iPad app","tags":["http-library"]},{"pattern":"Putin","url":"http://semanticjuice.com/","description":"SemanticJuice Putin semantic crawler","tags":["seo"]},{"pattern":"Python-httplib2","description":"Python httplib2 HTTP library","tags":["http-library"]},{"pattern":"Qirina Hurdler","url":"http://www.qirina.com/hurdler.html","description":"Qirina Hurdler SEO analyzer","tags":["seo"]},{"pattern":"QrafterPro","description":"QrafterPro iOS client","tags":["http-library"]},{"pattern":"QuerySeekerSpider","url":"http://queryseeker.com/bot.html","description":"QuerySeeker search bot","tags":["search-engine"]},{"pattern":"Quora Link Preview","url":"http://www.quora.com","description":"Quora link preview bot","tags":["social-preview"]},{"pattern":"RMA\\/","description":"RealMedia advertisement bot","tags":["advertising"]},{"pattern":"RSSMix","url":"http://www.rssmix.com/","description":"RSSMix feed mixer","tags":["feed-reader"]},{"pattern":"RSSOwl","url":"http://www.rssowl.org/","description":"RSSOwl news aggregator","tags":["feed-reader"]},{"pattern":"RadioPublicImageResizer","url":"https://www.radiopublic.com/","description":"RadioPublic podcast image service","tags":["http-library"]},{"pattern":"Railgun","url":"https://www.cloudflare.com/products/railgun/","description":"Cloudflare Railgun optimizer","tags":["http-library"]},{"pattern":"RapidLoad","description":"RapidLoad web optimizer","tags":["http-library"]},{"pattern":"RavenCrawler","description":"Raven crawler","tags":["http-library"]},{"pattern":"Re-re Studio","url":"http://re-re.ru/","description":"Re-re Studio Russian crawler","tags":["search-engine"]},{"pattern":"ReactorNetty","url":"https://projectreactor.io/","description":"Reactor Netty HTTP client library","tags":["http-library"]},{"pattern":"Readability","url":"http://readability.com/about/","description":"Readability content extraction service","tags":["ai-fetcher"]},{"pattern":"RealPlayer%20Downloader","url":"https://www.realplayer.com/","description":"RealPlayer media downloader","tags":["http-library"]},{"pattern":"RebelMouse","url":"http://rebelmouse.com","description":"RebelMouse social curation platform bot","tags":["social-preview"]},{"pattern":"RecurPost","url":"https://recurpost.com","description":"RecurPost social media scheduler bot","tags":["social-preview"]},{"pattern":"Reeder","url":"http://www.reederapp.com/","description":"Reeder RSS reader","tags":["feed-reader"]},{"pattern":"ResponseCodeTest","description":"Response code testing tool","tags":["monitoring"]},{"pattern":"RestSharp","url":"https://restsharp.dev/","description":"RestSharp .NET HTTP client","tags":["http-library"]},{"pattern":"Robosourcer","description":"Robosourcer content crawler","tags":["http-library"]},{"pattern":"Robozilla","description":"Robozilla web crawler","tags":["http-library"]},{"pattern":"RssReader","url":"http://www.rssreader.com","description":"RSS Reader aggregator","tags":["feed-reader"]},{"pattern":"link_thumbnailer","description":"Ruby link thumbnail generator","tags":["ai-fetcher"]},{"pattern":"RyzeCrawler","url":"http://www.domain2day.nl/crawler/","description":"Ryze domain tracking crawler","tags":["seo"]},{"pattern":"SAP NetWeaver","description":"SAP NetWeaver application server","tags":["http-library"]},{"pattern":"SEO Browser","description":"SEO Browser analysis tool","tags":["seo"]},{"pattern":"SEO Consulting","description":"SEO Consulting redirect checker","tags":["seo"]},{"pattern":"SEOENGBot","url":"http://www.seoengine.com/seoengbot.htm","description":"SEO Engine crawler","tags":["seo"]},{"pattern":"SEOstats","url":"https://github.com/eyecatchup/SEOstats","description":"SEO Stats analysis tool","tags":["seo"]},{"pattern":"SMRF URL Expander","description":"SMRF URL expander service","tags":["http-library"]},{"pattern":"SMUrlExpander","description":"SM URL expander utility","tags":["http-library"]},{"pattern":"SPDYCheck","url":"http://spdycheck.org/about.html","description":"SPDY protocol tester","tags":["monitoring"]},{"pattern":"SSI-LinkPreviewGenerator","description":"SSI link preview generator","tags":["ai-fetcher"]},{"pattern":"SafeAds.xyz bot","url":"https://safeads.xyz/","description":"SafeAds ad quality bot","tags":["advertising"]},{"pattern":"SalesIntelligent","url":"https://www.salesintelligent.com","description":"SalesIntelligent B2B intelligence bot","tags":["seo"]},{"pattern":"Saleslift","url":"https://saleslift.io/","description":"Saleslift sales intelligence bot","tags":["seo"]},{"pattern":"SanszBot","url":"http://www.sansz.org/sanszbot","description":"Sansz search engine bot","tags":["search-engine"]},{"pattern":"SauceNAO","url":"http://saucenao.com/","description":"SauceNAO reverse image search bot","tags":["ai-fetcher"]},{"pattern":"Scamadviser-Frontend","url":"https://www.scamadviser.com/","description":"Scamadviser website checker","tags":["scanner"]},{"pattern":"Scoop","description":"Scoop crawler","tags":["http-library"]},{"pattern":"Scooter","description":"Scooter web crawler","tags":["search-engine"]},{"pattern":"ScopeContentAG-HTTP-Client","url":"http://www.thescope.com/","description":"Scope content AG HTTP client","tags":["http-library"]},{"pattern":"Scopia crawler","url":"http://www.scopia.co","description":"Scopia video search crawler","tags":["search-engine"]},{"pattern":"ScoutURLMonitor","description":"Scout URL monitoring tool","tags":["monitoring"]},{"pattern":"ScrapeBox Page Scanner","url":"https://www.scrapebox.com/","description":"ScrapeBox SEO tool scanner","tags":["seo"]},{"pattern":"Search37","url":"http://www.search37.com","description":"Search37 web directory bot","tags":["search-engine"]},{"pattern":"SearchSight","url":"http://SearchSight.com/","description":"SearchSight website analyzer","tags":["seo"]},{"pattern":"SearchWP","url":"https://searchwp.com/","description":"SearchWP site search engine","tags":["http-library"]},{"pattern":"Searchie","url":"https://www.searchie.org","description":"Searchie Storm-based crawler","tags":["search-engine"]},{"pattern":"Seeker","url":"http://LookSeek.com/seeker/","description":"LookSeek Seeker bot","tags":["search-engine"]},{"pattern":"Semantifire1","url":"http://www.setooz.com/oozbot.html","description":"Setooz Semantifire crawler","tags":["seo"]},{"pattern":"Sendsay","url":"https://Sendsay.Ru/","description":"Sendsay email service bot","tags":["http-library"]},{"pattern":"Sens.ai","url":"http://sovereign.ai/contact","description":"Sovereign AI sensory crawler","tags":["ai-fetcher"]},{"pattern":"Seo Servis","description":"SEO Services source code analyzer","tags":["seo"]},{"pattern":"SeoCheck","url":"http://www.kfsw.de/bot.html","description":"FischerNetzDesign SEO checker","tags":["seo"]},{"pattern":"SeoCheckBot","url":"http://www.kfsw.de/bot.html","description":"SEO Check bot crawler","tags":["seo"]},{"pattern":"Seobility","url":"https://www.seobility.net/","description":"Seobility SEO audit bot","tags":["seo"]},{"pattern":"Seologies","description":"Seologies crawler","tags":["seo"]},{"pattern":"SetCronJob","url":"http://www.setcronjob.com/","description":"SetCronJob task scheduling service","tags":["monitoring"]},{"pattern":"Shelob","description":"Shelob web crawler","tags":["http-library"]},{"pattern":"ShopWiki","url":"http://www.shopwiki.com/wiki/Help:Bot","description":"ShopWiki product search bot","tags":["search-engine"]},{"pattern":"ShortLinkTranslate","description":"Short link translation service","tags":["http-library"]},{"pattern":"ShowyouBot","url":"http://showyou.com/crawler","description":"Showyou visual content bot","tags":["social-preview"]},{"pattern":"Sideqik","url":"http://www.sideqik.com","description":"Sideqik social media analytics","tags":["social-preview"]},{"pattern":"SilverReader","url":"http://silverreader.com","description":"SilverReader RSS reader","tags":["feed-reader"]},{"pattern":"SimpleChecker","description":"Simple website checker","tags":["monitoring"]},{"pattern":"SimplyFast.info","description":"SimplyFast HTTP headers checker","tags":["monitoring"]},{"pattern":"SiteBar","url":"http://sitebar.org/","description":"SiteBar bookmark server","tags":["social-preview"]},{"pattern":"SiteCheck","url":"http://downforeveryoneorjustme.com","description":"Down For Everyone Or Just Me checker","tags":["monitoring"]},{"pattern":"SiteGuardian","description":"SiteGuardian internet monitoring bot","tags":["monitoring"]},{"pattern":"SiteTruth","url":"http://www.sitetruth.com","description":"SiteTruth site reputation rater","tags":["scanner"]},{"pattern":"Sitedomain-Bot","url":"http://www.sitedomain.de/sitedomain-bot/","description":"Sitedomain crawler","tags":["seo"]},{"pattern":"Siteshooter","description":"Siteshooter screenshot bot","tags":["ai-fetcher"]},{"pattern":"SklikBot","url":"http://napoveda.sklik.cz/","description":"Sklik Czech search engine bot","tags":["search-engine"]},{"pattern":"SmabblerBot","url":"http://www.smabbler.com/","description":"Smabbler content sharing bot","tags":["social-preview"]},{"pattern":"SmerBot","description":"Smer language model dataset fetcher","tags":["ai-crawler"]},{"pattern":"Snapbot","url":"http://www.snap.com","description":"Snap Shots screenshot service bot","tags":["ai-fetcher"]},{"pattern":"Snappy","url":"http://www.urltrends.com/","description":"Snappy URL trends analyzer","tags":["monitoring"]},{"pattern":"Snarfer","url":"http://www.snarfware.com/","description":"Snarfer web crawler","tags":["http-library"]},{"pattern":"SniffRSS","url":"http://www.blogator.com/","description":"SniffRSS feed sniffer","tags":["feed-reader"]},{"pattern":"Snoopy","description":"Snoopy PHP crawler","tags":["http-library"]},{"pattern":"SnowHaze Search","url":"https://www.snowhaze.com/","description":"SnowHaze search engine bot","tags":["search-engine"]},{"pattern":"SortSite","url":"https://www.sortsitestudio.com/","description":"SortSite web quality analyzer","tags":["seo"]},{"pattern":"SpamExperts","description":"SpamExperts antivirus scanner","tags":["scanner"]},{"pattern":"SpeedKit","url":"https://speedkit.com/","description":"SpeedKit CDN optimizer","tags":["http-library"]},{"pattern":"Speedy Spider","url":"http://www.entireweb.com","description":"Entireweb Speedy Spider bot","tags":["search-engine"]},{"pattern":"Sprinklr","url":"https://www.sprinklr.com/","description":"Sprinklr social media management","tags":["social-preview"]},{"pattern":"Sqworm","description":"Sqworm web crawler","tags":["http-library"]},{"pattern":"Statastico","description":"Statastico analytics bot","tags":["monitoring"]},{"pattern":"Statically","url":"https://statically.io/","description":"Statically CDN image optimizer","tags":["http-library"]},{"pattern":"StatoolsBot","url":"http://www.statools.com/bot.html","description":"Statools website statistics","tags":["monitoring"]},{"pattern":"StormCrawler","url":"https://github.com/DigitalPebble/storm-crawler","description":"Storm Crawler open source framework","tags":["http-library"]},{"pattern":"Stratagems Kumo","description":"Stratagems Kumo crawler","tags":["http-library"]},{"pattern":"Stroke.cz","url":"http://stroke.cz","description":"Stroke.cz search engine","tags":["search-engine"]},{"pattern":"Super Monitoring","url":"https://www.supermonitoring.com/","description":"Super Monitoring uptime bot","tags":["monitoring"]},{"pattern":"SuperPagesUrlVerifyBot","description":"SuperPages URL verifier","tags":["monitoring"]},{"pattern":"Superarama.com-Tarama-Botu","url":"http://www.superarama.com","description":"Superarama Turkish search crawler","tags":["search-engine"]},{"pattern":"Surphace Scout","url":"http://www.surphace.com","description":"Surphace Scout web analytics","tags":["monitoring"]},{"pattern":"Swurl","description":"Swurl web aggregator","tags":["http-library"]},{"pattern":"SygolBot","url":"http://www.sygol.com","description":"Sygol search bot","tags":["search-engine"]},{"pattern":"Symfony.* BrowserKit","url":"https://symfony.com/doc/current/components/dom_crawler.html","description":"Symfony PHP browser automation","tags":["browser-automation"]},{"pattern":"Symfony Spider","url":"http://symfony.com/spider","description":"Symfony web spider","tags":["http-library"]},{"pattern":"SynHttpClient-Built","description":"Syn HTTP client","tags":["http-library"]},{"pattern":"Syndirella","description":"Syndirella feed aggregator","tags":["feed-reader"]},{"pattern":"TA SEO Crawler","url":"http://www.tripadvisor.com","description":"TripAdvisor SEO crawler","tags":["seo"]},{"pattern":"TBI-HttpOpenPlugin","url":"https://leakix.net/","description":"LeakIX HTTP plugin scanner","tags":["scanner"]},{"pattern":"TBI-WebScanner","url":"https://leakix.net/","description":"LeakIX web scanner","tags":["scanner"]},{"pattern":"TECHIAITH_CY","url":"https://techiaith.cymru","description":"Techiaith Welsh language tech bot","tags":["http-library"]},{"pattern":"TLSProbe","url":"https://scan.trustnet.venafi.com/","description":"Venafi TLS certificate scanner","tags":["scanner"]},{"pattern":"Tarantula","url":"http://www.enlle.com","description":"Enlle Spanish search engine bot","tags":["search-engine"]},{"pattern":"Taringa UGC","url":"http://www.taringa.net/","description":"Taringa user-generated content bot","tags":["social-preview"]},{"pattern":"Tarmot Gezgin","url":"http://www.tarmot.com/gezgin","description":"Tarmot Turkish search bot","tags":["search-engine"]},{"pattern":"TeamSpeak3-ImageFetcher","url":"https://www.teamspeak.com/","description":"TeamSpeak image fetcher","tags":["social-preview"]},{"pattern":"Technoratibot","url":"https://technorati.com/","description":"Technorati blog search bot","tags":["search-engine"]},{"pattern":"Tenon.io","url":"https://tenon.io/","description":"Tenon web accessibility tester","tags":["scanner"]},{"pattern":"The Expert HTML Source Viewer","url":"http://www.expert-html.com","description":"Expert HTML source viewer","tags":["http-library"]},{"pattern":"ThumbShots-Bot","url":"http://thumbshots.in/bot.html","description":"ThumbShots screenshot service","tags":["ai-fetcher"]},{"pattern":"ThumbSniper","url":"http://thumbsniper.com","description":"ThumbSniper thumbnail generator","tags":["ai-fetcher"]},{"pattern":"Thumbor","url":"https://thumbor.readthedocs.io/","description":"Thumbor image processing library","tags":["http-library"]},{"pattern":"TipTop","url":"http://feeltiptop.com","description":"TipTop crawler","tags":["http-library"]},{"pattern":"Tools4noobs.com","url":"http://tools4noobs.com","description":"Tools4noobs web spider","tags":["http-library"]},{"pattern":"Touche","url":"http://www.touche.com.ve","description":"Touche crawler","tags":["http-library"]},{"pattern":"Toweyabot","url":"http://toweya.com","description":"Toweya crawler","tags":["http-library"]},{"pattern":"Trackuity","url":"http://trackuity.com","description":"Trackuity analytics bot","tags":["monitoring"]},{"pattern":"TrapitAgent","url":"http://trapit.com/about","description":"Trapit feed processor","tags":["feed-reader"]},{"pattern":"Trendsmap Resolver","url":"https://www.trendsmap.com/","description":"Trendsmap trend resolver bot","tags":["monitoring"]},{"pattern":"TwengaBot","url":"http://www.twenga.com/","description":"Twenga product search crawler","tags":["seo"]},{"pattern":"Twikle","url":"http://twikle.com","description":"Twikle web crawler","tags":["http-library"]},{"pattern":"Twisted PageGetter","url":"https://twistedmatrix.com/","description":"Twisted Python page fetcher","tags":["http-library"]},{"pattern":"Typhoeus","url":"https://github.com/typhoeus/typhoeus","description":"Typhoeus Ruby HTTP library","tags":["http-library"]},{"pattern":"URL Verifier by Daanav","url":"http://daanav.com","description":"Daanav URL verification service","tags":["monitoring"]},{"pattern":"URL\\/Emacs","description":"Emacs URL library","tags":["http-library"]},{"pattern":"URLChecker","description":"URL checking service","tags":["monitoring"]},{"pattern":"URLTester","description":"URL testing tool","tags":["monitoring"]},{"pattern":"URLitor","url":"http://urlitor.com","description":"URLitor web utility","tags":["http-library"]},{"pattern":"UdmSearch","url":"http://www.udmsearch.com/","description":"UdmSearch engine crawler","tags":["search-engine"]},{"pattern":"UniversalFeedParser","url":"http://feedparser.org/","description":"Universal Feed Parser","tags":["feed-reader"]},{"pattern":"Untiny","url":"http://untiny.com/","description":"Untiny URL expander","tags":["http-library"]},{"pattern":"UnwindFetchor","url":"http://www.gnip.com/","description":"GNIP Unwind URL fetcher","tags":["http-library"]},{"pattern":"Updownerbot","url":"http://www.updowner.com/bot","description":"Updowner uptime monitoring bot","tags":["monitoring"]},{"pattern":"Uptime.com","url":"http://uptime.com/","description":"Uptime.com monitoring bot","tags":["monitoring"]},{"pattern":"UrlTrends Ranking Updater","url":"http://www.urltrends.com/","description":"URLTrends ranking updater bot","tags":["seo"]},{"pattern":"Urlstat","url":"http://urlstat.io/","description":"Urlstat URL statistics tracker","tags":["monitoring"]},{"pattern":"VB Project","description":"VB Project crawler","tags":["http-library"]},{"pattern":"VCPP71","description":"VCPP71 HTTP client","tags":["http-library"]},{"pattern":"VLC","url":"https://www.videolan.org/vlc/","description":"VLC media player","tags":["http-library"]},{"pattern":"VSB-TUO","url":"http://www.vsb.cz","description":"VSB technical university crawler","tags":["academic"]},{"pattern":"VSE\\/","description":"VSE crawler","tags":["http-library"]},{"pattern":"VYU2","description":"VYU2 OpenRISC crawler","tags":["http-library"]},{"pattern":"VeBot","url":"http://www.veinteractive.com/vebot","description":"VE Interactive bot","tags":["advertising"]},{"pattern":"VidibleScraper","description":"Vidible video metadata scraper","tags":["ai-fetcher"]},{"pattern":"Visbot","url":"http://www.visvo.com/en/webmasters.jsp","description":"Visvo visualization bot","tags":["ai-fetcher"]},{"pattern":"Visited by","url":"http://tools.geek-tools.org","description":"Geek Tools visitor tracker","tags":["monitoring"]},{"pattern":"Visual Rights Group","url":"https://visualrightsgroup.com/","description":"Visual Rights Group content scanner","tags":["scanner"]},{"pattern":"Vulnbusters Meter","url":"http://vulnbusters.com","description":"Vulnbusters vulnerability scanner","tags":["scanner"]},{"pattern":"WDT.io","url":"https://wdt.io/","description":"WDT.io crawler","tags":["http-library"]},{"pattern":"WEPA","url":"http://www.wepa.com/","description":"WEPA search engine","tags":["search-engine"]},{"pattern":"WP Engine","url":"https://wpengine.com/","description":"WP Engine WordPress performance API","tags":["monitoring"]},{"pattern":"WP Rocket","url":"https://wp-rocket.me/","description":"WP Rocket WordPress caching plugin","tags":["http-library"]},{"pattern":"WWW::Document","description":"Perl WWW document fetcher","tags":["http-library"]},{"pattern":"WWW::LayeredExtractor","description":"Perl feed extraction library","tags":["feed-reader"]},{"pattern":"WakeletLinkExpander","url":"https://www.wakelet.com/","description":"Wakelet link expansion service","tags":["http-library"]},{"pattern":"WeCrawlForThePeace","description":"WeCrawlForThePeace ethical crawler","tags":["http-library"]},{"pattern":"WeLikeLinks","description":"WeLikeLinks crawler","tags":["http-library"]},{"pattern":"WeSEE","url":"http://www.wesee.com/bot/","description":"WeSEE ad quality and image bot","tags":["advertising"]},{"pattern":"Web spyder","description":"Web spyder crawler","tags":["http-library"]},{"pattern":"Web-sniffer","url":"http://web-sniffer.net/","description":"Web Sniffer HTTP header analyzer","tags":["monitoring"]},{"pattern":"WebAlta Crawler","url":"http://www.webalta.net/ru/about_webmaster.html","description":"WebAlta Russian search bot","tags":["search-engine"]},{"pattern":"WebClient","description":"WebClient HTTP utility","tags":["http-library"]},{"pattern":"WebDoc","description":"WebDoc content crawler","tags":["http-library"]},{"pattern":"WebFetch(?!\\w)","description":"WebFetch HTTP client","tags":["http-library"]},{"pattern":"WebFuck","url":"http://www.t0p.xyz","description":"T0P security testing tool","tags":["scanner"]},{"pattern":"WebGazer","url":"https://www.webgazer.io","description":"WebGazer website monitoring bot","tags":["monitoring"]},{"pattern":"WebImages","description":"WebImages crawler","tags":["http-library"]},{"pattern":"WebIndex","description":"WebIndex search bot","tags":["search-engine"]},{"pattern":"WebRankSpider","url":"http://ulm191.server4you.de/crawler/","description":"WebRank spider ranking bot","tags":["seo"]},{"pattern":"WebsiteExtractor","description":"Website extractor tool","tags":["http-library"]},{"pattern":"Wfuzz","url":"https://wfuzz.readthedocs.io/","description":"Wfuzz web fuzzing tool","tags":["scanner"]},{"pattern":"WhereGoes","description":"WhereGoes redirect tracer","tags":["monitoring"]},{"pattern":"Whibse","description":"Whibse crawler","tags":["http-library"]},{"pattern":"Who.is Bot","url":"https://who.is/","description":"Who.is domain information bot","tags":["monitoring"]},{"pattern":"Whoismindbot","url":"http://www.whoismind.com/bot.html","description":"WhoIsMind bot","tags":["monitoring"]},{"pattern":"WikioFeedBot","url":"http://www.wikio.com","description":"Wikio feed crawler","tags":["feed-reader"]},{"pattern":"Willow Internet Crawler","description":"Willow web crawler","tags":["http-library"]},{"pattern":"WillyBot","url":"http://www.willyfogg.com/info/willybot","description":"Willy Fogg travel search bot","tags":["search-engine"]},{"pattern":"[Ww]inHTTP","url":"https://docs.microsoft.com/en-us/windows/win32/winhttp/about-winhttp","description":"Windows HTTP services","tags":["http-library"]},{"pattern":"WinPodder","url":"http://winpodder.com","description":"WinPodder podcast downloader","tags":["feed-reader"]},{"pattern":"WinWebBot","url":"http://www.balaena.com/winwebbot.html","description":"Balaena WinWebBot","tags":["http-library"]},{"pattern":"Windows-RSS-Platform","description":"Windows RSS Platform","tags":["feed-reader"]},{"pattern":"Winds","url":"https://getstream.io/winds/","description":"Winds open source RSS and podcast app","tags":["feed-reader"]},{"pattern":"Woko","description":"Woko crawler","tags":["http-library"]},{"pattern":"Wolfram HTTPClient","url":"https://www.wolfram.com/","description":"Wolfram Mathematica HTTP client","tags":["http-library"]},{"pattern":"WomlpeFactory","url":"http://www.Womple.com/bot.html","description":"Womple crawler factory","tags":["http-library"]},{"pattern":"WordupinfoSearch","description":"Wordupinfo search bot","tags":["search-engine"]},{"pattern":"WorldBrewBot","url":"http://www.marketbrew.com/","description":"MarketBrew SEO bot","tags":["seo"]},{"pattern":"Wtrace","description":"Wtrace HTTP tracer","tags":["monitoring"]},{"pattern":"XING-contenttabreceiver","url":"https://www.xing.com/","description":"XING professional network bot","tags":["social-preview"]},{"pattern":"XRL","url":"http://metamark.net/about","description":"Metamark URL shortener service","tags":["http-library"]},{"pattern":"XenForo","url":"https://www.recreationalflying.com","description":"XenForo forum platform","tags":["http-library"]},{"pattern":"XmlSitemapGenerator","url":"http://xmlsitemapgenerator.org","description":"XML Sitemap Generator tool","tags":["monitoring"]},{"pattern":"XoviBot","url":"https://www.xovi.net/","description":"Xovi SEO tool bot","tags":["seo"]},{"pattern":"YOOBI","description":"YOOBI web bot","tags":["http-library"]},{"pattern":"YOURLS","url":"http://yourls.org/","description":"YOURLS URL shortener","tags":["http-library"]},{"pattern":"YP.PL","url":"https://yp.pl/","description":"YP.PL Polish business directory","tags":["search-engine"]},{"pattern":"Yaanb","url":"http://www.yaanb.com/company/bot.hmtl","description":"Yaanb search engine bot","tags":["search-engine"]},{"pattern":"Yahoo! Site Explorer","url":"http://help.yahoo.com/l/us/yahoo/search/siteexplorer/manage/","description":"Yahoo Site Explorer feed validator","tags":["feed-reader"]},{"pattern":"YandeG","description":"YandeG crawler","tags":["search-engine"]},{"pattern":"Yandex\\/","url":"http://yandex.com/bots","description":"Yandex search engine bot","tags":["search-engine"]},{"pattern":"Yo-yo","description":"Yo-yo crawler","tags":["http-library"]},{"pattern":"YoYs.net","url":"http://yoys.net","description":"YoYs.net crawler","tags":["http-library"]},{"pattern":"Yoleo Consumer","description":"Yoleo consumer bot","tags":["http-library"]},{"pattern":"Zao","url":"http://www.kototoi.org/zao/","description":"Zao web crawler","tags":["http-library"]},{"pattern":"Zemanta Aggregator","url":"http://www.zemanta.com","description":"Zemanta content aggregator","tags":["ai-fetcher"]},{"pattern":"Zend[\\\\/_]Http[\\\\/_]Client","url":"https://framework.zend.com/","description":"Zend Framework HTTP client","tags":["http-library"]},{"pattern":"ZnajdzFoto","url":"http://www.znajdzfoto.pl","description":"ZnajdzFoto image search bot","tags":["search-engine"]},{"pattern":"Zookabot","url":"http://zookabot.com","description":"Zookabot search engine","tags":["search-engine"]},{"pattern":"ZoomInformation Bot","url":"https://www.zoominformation.com/","description":"Zoom Information B2B data bot","tags":["seo"]},{"pattern":"delve.ai","url":"https://delve.ai","description":"Delve AI website analyzer","tags":["ai-fetcher"]},{"pattern":"\\^Nail","url":"http://CaretNail.com","description":"CaretNail crawler","tags":["http-library"]},{"pattern":"a.pr-cy.ru","url":"http://a.pr-cy.ru","description":"Pr-cy rank checker bot","tags":["seo"]},{"pattern":"aboutthedomain","url":"http://aboutthedomain.com","description":"About the Domain crawler","tags":["http-library"]},{"pattern":"facebook","url":"http://www.facebook.com/externalhit_uatext.php","description":"Facebook link preview bot","tags":["social-preview"]},{"pattern":"adreview","description":"AdReview crawler","tags":["advertising"]},{"pattern":"adstxt-worker","url":"http://www.column6.com","description":"Ads.txt crawler service","tags":["advertising"]},{"pattern":"adstxt.com","url":"http://adstxt.com","description":"Ads.txt verification bot","tags":["advertising"]},{"pattern":"affilimate-puppeteer","url":"https://www.affilimate.com/","description":"Affilimate headless crawler","tags":["browser-automation"]},{"pattern":"agentslug.com","url":"http://agentslug.com","description":"AgentSlug website monitoring","tags":["monitoring"]},{"pattern":"akka-http","url":"https://doc.akka.io/docs/akka-http/current/","description":"Akka HTTP Scala client","tags":["http-library"]},{"pattern":"alexa","url":"http://www.openwebspider.org/","description":"Alexa web crawler","tags":["search-engine"]},{"pattern":"annotate_google","url":"http://ponderer.org/download/annotate_google.user.js","description":"Annotate Google user script","tags":["http-library"]},{"pattern":"apimon.de","url":"http://apimon.de/http-apis","description":"API Monitor HTTP API bot","tags":["monitoring"]},{"pattern":"argenfybot","description":"Argenfy crawler","tags":["http-library"]},{"pattern":"aria2","url":"https://aria2.github.io/","description":"aria2 download manager","tags":["http-library"]},{"pattern":"asafaweb.com","url":"http://asafaweb.com","description":"ASAFAweb accessibility checker","tags":["scanner"]},{"pattern":"asynchttp","description":"Async HTTP client","tags":["http-library"]},{"pattern":"attohttpc","description":"AttohttpC Rust HTTP client","tags":["http-library"]},{"pattern":"b0t","description":"B0t crawler","tags":["http-library"]},{"pattern":"baypup","url":"http://www.baypup.com/","description":"Baypup classified listings crawler","tags":["search-engine"]},{"pattern":"biNu image cacher","url":"http://binu.com","description":"biNu image cache service","tags":["http-library"]},{"pattern":"bluefish","url":"http://bluefish.openoffice.nl/","description":"Bluefish HTML editor","tags":["http-library"]},{"pattern":"boitho.com-dc","url":"http://www.boitho.com/dcbot.html","description":"Boitho search engine bot","tags":["search-engine"]},{"pattern":"bot.wsowner.com","url":"http://wsowner.com/","description":"WSOwner web services bot","tags":["monitoring"]},{"pattern":"calculon spider","description":"Calculon yield bot spider","tags":["advertising"]},{"pattern":"cb crawl","url":"http://www.companybooknetworking.com","description":"CompanyBook networking crawler","tags":["seo"]},{"pattern":"censys.io","url":"https://censys.io/","description":"Censys internet census scanner","tags":["scanner"]},{"pattern":"cf-facebook","description":"CloudFlare Facebook SDK","tags":["http-library"]},{"pattern":"cg-eye","description":"CG-Eye interactive monitoring","tags":["monitoring"]},{"pattern":"chatterino-api-cache","url":"https://chatterino.com/","description":"Chatterino API cache link resolver","tags":["http-library"]},{"pattern":"classbot","url":"http://allclasses.com","description":"AllClasses classifier bot","tags":["seo"]},{"pattern":"coccoc","url":"https://coccoc.com/","description":"Coccoc search engine crawler","tags":["search-engine"]},{"pattern":"commonscan.org","url":"http://commonscan.org","description":"CommonScan crawler","tags":["http-library"]},{"pattern":"Licorne Uri Checker","url":"http://www.jednorozec.cz","description":"Licorne Czech URI checker","tags":["monitoring"]},{"pattern":"copyright sheriff","url":"http://www.copyrightsheriff.com/","description":"Copyright Sheriff bot","tags":["scanner"]},{"pattern":"cortex\\/","description":"Cortex crawler","tags":["http-library"]},{"pattern":"crawler for netopian","url":"http://www.netopian.co.uk/","description":"Netopian crawler","tags":["http-library"]},{"pattern":"crazywebcrawler","url":"http://www.crazywebcrawler.com","description":"Crazy Web Crawler","tags":["http-library"]},{"pattern":"curb","url":"http://curb.rubyforge.org/","description":"Curb Ruby HTTP library","tags":["http-library"]},{"pattern":"cuwhois","url":"http://www.cuwhois.com/","description":"CUWHOIS domain checker","tags":["monitoring"]},{"pattern":"dBpoweramp","url":"https://www.dbpoweramp.com/","description":"dBpoweramp audio converter","tags":["http-library"]},{"pattern":"ddline.cn","description":"DDLine rank history tracker","tags":["seo"]},{"pattern":"deeris","url":"http://www.deeris.com","description":"Deeris crawler","tags":["http-library"]},{"pattern":"docker","url":"https://www.docker.com/","description":"Docker container runtime","tags":["http-library"]},{"pattern":"domainsbot","url":"http://www.domainsbot.com","description":"Domainsbot domain crawler","tags":["seo"]},{"pattern":"dubaiindex","description":"DubaiIndex German business crawler","tags":["search-engine"]},{"pattern":"eBot \\/","url":"http://alfa.elchron.cz","description":"eBot Czech search engine","tags":["search-engine"]},{"pattern":"eCatch","description":"eCatch classifier","tags":["http-library"]},{"pattern":"eCommerceBot","url":"http://www.ehandel.se/botinfo.html","description":"eCommerce product crawler","tags":["search-engine"]},{"pattern":"eContext","description":"eContext classification engine","tags":["http-library"]},{"pattern":"endo\\/","url":"http://kula.jp/endo","description":"Endo Mac crawler","tags":["http-library"]},{"pattern":"envolk","url":"http://www.envolk.com/envolkspiderinfo.html","description":"Envolk web spider","tags":["http-library"]},{"pattern":"facebookexternalua","url":"http://www.facebook.com","description":"Facebook external user agent bot","tags":["social-preview"]},{"pattern":"facebookplatform","url":"http://developers.facebook.com","description":"Facebook platform SDK","tags":["http-library"]},{"pattern":"fastbot crawler","url":"http://www.fastbot.de","description":"FastBot crawler","tags":["http-library"]},{"pattern":"fasthttp","url":"https://github.com/valyala/fasthttp","description":"Fasthttp Go HTTP library","tags":["http-library"]},{"pattern":"faviconarchive.org","url":"http://faviconarchive.org","description":"Favicon archive bot","tags":["http-library"]},{"pattern":"faviconkit","url":"https://faviconkit.com","description":"FaviconKit favicon service","tags":["http-library"]},{"pattern":"flatlandbot","url":"http://www.flatlandindustries.com/flatlandbot","description":"Flatland Industries spider","tags":["http-library"]},{"pattern":"focusbot","description":"Focusbot crawler","tags":["http-library"]},{"pattern":"fragFINN.de","url":"http://www.fragfinn.de","description":"FragFINN German kid-safe search bot","tags":["search-engine"]},{"pattern":"gSOAP","url":"https://www.genivia.com/dev.html","description":"gSOAP SOAP library","tags":["http-library"]},{"pattern":"ganarvisitas","url":"http://www.ganarvisitas.com/","description":"GanarVisitas traffic bot","tags":["http-library"]},{"pattern":"gimme60","url":"http://gimme60.com","description":"Gimme60 store ID bot","tags":["seo"]},{"pattern":"git\\/","url":"https://git-scm.com/","description":"Git version control CLI","tags":["http-library"]},{"pattern":"go-mtasts","description":"Go MTA-STS client","tags":["http-library"]},{"pattern":"gobuster","url":"https://github.com/OJ/gobuster","description":"Gobuster directory/DNS brute forcer","tags":["scanner"]},{"pattern":"gobyus","description":"Gobyus crawler","tags":["http-library"]},{"pattern":"gofetch","description":"GoFetch crawler","tags":["http-library"]},{"pattern":"gonzo","url":"http://www.suchen.de/faq.html","description":"Suchen.de Gonzo search bot","tags":["search-engine"]},{"pattern":"gooblog","url":"http://help.goo.ne.jp/contact/","description":"Goo blog indexer","tags":["search-engine"]},{"pattern":"gosquared-thumbnailer","url":"https://www.gosquared.com/","description":"GoSquared thumbnail generator","tags":["ai-fetcher"]},{"pattern":"grokkit","description":"Grokkit Pure Discovery bot","tags":["http-library"]},{"pattern":"gvfs","url":"https://wiki.gnome.org/Projects/gvfs","description":"GNOME virtual filesystem","tags":["http-library"]},{"pattern":"hackney","url":"https://github.com/benoitc/hackney","description":"Hackney Erlang HTTP client","tags":["http-library"]},{"pattern":"hawkReader","url":"http://www.hawkreader.com/","description":"HawkReader feed parser","tags":["feed-reader"]},{"pattern":"hello@knows.is","url":"http://knows.is","description":"Knows.is crawler","tags":["http-library"]},{"pattern":"help@dataminr.com","url":"https://www.dataminr.com/","description":"Dataminr real-time intelligence bot","tags":["ai-crawler"]},{"pattern":"hivaBot","url":"http://yooz.ir","description":"Yooz Iranian search engine bot","tags":["search-engine"]},{"pattern":"hledejLevne.cz","url":"http://hledejlevne.cz","description":"HledejLevne Czech price comparison","tags":["search-engine"]},{"pattern":"holmes","url":"http://morfeo.centrum.cz/bot","description":"Holmes Czech search bot","tags":["search-engine"]},{"pattern":"ht://check","url":"http://www.htdig.org/","description":"ht://check link checker","tags":["monitoring"]},{"pattern":"htdig","url":"http://www.htdig.org/","description":"ht://Dig search engine indexer","tags":["search-engine"]},{"pattern":"htmlayout","url":"http://www.terrainformatica.com","description":"HTMLayout GUI library","tags":["http-library"]},{"pattern":"http-kit","url":"http://http-kit.org/","description":"http-kit Clojure HTTP library","tags":["http-library"]},{"pattern":"http-request","url":"http://git.io/tl_S2w","description":"Node.js HTTP request library","tags":["http-library"]},{"pattern":"http.rb","url":"https://github.com/httprb/http","description":"http.rb Ruby HTTP client","tags":["http-library"]},{"pattern":"arachnode.net","url":"http://arachnode.net","description":"Arachnode web crawler framework","tags":["http-library"]},{"pattern":"IBM SAI Crawler|domino\\.research\\.ibm\\.com\\S*sai-crawler","url":"http://domino.research.ibm.com/comm/research_projects.nsf/pages/sai-crawler.callingcard.html","description":"IBM SAI research crawler","tags":["academic"]},{"pattern":"geek-tools\\.org\\/link-counter|geek-tools link-counter","url":"http://tools.geek-tools.org/link-counter/","description":"Geek Tools link counter","tags":["monitoring"]},{"pattern":"IBM Almaden|almaden\\.ibm\\.com/cs/crawler","url":"http://www.almaden.ibm.com/cs/crawler","description":"IBM Almaden research crawler","tags":["academic"]},{"pattern":"amagit.com","url":"http://www.amagit.com/","description":"Amagit crawler","tags":["http-library"]},{"pattern":"checkprivacy","url":"http://www.checkprivacy.or.kr:6600/RS/PRIVACY_ENFAQ.jsp","description":"Check Privacy bot","tags":["scanner"]},{"pattern":"mabontland","url":"http://www.mabontland.com","description":"Mabontland crawler","tags":["http-library"]},{"pattern":"Nominet|nominet\\.org\\.uk|nominet\\.uk","url":"https://www.nominet.uk/privacy-notice","description":"Nominet domain registry bot","tags":["monitoring"]},{"pattern":"httpRequest","description":"HTTP request utility","tags":["http-library"]},{"pattern":"http_requester","description":"HTTP requester tool","tags":["http-library"]},{"pattern":"getroot.sh survey","url":"https://getroot.sh","description":"GetRoot vulnerability survey bot","tags":["scanner"]},{"pattern":"httpscheck","description":"HTTPS check bot","tags":["monitoring"]},{"pattern":"httpssites_power","description":"HTTPS sites power bot","tags":["monitoring"]},{"pattern":"i2kconnect","description":"i2k connect crawler","tags":["http-library"]},{"pattern":"iZSearch","url":"http://iZSearch.com","description":"iZSearch engine crawler","tags":["search-engine"]},{"pattern":"igdeSpyder","url":"http://igde.ru/doc/tech.html","description":"IGDE Russian search spider","tags":["search-engine"]},{"pattern":"inbound.li parser","url":"http://inbound.li","description":"Inbound link parser","tags":["monitoring"]},{"pattern":"infohelfer","url":"http://www.infohelfer.de/","description":"InfoHelfer German search helper","tags":["search-engine"]},{"pattern":"internetwache.org","url":"http://internetwache.org","description":"InternetWache Swiss crawler","tags":["monitoring"]},{"pattern":"intraVnews","description":"IntraV news aggregator","tags":["feed-reader"]},{"pattern":"ipv6-test.com","url":"http://ipv6-test.com","description":"IPv6 test validator bot","tags":["monitoring"]},{"pattern":"iqdb","url":"http://iqdb.org/","description":"IQDB reverse image search","tags":["ai-fetcher"]},{"pattern":"isUp.li","url":"http://isup.li","description":"IsUp uptime monitoring bot","tags":["monitoring"]},{"pattern":"istellabot","description":"Istella search bot","tags":["search-engine"]},{"pattern":"khttp","url":"https://github.com/khttp/khttp","description":"khttp Kotlin HTTP library","tags":["http-library"]},{"pattern":"kube-probe","url":"https://kubernetes.io/","description":"Kubernetes liveness probe","tags":["monitoring"]},{"pattern":"kubectl","url":"https://kubernetes.io/","description":"Kubernetes command-line tool","tags":["http-library"]},{"pattern":"librabot","url":"http://academic.research.microsoft.com/","description":"Microsoft Academic librabot","tags":["academic"]},{"pattern":"link-check","url":"https://github.com/tcort/link-check","description":"Link checker tool","tags":["monitoring"]},{"pattern":"livedoor ScreenShot","url":"http://livedoor.com","description":"Livedoor screenshot service","tags":["ai-fetcher"]},{"pattern":"lmspider","description":"ScanSoft scanning spider","tags":["scanner"]},{"pattern":"longurl-r-package","url":"https://cran.r-project.org/","description":"R longurl package","tags":["http-library"]},{"pattern":"looid.com Search Engine","url":"http://looid.com","description":"Looid search engine crawler","tags":["search-engine"]},{"pattern":"looksystems.net","url":"http://looksystems.net","description":"Look Systems crawler","tags":["http-library"]},{"pattern":"lua-resty-http","url":"https://github.com/ledgetech/lua-resty-http","description":"Lua OpenResty HTTP library","tags":["http-library"]},{"pattern":"luis@cybo.com","url":"http://cybo.com","description":"Cybo business directory bot","tags":["search-engine"]},{"pattern":"makecontact","url":"https://makecontact.io","description":"Make Contact crawler","tags":["monitoring"]},{"pattern":"marketinggrader.com","url":"http://marketinggrader.com","description":"Marketing Grader bot","tags":["seo"]},{"pattern":"mattermost","url":"https://mattermost.com/","description":"Mattermost chat platform","tags":["http-library"]},{"pattern":"mbot","description":"MBot crawler","tags":["http-library"]},{"pattern":"mfibot","url":"http://www.mfisoft.ru/analyst/","description":"MFI Soft analyst bot","tags":["seo"]},{"pattern":"mio_httpc","description":"Mio HTTP client","tags":["http-library"]},{"pattern":"mogimogi","description":"Mogimogi crawler","tags":["http-library"]},{"pattern":"webmastercoffee","url":"http://webmastercoffee.com/about","description":"WebMaster Coffee bot","tags":["seo"]},{"pattern":"myseosnapshot","description":"MySEO snapshot tool","tags":["seo"]},{"pattern":"netEstate","url":"http://www.netestate.de/","description":"NetEstate German real estate crawler","tags":["search-engine"]},{"pattern":"nettle","url":"https://www.nettle.sk","description":"Nettle Slovak crawler","tags":["http-library"]},{"pattern":"newsme","url":"http://news.me","description":"News.me news aggregator","tags":["feed-reader"]},{"pattern":"nghttp2","url":"https://nghttp2.org/","description":"nghttp2 HTTP/2 library","tags":["http-library"]},{"pattern":"nic.it crawler","url":"http://www.nic.it","description":"NIC.it Italian domain registry bot","tags":["monitoring"]},{"pattern":"node-superagent","url":"https://visionmedia.github.io/superagent/","description":"Node.js superagent HTTP library","tags":["http-library"]},{"pattern":"node-urllib","url":"https://github.com/node-modules/urllib","description":"Node urllib HTTP client","tags":["http-library"]},{"pattern":"node.io","url":"https://github.com/chriso/node.io","description":"Node.io crawler framework","tags":["http-library"]},{"pattern":"nodestackbot","url":"http://nodestack.com/bot.html","description":"NodeStack crawler bot","tags":["http-library"]},{"pattern":"npm","url":"https://www.npmjs.com/","description":"NPM package manager","tags":["http-library"]},{"pattern":"nyawc","url":"https://github.com/Nyaawe/NYAWC","description":"NYAWC Python crawler","tags":["http-library"]},{"pattern":"oegp","description":"OEGP crawler","tags":["http-library"]},{"pattern":"og-scraper","description":"Open Graph scraper","tags":["social-preview"]},{"pattern":"oku-taka-lab-bot","url":"http://lr-www.pi.titech.ac.jp/wp/","description":"Tokyo Tech lab crawler","tags":["academic"]},{"pattern":"orion-semantics.com","description":"Orion Semantics crawler","tags":["http-library"]},{"pattern":"ownCloud","url":"https://owncloud.org/","description":"ownCloud personal cloud platform","tags":["http-library"]},{"pattern":"page_verifier","url":"http://www.securecomputing.com/goto/pv","description":"SecureComputing page verifier","tags":["scanner"]},{"pattern":"peerindex","url":"http://www.peerindex.com/","description":"PeerIndex influence crawler","tags":["seo"]},{"pattern":"percbotspider","url":"http://percolate.com","description":"Percolate bot spider","tags":["http-library"]},{"pattern":"php-requests","url":"https://github.com/rmccue/Requests","description":"PHP Requests HTTP library","tags":["http-library"]},{"pattern":"ping.blo.gs","url":"http://ping.blo.gs","description":"Blog ping service","tags":["monitoring"]},{"pattern":"^pip/[0-9]","url":"https://pip.pypa.io/","description":"Python package installer","tags":["http-library"]},{"pattern":"pnpm","url":"https://pnpm.io/","description":"pnpm package manager","tags":["http-library"]},{"pattern":"polybot","url":"http://cis.poly.edu/polybot/","description":"Polytechnic university crawler","tags":["academic"]},{"pattern":"postano","url":"http://postano.com","description":"Postano social media platform","tags":["social-preview"]},{"pattern":"postfix-mta-sts-resolver","url":"https://github.com/Flyingk/postfix-mta-sts-resolver","description":"Postfix MTA-STS resolver","tags":["http-library"]},{"pattern":"postplanner.com","url":"http://postplanner.com","description":"PostPlanner site scraper","tags":["social-preview"]},{"pattern":"pr-cy.ru Screenshot Bot","url":"http://pr-cy.ru","description":"PR-CY Russian SEO bot","tags":["seo"]},{"pattern":"pshtt","url":"https://github.com/dccnsf/pshtt","description":"PSHTT HTTPS scanner","tags":["scanner"]},{"pattern":"pyspider","url":"http://pyspider.org/","description":"PySpider Python crawler","tags":["http-library"]},{"pattern":"queuedriver","description":"Linqia queue driver","tags":["http-library"]},{"pattern":"quic-go-HTTP","url":"https://github.com/lucas-clemente/quic-go","description":"quic-go HTTP/3 library","tags":["http-library"]},{"pattern":"radian6","url":"http://www.radian6.com/crawler","description":"Radian6 social media crawler","tags":["social-preview"]},{"pattern":"raynette_httprequest","description":"Raynette HTTP request client","tags":["http-library"]},{"pattern":"redback","description":"Redback crawler","tags":["http-library"]},{"pattern":"request.js","url":"https://github.com/request/request","description":"Request.js HTTP library","tags":["http-library"]},{"pattern":"reqwest","url":"https://github.com/seanmonstar/reqwest","description":"Reqwest Rust HTTP client","tags":["http-library"]},{"pattern":"resolver","url":"http://github.com/andris9/resolver","description":"Email resolver utility","tags":["http-library"]},{"pattern":"safe-agent-scanner","url":"http://safe-agent.com","description":"Safe Agent security scanner","tags":["scanner"]},{"pattern":"sc-downloader","url":"http://www.searchconcepts.io","description":"SearchConcepts downloader","tags":["seo"]},{"pattern":"scalaj-http","url":"https://github.com/scalaj/scalaj-http","description":"Scalaj Scala HTTP library","tags":["http-library"]},{"pattern":"scrapy-redis","url":"https://github.com/rolando/scrapy-redis","description":"Scrapy Redis distributed crawler","tags":["http-library"]},{"pattern":"scrapyproject","url":"http://www.profound.net","description":"Scrapy project crawler","tags":["http-library"]},{"pattern":"scrutiny","url":"https://peacockmedia.software/mac/scrutiny/","description":"Scrutiny website analyzer","tags":["seo"]},{"pattern":"search.KumKie.com","url":"http://kumkie.com","description":"KumKie search engine","tags":["search-engine"]},{"pattern":"seegnifybot","url":"http://www.seegnify.com/bot","description":"Seegnify visual bot","tags":["ai-fetcher"]},{"pattern":"semanticdiscovery","description":"Semantic Discovery crawler","tags":["http-library"]},{"pattern":"seo-nastroj.cz","url":"http://seo-nastroj.cz","description":"Czech SEO tool","tags":["seo"]},{"pattern":"shortURL lengthener","url":"http://kevindriscoll.info/","description":"Short URL lengthener","tags":["http-library"]},{"pattern":"silk","url":"http://www.slider.com/silk.htm","description":"Silk slider service","tags":["http-library"]},{"pattern":"sogou spider","url":"http://www.sogou.com","description":"Sogou Chinese search spider","tags":["search-engine"]},{"pattern":"sosoimagespider","url":"http://help.soso.com/soso-image-spider.htm","description":"Soso image search spider","tags":["search-engine"]},{"pattern":"spray-can","url":"https://spray.io/","description":"Spray Scala web framework","tags":["http-library"]},{"pattern":"ssl-tools.net","url":"http://ssl-tools.net","description":"SSL tools checker","tags":["scanner"]},{"pattern":"studylib download bot","url":"http://studylib.es/public/bot","description":"StudyLib document crawler","tags":["ai-fetcher"]},{"pattern":"swcd","description":"SWCD crawler","tags":["http-library"]},{"pattern":"symbolicator","url":"https://github.com/getsentry/symbolicator","description":"Sentry symbolicator service","tags":["http-library"]},{"pattern":"taptubot","url":"http://www.taptu.com/corp/taptubot","description":"Taptu content discovery bot","tags":["social-preview"]},{"pattern":"topicbot","url":"http://www.topic.bot/contact_page.html","description":"Topic bot crawler","tags":["http-library"]},{"pattern":"topster.de HTTP-Header","url":"http://topster.de","description":"Topster German site crawler","tags":["seo"]},{"pattern":"trendspottr.com","url":"http://trendspottr.com","description":"Trendspottr bot","tags":["monitoring"]},{"pattern":"truwoGPS","url":"http://www.lan4lano.net/browser.html","description":"TruwoGPS browser","tags":["http-library"]},{"pattern":"twibble.io","url":"http://twibble.io","description":"Twibble Twitter analysis bot","tags":["social-preview"]},{"pattern":"twine","url":"https://twine.readthedocs.io/","description":"Twine PyPI upload utility","tags":["http-library"]},{"pattern":"uclassify.com","url":"http://uclassify.com/","description":"UClassify text classification","tags":["http-library"]},{"pattern":"ultimate_sitemap_parser","url":"https://github.com/internetarchive/ultimate_sitemap_parser","description":"Sitemap parser utility","tags":["http-library"]},{"pattern":"unfurlist","url":"https://github.com/Doist/unfurlist","description":"Todoist unfurlist link parser","tags":["http-library"]},{"pattern":"unirest-java","url":"http://unirest.io/","description":"Unirest Java HTTP library","tags":["http-library"]},{"pattern":"unshortenit","description":"Unshorten URL utility","tags":["http-library"]},{"pattern":"updated","url":"http://www.updated.com","description":"Updated web crawler","tags":["http-library"]},{"pattern":"ureq","url":"https://github.com/algesten/ureq","description":"Ureq Rust HTTP library","tags":["http-library"]},{"pattern":"urlfan-bot","url":"http://www.urlfan.com/site/bot/350.html","description":"URLFan link tracking bot","tags":["seo"]},{"pattern":"urlresolver","description":"URL resolver utility","tags":["http-library"]},{"pattern":"visionutils","description":"Vision utilities crawler","tags":["http-library"]},{"pattern":"voltron","description":"Voltron crawler","tags":["http-library"]},{"pattern":"voyager","url":"http://www.kosmix.com/crawler.html","description":"Kosmix Voyager crawler","tags":["search-engine"]},{"pattern":"w3af.org","url":"http://w3af.org","description":"W3AF web attack framework","tags":["scanner"]},{"pattern":"w3dt.net httphr","url":"http://w3dt.net","description":"W3DT HTTP response checker","tags":["monitoring"]},{"pattern":"wangling","url":"https://github.com/facebook/wangle","description":"Wangle C++ networking library","tags":["http-library"]},{"pattern":"webcollage","url":"http://www.webcollage.com","description":"WebCollage product feed crawler","tags":["ai-fetcher"]},{"pattern":"webeval.researchready.com","url":"http://webeval.researchready.com","description":"ResearchReady web evaluator","tags":["seo"]},{"pattern":"webinatorbot","url":"http://www.webinator.de","description":"Webinator German search bot","tags":["search-engine"]},{"pattern":"webnumbrFetcher","url":"http://webnumbr.com/","description":"WebNumbr page fetcher","tags":["http-library"]},{"pattern":"webster-internet.de","url":"http://webster-internet.de","description":"Webster internet pad browser","tags":["http-library"]},{"pattern":"webtech","url":"http://webtech.io","description":"WebTech detection tool","tags":["seo"]},{"pattern":"wf_crawler","url":"http://websitefigures.com","description":"Website Figures crawler","tags":["seo"]},{"pattern":"wikiwix-bot","url":"http://www.wikiwix.com","description":"Wikiwix web archive bot","tags":["archiver"]},{"pattern":"worldping-api","url":"http://worldping.io","description":"WorldPing API monitoring","tags":["monitoring"]},{"pattern":"wscheck.com","url":"http://wscheck.com/","description":"Web Services checker","tags":["monitoring"]},{"pattern":"www.adressendeutschland.de","url":"http://www.adressendeutschland.de","description":"German address directory crawler","tags":["search-engine"]},{"pattern":"www.probethenet.com","url":"http://www.probethenet.com","description":"ProbeTheNet security scanner","tags":["scanner"]},{"pattern":"x28-job-bot","url":"http://x28.ch/bot.html","description":"X28 job crawler","tags":["seo"]},{"pattern":"xpymep","description":"XPyMEP Python crawler","tags":["http-library"]},{"pattern":"yomins.com","url":"http://yomins.com","description":"Yomins crawler","tags":["http-library"]},{"pattern":"yoogliFetchAgent","description":"Yoogli fetch agent","tags":["http-library"]},{"pattern":"YRSpider","url":"http://www.yunrang.com/yrspider.html","description":"Yunrang Chinese search spider","tags":["search-engine"]},{"pattern":"NetNewsWire\\/","url":"http://ranchero.com/netnewswire/","description":"NetNewsWire RSS/Atom feed reader for macOS","tags":["feed-reader"]},{"pattern":"SpeedySpider","url":"http://www.entireweb.com","description":"Entireweb search engine crawler","tags":["search-engine"]},{"pattern":"WinHttp-Autoproxy-Service","description":"Windows HTTP auto-proxy discovery service","tags":["http-library"]},{"pattern":"YandexSomething","url":"https://yandex.com/bots","description":"Yandex unspecified bot","tags":["search-engine"]}]