# robots.txt file for foxriver.net # 260811 # Violations reported and/or firewalled/blocked; # -------------------------------------------------- # "LOOK BUT DON'T LOOT" AI DIRECTIVES # -------------------------------------------------- # Stop Anthropic/Claude from scraping your assets User-agent: ClaudeBot Disallow: / User-agent: Claude-SearchBot Disallow: / # Stop OpenAI/ChatGPT from harvesting your data User-agent: GPTBot Disallow: / User-agent: ChatGPT-User Disallow: / # Stop Google's AI training bots (while keeping standard Google Search) User-agent: Google-Extended Disallow: / User-agent: 007AC9 Disallow: / User-agent: 008 Disallow: / User-agent: 08_800w_web1 Disallow: / User-agent: 123peoplebot Disallow: / User-agent: 8legs Disallow: / User-agent: Aboundex Disallow: / User-agent: Accoona-AI-Agent Disallow: / User-agent: adidxbot Disallow: / User-agent: AdnormCrawler Disallow: / User-agent: AdsBot Disallow: / User-agent: AdsBot-Google Disallow: / User-agent: AdvBot Disallow: / User-agent: AESOP_com_SpiderMan Disallow: / User-agent: aftenie.com Disallow: / User-agent: ah-ha.com Disallow: / User-agent: ah-ha.com crawler Disallow: / User-agent: AhrefsBot Disallow: / User-agent: aiHitBot Disallow: / User-agent: aiHitBot-BP Disallow: / User-agent: aipbot Disallow: / User-agent: AJCrawler Disallow: / User-agent: al_viewer Disallow: / User-agent: Alexa Disallow: / User-agent: Alexibot Disallow: / User-agent: ALittle Client Disallow: / User-agent: Aloofly Disallow: / User-agent: AlphaBot Disallow: / User-agent: Amfibibot Disallow: / User-agent: Amazonbot Disallow: / User-agent: Amzn-SearchBot Disallow: / User-agent: Anarchy99 Disallow: / User-agent: AnswerChase PROve 3.0 Disallow: / User-agent: antbot Disallow: / User-agent: antibot-V1.1 Disallow: / User-agent: AOL_Daniel_Clark_Spider Disallow: / User-agent: Applebot Disallow: / User-agent: Arachnida Web Crawler Disallow: / User-agent: archive.org_bot Disallow: / User-agent: ascribeBot Disallow: / User-agent: asearcher Disallow: / User-agent: ASPSeek Disallow: / User-agent: ASR Ranking Technology/Spider/Crawler Disallow: / User-agent: asterias Disallow: / User-agent: awcheckBot Disallow: / User-agent: axios Disallow: / User-agent: Babya Disallow: / User-agent: BackDoorBot Disallow: / User-agent: BacklinkCrawler Disallow: / User-agent: BaiDuSpider Disallow: / User-agent: Balihoo Disallow: / User-agent: Barkrowler Disallow: / User-agent: baypup Disallow: / User-agent: bbbot Disallow: / User-agent: BeamFall Disallow: / User-agent: BecomeBot Disallow: / User-agent: bemash Disallow: / User-agent: BeNosey Disallow: / User-agent: Bigsearch.ca Disallow: / User-agent: binance.com Disallow: / User-agent: binlar Disallow: / User-agent: Bytespider Disallow: / User-agent: BIXOCRAWLER Disallow: / User-agent: BLABLUBB Disallow: / User-agent: Black Disallow: / User-agent: BlackBerry9000 Disallow: / User-agent: Blekkobot Disallow: / User-agent: BLEXBot Disallow: / User-agent: BlitzBOT Disallow: / User-agent: blogtop.us Disallow: / User-agent: BlowFish Disallow: / User-agent: BlueRobot Disallow: / User-agent: BlueskyClassifierFeatureEngine Disallow: / User-agent: boitho.com-dc Disallow: / User-agent: BoogleBot Disallow: / User-agent: borchuk Disallow: / User-agent: bot.wsowner.com Disallow: / User-agent: BotALot Disallow: / User-agent: BPImageWalker Disallow: / User-agent: BruinBot Disallow: / User-agent: BUbiNG Disallow: / User-agent: BublupBot Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: Bullseye Disallow: / User-agent: Butterfly Disallow: / User-agent: bumblebee Disallow: / User-agent: BunnySlippers Disallow: / User-agent: BusinessBot Disallow: / User-agent: BusinessSeek.biz Disallow: / User-agent: c3.sh Disallow: / User-agent: Cabot Disallow: / User-agent: ca-crawler Disallow: / User-agent: caizhi Disallow: / User-agent: CakePHP Disallow: / User-agent: calculon Disallow: / User-agent: CamontSpider Disallow: / User-agent: CareerBot Disallow: / User-agent: CatchBot Disallow: / User-agent: CATExplorador Disallow: / User-agent: CazoodleBot Disallow: / User-agent: CC-rget Disallow: / User-agent: ccubee Disallow: / User-agent: CCBot Disallow: / User-agent: CensysInspect Disallow: / User-agent: CentiverseBot Disallow: / User-agent: Cegbfeieh Disallow: / User-agent: CFNetwork Disallow: / User-agent: Charlotte Disallow: / User-agent: CheckMarkNetwork Disallow: / User-agent: checks.panopta.com Disallow: / User-agent: CheeseBot Disallow: / User-agent: CherryPicker Disallow: / User-agent: Chilkat Disallow: / User-agent: CipaCrawler Disallow: / User-agent: cipherkey Disallow: / User-agent: CirrusExplorer Disallow: / User-agent: CJNetworkQuality Disallow: / User-agent: Clarabot Disallow: / User-agent: ClariaBot Disallow: / User-agent: classbot Disallow: / User-agent: Claymont.com Disallow: / User-agent: Cliqzbot Disallow: / User-agent: coccoc Disallow: / User-agent: Colly Disallow: / User-agent: Comodo Disallow: / User-agent: Comodo-Webinspector-Crawler Disallow: / User-agent: CompSpyBot Disallow: / User-agent: Computer_and_Automation_Research_Institute_Crawler Disallow: / User-agent: Contacts-Crawler Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: core-project Disallow: / User-agent: cosmos Disallow: / User-agent: cowbot Disallow: / User-agent: cpp-httplib Disallow: / User-agent: CPython Disallow: / User-agent: CrawlBot Disallow: / User-agent: Crawler Disallow: / User-agent: crawler4j Disallow: / User-agent: CRAZYWEBCRAWLER Disallow: / User-agent: Crescent Disallow: / User-agent: CslBot Disallow: / User-agent: CukBot Disallow: / User-agent: curl Disallow: / User-agent: cuwhois Disallow: / User-agent: Dalvik Disallow: / User-agent: Darwin Disallow: / User-agent: DataCha0s Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: DataFountains Disallow: / User-agent: DataparkSearch Disallow: / User-agent: Dataprovider Disallow: / User-agent: Daum Disallow: / User-agent: Daumoa Disallow: / User-agent: DBot Disallow: / User-agent: DBLBot Disallow: / User-agent: DCPBot Disallow: / User-agent: Debian Disallow: / User-agent: del.icio.us-thumbnails Disallow: / User-agent: DeuSu Disallow: / User-agent: DF Disallow: / User-agent: DialogSearch.com Disallow: / User-agent: DiamondBot Disallow: / User-agent: DigExt Disallow: / User-agent: Digincore Disallow: / User-agent: disco Disallow: / User-agent: discobot Disallow: / User-agent: discoverybot Disallow: / User-agent: Dispatch Disallow: / User-agent: DittoSpyder Disallow: / User-agent: dloader Disallow: / User-agent: DnBCrawler-Analytics Disallow: / User-agent: DNSMine Disallow: / User-agent: doccheckbot Disallow: / User-agent: DoCoMo Disallow: / User-agent: DomainAppender Disallow: / User-agent: Domain Disallow: / User-agent: DomainCrawler Disallow: / User-agent: domainsbot Disallow: / User-agent: DomainTaggingbot Disallow: / User-agent: DotBot Disallow: / User-agent: Dow Disallow: / User-agent: DownloadBot Disallow: / User-agent: DTS Disallow: / #User-agent: DuckDuckBot #Disallow: / User-agent: Dulance Disallow: / User-agent: EasyDL Disallow: / User-agent: EC2LinkFinder Disallow: / User-agent: ELB-HealthChecker Disallow: / User-agent: elefent Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Embarcadero Disallow: / User-agent: Embedly Disallow: / User-agent: emefgebot Disallow: / User-agent: emeraldshield.com Disallow: / User-agent: Enolyst Disallow: / User-agent: enrichment-prefilter Disallow: / User-agent: env-checker Disallow: / User-agent: eripp Disallow: / User-agent: EroCrawler Disallow: / User-agent: Eurobot Disallow: / User-agent: Exabot Disallow: / User-agent: exactseek.com Disallow: / User-agent: Exalead Disallow: / User-agent: exooba Disallow: / User-agent: Expanse Disallow: / User-agent: Experibot-v3 Disallow: / User-agent: expo-research-bot Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Ezooms Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: facebookscraper Disallow: / User-agent: FacetEngine Disallow: / User-agent: FAST Disallow: / User-agent: fasthttp Disallow: / User-agent: FatBoyKimComBot Disallow: / User-agent: Faxobot Disallow: / User-agent: FDSE Disallow: / User-agent: feedfetcher Disallow: / User-agent: feedfinder Disallow: / User-agent: Feosey Disallow: / User-agent: Fetcher Disallow: / User-agent: FHScan Disallow: / User-agent: findlinks Disallow: / User-agent: fidget-spinner-bot Disallow: / User-agent: flatlandbot Disallow: / User-agent: FlightDeckReportsBot Disallow: / User-agent: Fluid Disallow: / User-agent: Foobot Disallow: / User-agent: fr-crawler Disallow: / User-agent: FreeWebMonitoring Disallow: / User-agent: fujilabolx5 Disallow: / User-agent: FusionBot Disallow: / User-agent: FyberSpider Disallow: / User-agent: Gaisbot Disallow: / User-agent: GarlikCrawler Disallow: / User-agent: GCore Disallow: / User-agent: GD Disallow: / User-agent: Gecko Disallow: / User-agent: genieBot Disallow: / User-agent: Genrepsoft Disallow: / User-agent: GeoHasher Disallow: / User-agent: GenomeCrawlerd Disallow: / User-agent: gh-index-bot Disallow: / User-agent: giga Disallow: / User-agent: GIGRIB Disallow: / User-agent: Gimme60bot Disallow: / User-agent: GingerCrawler Disallow: / User-agent: Girafabot Disallow: / User-agent: Gluten Disallow: / User-agent: Go Disallow: / User-agent: gobyus Disallow: / User-agent: GOFORITBOT Disallow: / User-agent: goose Disallow: / User-agent: Gowikibot Disallow: / User-agent: GPTBot Disallow: / User-agent: grabber Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: Grasshopper Disallow: / User-agent: great-plains-web-spider Disallow: / User-agent: GRequests Disallow: / User-agent: grub Disallow: / User-agent: Gulliver Disallow: / User-agent: Gulper Disallow: / User-agent: GurujiBot Disallow: / User-agent: GuzzleHttp Disallow: / User-agent: Harvest Disallow: / User-agent: HEADMasterSEO Disallow: / User-agent: HELLO Crawler Disallow: / User-agent: Heretrix Disallow: / User-agent: Heritix Disallow: / User-agent: Hivemind Disallow: / User-agent: hloader Disallow: / User-agent: holmes Disallow: / User-agent: HoneyBee Disallow: / User-agent: HostTracker Disallow: / User-agent: HPL Disallow: / User-agent: HubSpot Disallow: / User-agent: hunter Disallow: / User-agent: htdig Disallow: / User-agent: http-kit Disallow: / User-agent: httplib Disallow: / User-agent: HTTP_Request2 Disallow: / User-agent: humanlinks Disallow: / User-agent: HyPeR Disallow: / User-agent: ia_archiver Disallow: / User-agent: Ibot Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: IconSurf Disallow: / User-agent: Ideare+-+SignSite Disallow: / User-agent: idmarch Disallow: / User-agent: ilial Disallow: / User-agent: Iltrovatore-Setaccio Disallow: / User-agent: imbot Disallow: / User-agent: ImplisenseBot Disallow: / User-agent: IonCrawl Disallow: / User-agent: IncyWincy Disallow: / User-agent: IndeedBot Disallow: / User-agent: Indy Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: InfoSeek Disallow: / User-agent: Insites-scanner Disallow: / User-agent: IntegromeDB Disallow: / User-agent: intelium_bot Disallow: / User-agent: InternetMeasurement Disallow: / User-agent: InternetSeer Disallow: / User-agent: IPiumBot Disallow: / User-agent: ips-agent Disallow: / User-agent: ip-web-crawler.com Disallow: / User-agent: Isidorus Disallow: / User-agent: IstellaBot Disallow: / User-agent: izsearch.com Disallow: / User-agent: Jack Disallow: / User-agent: Jakarta Disallow: / User-agent: Java Disallow: / User-agent: JennyBot Disallow: / User-agent: JetBot Disallow: / User-agent: Jigsaw Disallow: / User-agent: Kavam Disallow: / User-agent: Kenjin Disallow: / User-agent: Keyword Disallow: / User-agent: KickFire Disallow: / User-agent: kmky-not-a-bot Disallow: / User-agent: KnowItAll Disallow: / User-agent: KomodiaBot Disallow: / User-agent: Krugle Disallow: / User-agent: Kryptos Disallow: / User-agent: kuloko Disallow: / User-agent: Kyoto Disallow: / User-agent: Lachesis Disallow: / User-agent: lanshanbot Disallow: / User-agent: Larbin Disallow: / User-agent: LegalX Disallow: / User-agent: LexiBot Disallow: / User-agent: lib Disallow: / User-agent: LightspeedSystemsCrawler Disallow: / User-agent: lincobot Disallow: / User-agent: Linguee Disallow: / User-agent: LinkChecker Disallow: / User-agent: linkdexbot Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: LinkedInBot Disallow: / User-agent: linko Disallow: / User-agent: LinkpadBot Disallow: / User-agent: LinkScan Disallow: / User-agent: LinkStats Disallow: / User-agent: LinkWalker Disallow: / User-agent: Lipperhey Disallow: / User-agent: listfender Disallow: / User-agent: LocalcomBot Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: Lsearch Disallow: / User-agent: LSSRocketCrawler Disallow: / User-agent: ltx71 Disallow: / User-agent: LumeWebScan Disallow: / User-agent: lwp Disallow: / User-agent: lwp-trivial Disallow: / User-agent: Mag Disallow: / User-agent: Magic Disallow: / User-agent: mahonie Disallow: / User-agent: Mail.Ru Disallow: / User-agent: Marvin Disallow: / User-agent: MarkWatch Disallow: / User-agent: masscan Disallow: / User-agent: Mata Disallow: / User-agent: MauiBot Disallow: / User-agent: meanpathbot Disallow: / User-agent: MegaIndex Disallow: / User-agent: memoryBot Disallow: / User-agent: meta-externalagent Disallow: / User-agent: meta-externalfetcher Disallow: / User-agent: metacarta Disallow: / User-agent: MetaIntelligence Disallow: / User-agent: Metaspinner Disallow: / User-agent: mfibot Disallow: / User-agent: MIA Disallow: / User-agent: MIIxpc Disallow: / User-agent: Missigua Disallow: / User-agent: Mister Disallow: / User-agent: MixrankBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: MnoGoSearch Disallow: / User-agent: moget Disallow: / User-agent: MojeekBot Disallow: / User-agent: Mojolicious Disallow: / User-agent: Morfeus Disallow: / User-agent: MOTIbot Disallow: / User-agent: MQBOT Disallow: / User-agent: MSIECrawler Disallow: / User-agent: MSRBOT Disallow: / User-agent: MTRobot Disallow: / User-agent: My User Agent Disallow: / User-agent: MyNutchSpider Disallow: / User-agent: myseosnapshot Disallow: / User-agent: Muenster Disallow: / User-agent: muhstik Disallow: / User-agent: MXT Disallow: / User-agent: myse Disallow: / User-agent: nabot Disallow: / User-agent: NaverBot Disallow: / User-agent: NAMEPROTECT Disallow: / User-agent: NCBot Disallow: / User-agent: NerdyBot Disallow: / User-agent: Neevabot Disallow: / User-agent: NetAnts Disallow: / User-agent: NetcraftSurveyAgent Disallow: / User-agent: netEstate Disallow: / User-agent: NetID.com Disallow: / User-agent: NetMechanic Disallow: / User-agent: NetNoseCrawler Disallow: / User-agent: Net Disallow: / User-agent: newspaper Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: NG Disallow: / User-agent: nicebot Disallow: / User-agent: NimbleCrawler Disallow: / User-agent: Nimbostratus Disallow: / User-agent: NICErsPRO Disallow: / User-agent: nlpproject.info Disallow: / User-agent: node-fetch Disallow: / User-agent: noyona_0_1 Disallow: / User-agent: noxtrumbot Disallow: / User-agent: NPBot Disallow: / User-agent: NRLCorpusBuilder Disallow: / User-agent: nrsbot Disallow: / User-agent: Nutch Disallow: / User-agent: oBot Disallow: / User-agent: Offline Explorer Disallow: / User-agent: OmniExplorer_Bot Disallow: / User-agent: OmvionLeadLake Disallow: / User-agent: OnetSzukaj Disallow: / User-agent: OOZBOT Disallow: / User-agent: openbot Disallow: / User-agent: Openfind Disallow: / User-agent: OpenfosBot Disallow: / User-agent: openwebspider Disallow: / User-agent: OrangeBot Disallow: / User-agent: OrgProbe Disallow: / User-agent: ort Disallow: / User-agent: OutclicksBot Disallow: / User-agent: OutfoxBot Disallow: / User-agent: Owler Disallow: / User-agent: page_prefetcher Disallow: / User-agent: page_test Disallow: / User-agent: PageAnalyzer Disallow: / User-agent: PagePeeker.com Disallow: / User-agent: PagesInventory Disallow: / User-agent: Pagespeed Disallow: / User-agent: PageThing Disallow: / User-agent: Pangalactic Disallow: / User-agent: panscient Disallow: / User-agent: pantest Disallow: / User-agent: Pcore-HTTP Disallow: / User-agent: PEERbot Disallow: / User-agent: Peter Disallow: / User-agent: PHPCrawl Disallow: / User-agent: Pingdom Disallow: / User-agent: PinkPad_Free Disallow: / User-agent: Pinterest Disallow: / User-agent: PiplBot Disallow: / User-agent: Pixray-Seeker Disallow: / User-agent: Plukkie Disallow: / User-agent: Pompos Disallow: / User-agent: PrivacyAwareBot Disallow: / User-agent: prlog Disallow: / User-agent: probethenet Disallow: / User-agent: Program Disallow: / User-agent: Project Disallow: / User-agent: ProPowerBot Disallow: / User-agent: ProWebWalker Disallow: / User-agent: proximic Disallow: / User-agent: psbot Disallow: / User-agent: psycheclone Disallow: / User-agent: puf Disallow: / User-agent: Purebot Disallow: / User-agent: Pycurl Disallow: / User-agent: python-requests Disallow: / User-agent: Python-urllib Disallow: / User-agent: QACC Disallow: / User-agent: QueryN Disallow: / User-agent: Quick-Crawler Disallow: / User-agent: Qwantify Disallow: / User-agent: rabaz Disallow: / User-agent: rabot Disallow: / User-agent: RageComics Disallow: / User-agent: RAMPyBot Disallow: / User-agent: Raven Disallow: / User-agent: Re-re Disallow: / User-agent: ReactorNetty Disallow: / User-agent: REAP Disallow: / User-agent: redback Disallow: / User-agent: RepoMonkey Disallow: / User-agent: research-scanner Disallow: / User-agent: research-spider Disallow: / User-agent: ResearchProject Disallow: / User-agent: rest-client Disallow: / User-agent: RestSharp Disallow: / User-agent: RiceComputerArchitecture Disallow: / User-agent: rico Disallow: / User-agent: Riddler Disallow: / User-agent: RMA Disallow: / User-agent: Robotgenius Disallow: / User-agent: roboto Disallow: / User-agent: rogerbot Disallow: / User-agent: RPT-HTTPClient Disallow: / User-agent: Ruby Disallow: / User-agent: Runet-Research-Crawler Disallow: / User-agent: SafeDNSBot Disallow: / User-agent: SalesIntelligent Disallow: / User-agent: SBIder Disallow: / User-agent: Scanbot Disallow: / User-agent: ScoutAbout Disallow: / User-agent: ScoutJet Disallow: / User-agent: Scrapy Disallow: / User-agent: Screaming Disallow: / User-agent: ScreenerBot Disallow: / User-agent: search.ch Disallow: / User-agent: search.marginalia.nu Disallow: / User-agent: SearchDaimon.com-dc Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: Seekport Disallow: / User-agent: SeekportBot Disallow: / User-agent: sees.co Disallow: / User-agent: semalt Disallow: / User-agent: semanticdiscovery Disallow: / User-agent: SemanticJuice Disallow: / User-agent: SemrushBot Disallow: / User-agent: Sensis Disallow: / User-agent: SEOkicks-Robot Disallow: / User-Agent: SEOlyt Disallow: / User-agent: seoscanners.net Disallow: / User-agent: SERankingBacklinksBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: SETOOZBOT Disallow: / User-agent: SeznamBot Disallow: / User-agent: Sgrunt Disallow: / User-agent: SheenBot Disallow: / User-agent: shelob Disallow: / User-agent: sherlock Disallow: / User-agent: Shim-Crawler Disallow: / User-agent: ShrinkTheWeb.com Disallow: / User-agent: SimilarPages Disallow: / User-agent: SISTRIX Disallow: / User-agent: site-checker Disallow: / User-agent: SiteBot Disallow: / User-agent: sitecheck Disallow: / User-agent: SiteExplorer Disallow: / User-agent: siteintel.net Disallow: / User-agent: SiteLockSpider Disallow: / User-agent: SiteOne-Crawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: Skywalker Disallow: / User-agent: SlySearch Disallow: / User-agent: smart.apnoti.com Disallow: / User-agent: SMEALSearch-Bot Disallow: / User-agent: Snapbot Disallow: / User-agent: SnapPreviewBot Disallow: / User-agent: sogou Disallow: / User-agent: sohu-search Disallow: / User-agent: SolomonoBot Disallow: / User-agent: Sosospider Disallow: / User-agent: spbot Disallow: / User-agent: SpankBot Disallow: / User-agent: spanner Disallow: / User-agent: speedfind Disallow: / User-agent: Speedy Disallow: / User-agent: sphider Disallow: / User-agent: Spijdar Disallow: / User-agent: Sproose Disallow: / User-agent: Spyder Disallow:/ User-agent: Sqworm Disallow: / User-agent: ssearch_bot Disallow: / User-agent: StackRambler Disallow: / User-agent: Steeler Disallow: / User-agent: STINGbot Disallow: / User-agent: Stratagems Disallow: / User-agent: Struct.it Disallow: / User-agent: StructuredWeb Disallow: / User-agent: StumbleUpon Disallow: / User-agent: sufog Disallow: / User-agent: suggybot Disallow: / User-agent: SuperPagesUrlVerifyBot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: SurveyBot Disallow: / User-agent: Symfony Disallow: / User-agent: Synapse Disallow: / User-agent: sysscan Disallow: / User-agent: Szukacz Disallow: / User-agent: Tagoobot Disallow: / User-agent: tags2dir.net Disallow: / User-agent: Taiga Disallow: / User-agent: tbot-nutch Disallow: / User-agent: Teemer Disallow: / User-agent: TeeRaidBot Disallow: / User-agent: Teleport Disallow: / User-agent: Telesoft Disallow: / User-agent: Telesphoreo Disallow: / User-agent: Teradex Disallow: / User-agent: test Disallow: / User-agent: textdigger Disallow: / User-agent: TheKnowledgeAI Disallow: / User-agent: TheNomad Disallow: / User-agent: ThumbSniper Disallow: / User-agent: T-H-U-N-D-E-R-S-T-O-N-E Disallow: / User-agent: TightTwatBot Disallow: / User-agent: Tipiweb Disallow: / User-agent: Titan Disallow: / User-agent: TMCrawler Disallow: / User-agent: toCrawl/UrlDispatcher Disallow: / User-agent: Toplistbot Disallow: / User-agent: Topsy Disallow: / User-agent: TosCrawler Disallow: / User-agent: Toweyabot Disallow: / User-agent: TprAdsTxtCrawler Disallow: / User-agent: trafilatura Disallow: / User-agent: True_Robot Disallow: / User-agent: TScoutBot Disallow: / User-agent: turingos Disallow: / User-agent: TurnitinBot Disallow: / User-agent: TweetmemeBot Disallow: / User-agent: TwengaBot Disallow: / User-agent: Twiceler Disallow: / User-agent: Twitturly Disallow: / User-agent: u12Bot Disallow: / User-agent: UCmore Disallow: / User-agent: UK Disallow: / User-agent: Uptimebot Disallow: / User-agent: URLAppendBot Disallow: / User-agent: URLy Disallow: / User-agent: URL_Spider_SQL Disallow: / User-agent: UXCrawlerBot Disallow: / User-agent: Vagabondo Disallow: / User-agent: Valuationbot Disallow: / User-agent: VCI Disallow: / User-agent: vebidoobot Disallow: / User-agent: verticalman Disallow: / User-agent: verzamelgids.nl Disallow: / User-agent: Virusdie Disallow: / User-agent: VisBot Disallow: / User-agent: VisiStat Disallow: / User-agent: vision Disallow: / User-agent: visual Disallow: / User-agent: VoilaBot Disallow: / User-agent: voltron Disallow: / User-agent: voyager Disallow: / User-agent: VSE Disallow: / User-agent: vuhuvBot Disallow: / User-agent: warc Disallow: / User-agent: Waterfall Disallow: / User-agent: wauuu Disallow: / User-agent: wbdbot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: Web Disallow: / User-agent: Wegtam Disallow: / User-agent: Wells Disallow: / User-agent: Wget Disallow: / User-agent: WikiDo Disallow: / User-agent: WinInet Disallow: / User-agent: WhatWeb Disallow: / User-agent: whiteiexpres Disallow: / User-agent: whizbang Disallow: / User-agent: Who.is Disallow: / User-agent: wonderbot Disallow: / User-agent: woriobot Disallow: / User-agent: WorldLight Disallow: / User-agent: Wotbot Disallow: / User-agent: Wotbox Disallow: / User-agent: wsr-agent Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: WWW-Mechanize Disallow: / User-agent: www.integromedb.org Disallow: / User-agent: www.probethenet.com Disallow: / User-agent: www.socialayer.com Disallow: / User-agent: wwwster Disallow: / User-agent: wwwtype Disallow: / User-agent: WyoBot Disallow: / User-agent: Xenu Disallow: / User-agent: XoviBot Disallow: / User-agent: xpymep.exe Disallow: / User-agent: Y!J-ASR Disallow: / User-agent: yacybot Disallow: / User-agent: Yahoo! Slurp China Disallow: / User-agent: YahooMobileMessenger Disallow: / User-agent: Yandex Disallow: / User-agent: Yanga Disallow: / User-agent: Yeti Disallow: / User-agent: YisouSpider Disallow: / User-agent: YodaoBot Disallow: / User-agent: Yowedo Disallow: / User-agent: YPBot Disallow: / User-agent: ZaldomoSearchBot Disallow: / User-agent: Zauba Disallow: / User-agent: ZemlyaCrawl Disallow: / User-agent: Zeus Disallow: / User-Agent: zgrab Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: ZSEBOT Disallow: / User-agent: ZyBorg Disallow: / # -------------------------------------------------- # GLOBAL DEFAULT FOR STANDARD VISITORS / NORMAL BOTS # -------------------------------------------------- User-agent: * Crawl-delay: 5 Disallow: