# Robots.txt #Exclude all robots from part of the server User-agent: * Crawl-delay: 2 Disallow: /cfide/ Disallow: /pbbonus.cfm Disallow: /admin/ Disallow: /audience/ Disallow: /images/ Disallow: /pbrsvp/ Disallow: /community/exercises/ Disallow: /bonus2/ Disallow: /100Days/images/ Disallow: /100Days/today/ Disallow: /100Days/_registrationModule/ # Disallow Collectors and Spam User-agent: atSpider Disallow: / User-agent: cherrypicker Disallow: / User-agent: DSurf Disallow: / User-agent: EliteSys Entry Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Mail Sweeper Disallow: / User-agent: munky Disallow: / User-agent: Roverbot Disallow: / User-agent: WebEmailExtrac Disallow: /