User-agent: * # applies to all bots, they continue reading
#Disallow: / # applies to all bots, they LEAVE thus removed
Disallow: /cgi-bin/ # robots, stay away from our back-end
Disallow: /images/ # disallow indexing of images, may change later

#Establish allowed bots and area over rides for Google
User-agent: Googlebot
Disallow: # Allows everything EXCEPT the Global set above
User-agent: Googlebot-Image
Disallow: # Allows everything EXCEPT the Global set above
User-agent: Yahoo
Disallow: # Allows everything EXCEPT the Global set above
User-agent: msnbot
Disallow: # Allows everything EXCEPT the Global set above
User-agent: AOL
Disallow: # Allows everything EXCEPT the Global set above
# End of ALLOWED Bots!

# Specific Blocks for known problem Bots or marketing sites
User-agent: Slurp
Disallow: /
User-agent: Teoma
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Scrubby
Disallow: /
User-agent: Robozilla
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: Nutch
Disallow: /

# Last, Block ALL other bots NOT listed above (allowed or blocked)
User-agent: *
Disallow: /

# Moderates Bandwidth during crawls so server doesn't bog down
# slow down Googlebot
User-agent: Googlebot
Crawl-delay: 120
# slow down Yahoo
User-agent: yahoo
Crawl-delay: 120
# slow down MSNbot
User-agent: msnbot
Crawl-delay: 120
# slow down AOLbot
User-agent: AOL
Crawl-delay: 120