# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# Disallow all user agents
User-agent: *
Disallow: /

# Disable cuil.com
User-agent: twiceler
Disallow: /

# Disable daum.net
User-agent: daumoa
Disallow: /

# Disable baidu
User-agent: Baiduspider
Disallow: /

# Disable dotnetdotcom.org bot
User-agent: dotbot
Disallow: /

# Disable entireweb.com crawler
User-agent: Speedy
Disallow: /

# Disable yandex.ru crawler
User-agent: Yandex
Disallow: /

# Allow specified user agents
# ---------------------------------------- Google
User-agent: Googlebot
# ---------------------------------------- Yahoo
User-agent: Slurp
# ---------------------------------------- Yahoo News
User-agent: yahoo-newscrawler
# ---------------------------------------- Bing
User-agent: msnbot
User-agent: MSNBot
# ---------------------------------------- Ask.com
User-agent: Teoma
# ---------------------------------------- AltaVista
User-agent: Scooter
# ---------------------------------------- AOL
User-agent: ArchitextSpider
# ---------------------------------------- Alexa/Wayback
User-agent: ia_archiver
# ---------------------------------------- Facebook
User-agent: facebookexternalhit/1.0
User-agent: facebookexternalhit
# ---------------------------------------- directives
Disallow: /classifieds
Disallow: /search
Disallow: /user/
Disallow: /images/
Disallow: /uploads/
Disallow: /migrated/
Disallow: /javascripts/
Disallow: /stylesheets/
Disallow: /favicon.ico
# ---------------------------------------- wildcard directives
Disallow: /*?star_filter
Disallow: /*?order
Disallow: /*?sort
Disallow: /*/write_review
Disallow: /*/add_fan
Disallow: /*/remove_fan
Disallow: /place/*/book
Disallow: /place/*/map
Disallow: /place/*/nearby
# ---------------------------------------- events directives
# ---------------------------------------- wildcards SHOULD be copied
Disallow: /events/events:*?star_filter
Disallow: /events/events:*?order
Disallow: /events/events:*?sort
Allow: /events/events:
Disallow: /events/
# ---------------------------------------- limitations
Crawl-delay: 120
# Request-rate: 1/10
# Visit-time: 0000-0700


# Allow Yahoo Pipes
# ---------------------------------------- Yahoo Pipes
User-agent: Yahoo Pipes 1.0
Disallow: /classifieds
Disallow: /search
Disallow: /user/
Disallow: /images/
Disallow: /uploads/
Disallow: /migrated/
Disallow: /javascripts/
Disallow: /stylesheets/
Disallow: /favicon.ico
Crawl-delay: 120

# Allow Webmaster tool
# ---------------------------------------- Webmaster Tool
User-agent: AuditMyPC Webmaster Tool
Disallow: /search
Disallow: /images/
Disallow: /uploads/
Disallow: /migrated/
Disallow: /javascripts/
Disallow: /stylesheets/
Disallow: /favicon.ico
Disallow: /*?star_filter
Disallow: /*?order
Disallow: /*?sort