# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
User-agent: Google-Legal-Removals
Allow: /
User-agent: Googlebot
Allow: /$
Allow: /pages
Disallow: /notices
Disallow: /faceted_search
Disallow: /captcha_gateway
User-agent: ia_archiver
Allow: /
Disallow: /faceted_search
Disallow: /captcha_gateway
User-agent: *
Disallow: /
Disallow: /notices
Disallow: /faceted_search
Disallow: /captcha_gateway
Allow: /pages
Allow: /$
Crawl-Delay: 86400 # one day