# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file # User-agent: Google-Legal-Removals Allow: / User-agent: Googlebot Allow: /$ Allow: /pages Disallow: /notices Disallow: /faceted_search Disallow: /captcha_gateway User-agent: ia_archiver Allow: / Disallow: /faceted_search Disallow: /captcha_gateway User-agent: * Disallow: / Disallow: /notices Disallow: /faceted_search Disallow: /captcha_gateway Allow: /pages Allow: /$ Crawl-Delay: 86400 # one day