# BEGIN Limit crawl rate to no more than 1 page every 5 seconds) User-agent: * Crawl-delay: 20 # END SET CRAWL RATE