# Applies to all robots User-agent: * # Stop these folders from being trawled Disallow: /Umbraco/ Disallow: /UserControls/ Disallow: /Common/Images/ Disallow: /Common/JS/ Disallow: /Common/MasterPages/ Disallow: /Common/Style/ # Stop these files from being trawled Disallow: /Error.aspx Disallow: /FileNotFound.aspx Disallow: /File-Not-Found.aspx Disallow: /Aus/File-Not-Found.aspx Disallow: /Aus/FileNotFound.aspx # Number of secs to wait between successive requests Crawl-delay: 5 # Maximum rate is one page every 5 secs Request-rate: 1/5 # Only visit between 6:00 AM and 8:45 AM UT (GMT) Visit-time: 0600-0845