#robots.txt - rsk 2001/04/12 - disallows spidering of certain directories # - see http://www.go.com/AddUrl?pg=SubmitTips.html#ROBOT User-agent: * # This excludes all crawlers Disallow: /Admin Disallow: /Junk Disallow: /Scripts Disallow: /Download Disallow: /_private Disallow: /_mysql Disallow: /_oldsite