# see http://tool.motoricerca.info/robots-checker.phtml validator # remove from archive.org (wayback machine) User-agent: ia_archiver Disallow: / # remove from aboutUs: User-agent: AboutUsBot Disallow: / # Allow all robots in (DEFAULT) #User-agent: * #Disallow: #Crawl-delay: 30