# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism User-agent: *Disallow: /binDisallow: /catalog_temp.txtDisallow: /catrequest.htmlDisallow: /cgi-localDisallow: /collectionDisallow: /comment_temp.txtDisallow: /coupon.htmlDisallow: /graphicsDisallow: /dataDisallow: /hiddenDisallow: /lib32Disallow: /logosDisallow: /movedDisallow: /statsDisallow: /Mailus.htmlDisallow: /ORDER_temp.txtDisallow: /orderform.htmlDisallow: /response.htmlDisallow: /robots.txtDisallow: /Tecra_results.htmlDisallow: /Tecra_search.htmlDisallow: /thanks.htmlDisallow: /welcomeDisallow: /LINKS.html