# All Spiders User-Agent: * # Disallow inappropriate content Disallow: /checkout Disallow: /customer Disallow: /securecheckout # Paths (clean URLs) Disallow: /where-is-my-order/tracking* Disallow: /thelockerroom/wp-login.php* Disallow: /wp-login.php* Disallow: /cgi-cdn* Disallow: /index.php/ Disallow: /catalogsearch/ Disallow: /control/ Disallow: /contacts/ Disallow: /customer/ Disallow: /customize/ Disallow: /newsletter/ Disallow: /poll/ Disallow: /review/ Disallow: /sendfriend/ Disallow: /tag/ Disallow: /catalog/product/gallery/ Disallow: */wishlist/* Disallow: */account/* Disallow: */user/* Disallow: */add/product* Disallow: /stats/ Disallow: /var/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /lib/ Disallow: /downloader/ Disallow: /includes/ Disallow: /wizard/ Disallow: /blog/ Disallow: /skin/frontend/sparta/theme/js/ Disallow: /skin/frontend/sparta/theme/css/ Disallow: /skin/frontend/tpw/theme/js/ Disallow: /skin/frontend/tpw/theme/css/ Disallow: /directory/currency/switch/ Disallow: /catalog/product/upload/option_id/ # Files Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /STATUS.txt Disallow: /*.php$ Disallow: /*?p=*& Disallow: /*?SID= Allow: /magento_ajax_header.php # Disallow: /advancedreviews/ # Disallow: /tpw/ # Disallow: */special-offers/ # Disallow: */when-to-take/ # Disallow: */nutritional-highlights/ # Disallow: */body-goal/ # Disallow: */diet-type/ # Disallow: *?dietary* Disallow: *?flavours* Disallow: *?product_family* Disallow: */choices-lab/* Disallow: /tv/* Disallow: *.pdf # Disallow Internet Archiver Wayback Machine # User-agent: ia_archiver # Disallow: / Sitemap:https://www.theproteinworks.com/sitemap.xml Sitemap:https://www.theproteinworks.com/thelockerroom/sitemap_index.xml User-agent: Linguee Disallow: / User-agent: Yandex Disallow: / User-agent: rogerbot Disallow: / Crawl-delay: 2 User-agent: SemrushBot Crawl-delay:2 User-agent: msnbot Crawl-delay: 10 User-agent: bingbot Disallow: *?92=* Disallow: *?140=* Disallow: *?179=* Disallow: *?180=* Crawl-delay: 10 Sitemap: https://www.theproteinworks.com/sitemap.xml