# This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. User-agent: * Allow: / User-Agent: Baiduspider Disallow: / User-Agent: omgili Disallow: / User-Agent: Slurp Crawl-delay: 20 User-agent: rogerbot Disallow: / User-agent: YandexBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Pinterestbot Disallow: / User-agent: barkrowler Crawl-Delay: 30 # Sitemap Sitemap: https://speelgoedfamilie.nl/sitemap-nl-index.xml Sitemap: https://speelgoedfamilie.nl/sitemap-with-images-index.xml # Allow Directives Allow: */modules/*.css Allow: */modules/*.js Allow: */modules/*.png Allow: */modules/*.jpg Allow: /js/jquery/* # Private pages Disallow: /*?order= Disallow: /*?tag= Disallow: /*?id_currency= Disallow: /*?search_query= Disallow: /*?back= Disallow: /*?n= Disallow: /*&order= Disallow: /*&tag= Disallow: /*&id_currency= Disallow: /*&search_query= Disallow: /*&back= Disallow: /*&n= Disallow: /*controller=addresses Disallow: /*controller=address Disallow: /*controller=authentication Disallow: /*controller=cart Disallow: /*controller=discount Disallow: /*controller=footer Disallow: /*controller=get-file Disallow: /*controller=header Disallow: /*controller=history Disallow: /*controller=identity Disallow: /*controller=images.inc Disallow: /*controller=init Disallow: /*controller=my-account Disallow: /*controller=order Disallow: /*controller=order-slip Disallow: /*controller=order-detail Disallow: /*controller=order-follow Disallow: /*controller=order-return Disallow: /*controller=order-confirmation Disallow: /*controller=pagination Disallow: /*controller=password Disallow: /*controller=pdf-invoice Disallow: /*controller=pdf-order-return Disallow: /*controller=pdf-order-slip Disallow: /*controller=product-sort Disallow: /*controller=search Disallow: /*controller=statistics Disallow: /*controller=attachment Disallow: /*controller=guest-tracking Disallow: /app/ Disallow: /cache/ Disallow: /classes/ Disallow: /config/ Disallow: /controllers/ Disallow: /download/ Disallow: /js/ Disallow: /localization/ Disallow: /log/ Disallow: /mails/ Disallow: /modules/ Disallow: /override/ Disallow: /pdf/ Disallow: /src/ Disallow: /tools/ Disallow: /translations/ Disallow: /upload/ Disallow: /var/ Disallow: /vendor/ Disallow: /webservice/ Disallow: /nl/app/ Disallow: /nl/cache/ Disallow: /nl/classes/ Disallow: /nl/config/ Disallow: /nl/controllers/ Disallow: /nl/download/ Disallow: /nl/js/ Disallow: /nl/localization/ Disallow: /nl/log/ Disallow: /nl/mails/ Disallow: /nl/modules/ Disallow: /nl/override/ Disallow: /nl/pdf/ Disallow: /nl/src/ Disallow: /nl/tools/ Disallow: /nl/translations/ Disallow: /nl/upload/ Disallow: /nl/var/ Disallow: /nl/vendor/ Disallow: /nl/webservice/ Disallow: /cdn-cgi/ # Files Disallow: /wachtwoord-opvragen Disallow: /adres Disallow: /adressen Disallow: /login Disallow: /winkelmandje Disallow: /bestel-overzicht Disallow: /identiteit Disallow: /mijn-account Disallow: /bestelling-volgen Disallow: /bestel-bon Disallow: /bestelling Disallow: /zoeken Disallow: /fabrikanten Disallow: /snel bestellen Disallow: /gast traceren Disallow: /order-bevestiging Disallow: /aanmelden?back=my-account Disallow: /module/iqitwishlist/view Disallow: /winkelmandje?action=show Disallow: /module/xtsearchforalgolia/instantsearch