# Make the internet greener
# Reduce your carbon footprint
# Reduce data centre waste
# Reduce the number of search engines and bots from all over the world
# Only allow search engines that matters
# Questions, want to be allowed? Contact me: info@ivanit.nl

# Our default for all search engines is to not allow crawling:
User-agent: *
Disallow: /

# Below are the crawlers that are allowed to crawl this site.
# Below that list, you will find paths that are blocked, even for them,
# and then paths within those blocked paths that are allowed.		

# Google Bot
User-agent: Googlebot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Google adsbot
User-agent: AdsBot-Google
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Google Image
User-agent: Googlebot-Image
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Google Video
User-agent: Googlebot-Video
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Google AdSense
User-agent: Mediapartners-Google
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Bing
User-Agent: bingbot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Duck Duck Go search
User-Agent: DuckDuckBot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# W3C link checker
User-Agent: W3C-checklink
Allow: /

# Morningscore bot
User-agent: Morningscore Bot
Allow: /

# Pinterest bot
User-agent: Pinterestbot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Internet Archive (archive.org) WayBack Machine
User-agent: ia_archiver
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Internet Archive (archive.org) WayBack Machine
User-agent: archive.org_bot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Facebook
User-agent: FacebookExternalHit
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Linkedin
User-agent: LinkedInBot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# WhatsApp
User-agent: WhatsApp
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Twitter
User-agent: Twitterbot
Allow: /
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /xmlrpc.php
Disallow: /wp-includes/
Disallow: /*add-to-cart=*
Disallow: /feed/
Disallow: */feed
Disallow: */feed$
Disallow: /feed/$
Disallow: /comments/feed
Disallow: /?feed=
Disallow: /wp-feed
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to admin section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
Disallow: /tag/ #block access to tag pages
Disallow: /author/ #block access to author pages
Disallow: /404-error/ #block access to 404 page
Disallow: /cart/
Disallow: /winkelmand/
Disallow: /checkout/
Disallow: /afrekenen/
Disallow: /my-account/
Disallow: /mijn-account/
Allow: /wp-includes/css/
Allow: /wp-includes/js/
Allow: /wp-admin/admin-ajax.php

# Block webcrawlers individually because stuborn or not listening to first directive.

# Applebot
User-agent: Applebot
Disallow: /

# Barkrowler
User-agent: barkrowler
Disallow: /

# Baiduspider (my websites are not for Chinese market)
User-agent: Baiduspider
Disallow: /

# BuiltWith
User-agent: BuiltWith
Disallow: /

# Common Crawl CCbot
User-agent: CCBot
Disallow: /

# GeedoProductSearch
User-agent: GeedoProductSearch
Disallow: /

# Imagesiftbot
User-Agent: ImagesiftBot 
Disallow: /

# Serpstatbot
User-agent: SeRanking SEOChecker
Disallow: /

# Serpstatbot
User-agent: serpstatbot
Disallow: /

# SiteChecker
User-agent: SiteCheckerBotCrawler
Disallow: /

# Yahoo slurp (is Yahoo still used?)
User-agent: Slurp
Disallow: /

# Yandex (my websites are not for Russian market)
User-agent: Yandex
Disallow: /

# Zoominfobot
User-agent: Zoominfobot
Disallow: /

#Sitemap
Sitemap: https://caleche-automotive.nl/sitemaps.xml