# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
User-agent: *
Disallow: /client/
Disallow: /banners/
Disallow: /administration/
Disallow: /adverts/search_box/
Disallow: /adverts/phone_num/
Disallow: /bikes/contact/
Disallow: /advert/contact/
Disallow: /baby_kids_toddler/contact/
Disallow: /admin
Disallow: /api
Disallow: /carts
Disallow: /carts/add_variant
Disallow: /subscribe/thank_you
Disallow: /orders/thank_you
Disallow: /competition/thanks

User-Agent: Googlebot
Allow: /api/mobile
Disallow: /api/mobile/orders
Disallow: /api/mobile/user

Sitemap: https://www.bobjane.com.au/sitemap.xml.gz

User-Agent: MJ12bot
Crawl-Delay:   5

User-agent: rogerbot
Crawl-delay: 10

User-Agent:  FacebookBot
Crawl-delay: 5

User-Agent: Yandex
Disallow: /

User-Agent: DealGates Bot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: meanpathbot
Disallow: /

User-agent: wise-guys
Disallow: /