# robots.txt for all sites

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block Ahrefs
User-agent: AhrefsBot
Disallow: /

# Block SEOkicks
User-agent: SEOkicks-Robot
Disallow: /

# Block Uptime robot
#User-agent: UptimeRobot/2.0
#Disallow: /

# Block Ezooms Robot
User-agent: Ezooms Robot
Disallow: /

# Block BlexBot
User-agent: BLEXBot
Disallow: /

# Block Searchmetrics Bot
User-agent: SearchmetricsBot
Disallow: /

# Block YandexBot
User-agent: Yandex
Disallow: /

# Block Baidu
User-agent: Baiduspider
User-agent: Baiduspider/2.0
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

# Block SoGou
User-agent: Sogou Spider
Disallow: /

# Block Youdao
User-agent: YoudaoBot
Disallow: /

# Block  MegaIndex.ru
User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: megaIndex.ru
Disallow: /

User-agent: *
Disallow: /MTADev/ # This is just in case Dev is configured incorrectly
Disallow: /MTADev2/
Disallow: /MTADev3/
Disallow: /*.jsp$
Disallow: /ChildDev/
Disallow: /SKLDev/
Disallow: /ChalkDev/
Disallow: /ChalkNZDev/
Disallow: /WRMicroDev/
Disallow: /RBAUDev/
Disallow: /MTADev/
Disallow: /MTADev2/
Disallow: /MTADev3/
Disallow: /PDDev/
Disallow: /PDNZDev/
Disallow: /WRDev/
Disallow: /KGAUDev/
Disallow: /KGNZDev/
Disallow: /MTAINT/
Disallow: /KescoAUDev/
Disallow: /KescoNZDev/
Disallow: /MBDev/
Disallow: /MTANZDev/
Disallow: /MBNZDev/
Crawl-delay: 30