# Robots.txt for https://www.businessgreen.com
# Updated 12th July 2017 by JH
# Version 1.10 - https declarations

# Sitemap declarations
User-agent: *
Sitemap: https://www.businessgreen.com/sitemap_index.xml
Sitemap: https://www.businessgreen.com/news-sitemap.xml
Sitemap: https://www.businessgreen.com/videositemap.xml
Crawl-delay: 20
Disallow: /search
Disallow: /feeds/rss/search
Disallow: /4818/
Disallow: /print-article/
Disallow: /digital_assets/
Disallow: /v3_ie8_meta.html
Disallow: /send-to-friend/
Disallow: /tag/page/
Disallow: /pdf/
Disallow: /blog?month
Disallow: /home/show_comment_page/

# Agent specific disallowed sections
# Googlebot
User-agent: Googlebot
Sitemap: https://www.businessgreen.com/sitemap_index.xml
Sitemap: https://www.businessgreen.com/news-sitemap.xml
Sitemap: https://www.businessgreen.com/videositemap.xml
Disallow: /search
Disallow: /feeds/rss/search
Disallow: /4818/
Disallow: /print-article/
Disallow: /digital_assets/
Disallow: /v3_ie8_meta.html
Disallow: /send-to-friend/
Disallow: /tag/page/
Disallow: /pdf/
Disallow: /blog?month
Disallow: /home/show_comment_page/

# Bingbot
User-agent: bingbot
Sitemap: https://www.businessgreen.com/sitemap_index.xml
Sitemap: https://www.businessgreen.com/news-sitemap.xml
Sitemap: https://www.businessgreen.com/videositemap.xml
Crawl-delay: 1
Disallow: /search
Disallow: /feeds/rss/search
Disallow: /4818/
Disallow: /print-article/
Disallow: /digital_assets/
Disallow: /v3_ie8_meta.html
Disallow: /send-to-friend/
Disallow: /tag/page/
Disallow: /pdf/
Disallow: /blog?month
Disallow: /home/show_comment_page/

# Yahoo
User-agent: Yahoo Slurp!
Sitemap: https://www.businessgreen.com/sitemap_index.xml
Sitemap: https://www.businessgreen.com/news-sitemap.xml
Sitemap: https://www.businessgreen.com/videositemap.xml
Crawl-delay: 1
Disallow: /search
Disallow: /feeds/rss/search
Disallow: /4818/
Disallow: /print-article/
Disallow: /digital_assets/
Disallow: /v3_ie8_meta.html
Disallow: /send-to-friend/
Disallow: /tag/page/
Disallow: /pdf/
Disallow: /blog?month
Disallow: /home/show_comment_page/

#Ad serving - allow adsense access
User-agent: Mediapartners-Google
Disallow:

# Fully exclude these robots from crawling anything
User-agent: AhrefsBot
User-agent: CompSpyBot
User-agent: CrystalSemanticsBot
User-agent: Curious George
User-agent: CybEye.com
User-agent: daumoa
User-agent: DoCoMo
User-agent: ExB Language Crawler
User-agent: Ezooms
User-agent: Flamingo_SearchEngine
User-agent: Genieo
User-agent: Genio
User-agent: gsa-crawler
User-agent: LexxeBot
User-agent: libcrawl
User-agent: linkdex
User-agent: LWNutch
User-agent: magpie-crawler
User-agent: Meltwater
User-agent: MnoGoSearch
User-agent: omgilibot/0.3
User-agent: OpenWebIndex
User-agent: psbot
User-agent: RediffNewsBot
User-agent: repparser
User-agent: Scanmine
User-agent: Screaming Frog SEO Spider
User-agent: SEOENGWorldBot
User-agent: ShopWiki
User-agent: ShowyouBot
User-agent: sindice-site-manager
User-agent: sogou
User-agent: sogou spider
User-agent: Sosospider
User-agent: WebVac
User-agent: WocBot
User-agent: woriobot
User-agent: yacybot
User-agent: Yeti
User-agent: yolinkBot_text
User-agent: YoudaoBot
Disllow: /