Sitemap: https://www.newsbreak.com/sitemap.xml

User-agent: CCBot
Disallow: /

User-agent: CCBot/2.0
Disallow: /

User-agent: CCBot/2.0 (http://commoncrawl.org/faq/)
Disallow: /

User-agent: WikiDo
Disallow: /

User-agent: fr_crawler
Disallow: /

User-agent: Yandex
Disallow: /
User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-favo
Disallow: /

User-agent: Baiduspider-news
Disallow: /

User-agent: Baiduspider-cpro
Disallow: /

User-agent: Baiduspider-ads
Disallow: /

# New crawlers to block 2016
User-Agent: trendictionbot
Disallow: /

User-Agent: Bitvorebot
Disallow: /

User-Agent: BLP_bbot
Disallow: /

User-Agent: heritrix
Disallow: /

User-Agent: magpie-crawler
Disallow: /

User-Agent: Kraken
Disallow: /

User-Agent: moatbot
Disallow: /

User-Agent: bhcBot
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: Synthesio
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-Agent: BrandONbot
Disallow: /

User-Agent: GermCrawler
Disallow: /

User-Agent: Sogou
Disallow: /

User-Agent: Exabot
Disallow: /

User-Agent: MaxPointCrawler
Disallow: /

User-Agent: ADmantX
Disallow: /

User-agent: *
Disallow: /_api/
Disallow: /privacy
Disallow: /terms