# https://ahrefs.com/robot (too much email spam from them)
User-agent: AhrefsBot
Disallow: /

# Seo: Moz
User-agent: rogerbot
Disallow: /

# Seo Moz https://moz.com/help/moz-procedures/crawlers/dotbot
User-agent: dotbot
Disallow: /

# Seo: https://www.deepcrawl.com/
User-agent: deepcrawl
Disallow: /


# Chinese search engine spider so.360.cn
# http://so.360.cn/index.htm
# User-agent: 360Spider

# Google: https://support.google.com/webmasters/answer/1061943?hl=en
# Checks Android web page ad quality
# User-agent: AdsBot-Google
# User-agent: AdsBot-Google-Mobile
# https://support.google.com/webmasters/answer/1061943?hl=en
# User-agent: APIs-Google
# User-agent: Applebot
# User-agent: Googlebot
# User-agent: Googlebot-Image
# User-agent: Googlebot-Mobile
# User-agent: Googlebot-News
# User-agent: Googlebot-Video
# Ad Sense: https://support.google.com/webmasters/answer/1061943
# User-agent: Mediapartners-Google


# User-agent: baiduspider

# User-agent: Bingbot

# User-agent: DuckDuckBot

# User-agent: ia_archiver


# https://developers.facebook.com/docs/sharing/webmasters/crawler
# User-agent: facebookexternalhit/1.1

# User-agent: Lycos

# User-agent: msnbot
# User-agent: msnbot-media

# User-agent: OrangeBot

# User-agent: OrangeBot-Collector

# Yahoo
# User-agent: Slurp

# Search engine
# https://en.wikipedia.org/wiki/Sogou
# User-agent: Sogou

# Russe
# User-agent: StackRambler

# https://en.wikipedia.org/wiki/Teoma
# User-agent: teoma

# A feed fetcher that retrieves details associated with external links displayed on Twitter.
# https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started#url-crawling-and-caching
# User-agent: Twitterbot

# User-agent: Yandex

# User-agent: Mail.RU_Bot

User-agent: *
Disallow: /*?*do=search*

User-agent: *
Disallow: /*?*do=diff*

Sitemap: https://datacadamia.com/doku.php?do=sitemap
Crawl-delay: 3