# References:
# https://en.wikipedia.org/wiki/Robots.txt
# https://support.google.com/webmasters/answer/1061943?hl=en
# https://blogs.bing.com/webmaster/2012/05/03/to-crawl-or-not-to-crawl-that-is-bingbots-question/
# https://www.semrush.com/blog/beginners-guide-robots-txt/

Sitemap: https://www.sitepoint.com/sitemap-index.xml
Sitemap: https://www.sitepoint.com/community/sitemap.xml
Sitemap: https://www.sitepoint.com/premium/sitemap.xml

User-agent: *
Allow: /ads.txt
# Blog Related Examples
Disallow: /examples/
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/cert/
# Discourse-related
Disallow: */community/*?*api_key*
Disallow: */community/*?api_key*
Disallow: */community/admin/*
Disallow: */community/assets/browser-update*.js
Disallow: */community/auth/*
Disallow: */community/auth/cas*
Disallow: */community/auth/cas/callback*
Disallow: */community/auth/facebook/callback*
Disallow: */community/auth/github/callback*
Disallow: */community/auth/google/callback*
Disallow: */community/auth/twitter/callback*
Disallow: */community/auth/yahoo/callback*
Disallow: */community/badges/*
Disallow: */community/c/*.rss
Disallow: */community/email/*
Disallow: */community/groups/*
Disallow: */community/my*
Disallow: */community/search*
Disallow: */community/session/*
Disallow: */community/t/*/*.rss
Disallow: */community/tag/*/l*
Disallow: */community/tags*
Disallow: */community/tags/*
Disallow: */community/tags/*.rss
Disallow: */community/u/*
Disallow: */community/user-api-key/*
Disallow: */community/users/*
# Unbounce-related
Disallow: /*/clkg/*
Disallow: /*/clkn/*
# S3 replication source content
Disallow: *.json
Disallow: *.php
Disallow: *.srt
Disallow: */premium/search/*
Disallow: */us-west-2/*
Disallow: *?*no_definitions*=*
Disallow: *?*no_subcategories*=*
Disallow: *?TB_iframe*=*
Disallow: *?from*=*
Disallow: *?gclid*=*
Disallow: *?gp*
Disallow: *?period*=*
Disallow: *?publisher*=*
Disallow: *?q*=*
Disallow: *?replytocom*=*
Disallow: *?source*=*
Disallow: *?time*=*
Disallow: *?topic*=*
Disallow: *?u*=*
Disallow: *utm*=*
Allow: */forums/*
Allow: */premium/*

# Adds a Crawl-delay of 30 to bingbot. Otherwise, identical to the
# above. Requested by Discourse.
User-agent: bingbot
Crawl-delay: 30
Allow: /ads.txt
# Blog Related Examples
Disallow: /examples/
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/cert/
# Discourse-related
Disallow: */community/*?*api_key*
Disallow: */community/*?api_key*
Disallow: */community/admin/*
Disallow: */community/assets/browser-update*.js
Disallow: */community/auth/*
Disallow: */community/auth/cas*
Disallow: */community/auth/cas/callback*
Disallow: */community/auth/facebook/callback*
Disallow: */community/auth/github/callback*
Disallow: */community/auth/google/callback*
Disallow: */community/auth/twitter/callback*
Disallow: */community/auth/yahoo/callback*
Disallow: */community/badges/*
Disallow: */community/c/*.rss
Disallow: */community/email/*
Disallow: */community/groups/*
Disallow: */community/my*
Disallow: */community/search*
Disallow: */community/session/*
Disallow: */community/t/*/*.rss
Disallow: */community/tag/*/l*
Disallow: */community/tags*
Disallow: */community/tags/*
Disallow: */community/tags/*.rss
Disallow: */community/u/*
Disallow: */community/user-api-key/*
Disallow: */community/users/*
# Unbounce-related
Disallow: /*/clkg/*
Disallow: /*/clkn/*
# S3 replication source content
Disallow: *.json
Disallow: *.php
Disallow: *.srt
Disallow: */premium/search/*
Disallow: */us-west-2/*
Disallow: *?*no_definitions*=*
Disallow: *?*no_subcategories*=*
Disallow: *?TB_iframe*=*
Disallow: *?from*=*
Disallow: *?gclid*=*
Disallow: *?gp*
Disallow: *?period*=*
Disallow: *?publisher*=*
Disallow: *?q*=*
Disallow: *?replytocom*=*
Disallow: *?source*=*
Disallow: *?time*=*
Disallow: *?topic*=*
Disallow: *?u*=*
Disallow: *utm*=*
Allow: */forums/*
Allow: */premium/*

# Prevents SemRush from abusing our search page, and adds a crawl delay.
User-agent: SemrushBot
Crawl-delay: 30
Allow: /ads.txt
# Blog Related Examples
Disallow: /examples/
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/cert/
# Discourse-related
Disallow: */community/*?*api_key*
Disallow: */community/*?api_key*
Disallow: */community/admin/*
Disallow: */community/assets/browser-update*.js
Disallow: */community/auth/*
Disallow: */community/auth/cas*
Disallow: */community/auth/cas/callback*
Disallow: */community/auth/facebook/callback*
Disallow: */community/auth/github/callback*
Disallow: */community/auth/google/callback*
Disallow: */community/auth/twitter/callback*
Disallow: */community/auth/yahoo/callback*
Disallow: */community/badges/*
Disallow: */community/c/*.rss
Disallow: */community/email/*
Disallow: */community/groups/*
Disallow: */community/my*
Disallow: */community/search*
Disallow: */community/session/*
Disallow: */community/t/*/*.rss
Disallow: */community/tag/*/l*
Disallow: */community/tags*
Disallow: */community/tags/*
Disallow: */community/tags/*.rss
Disallow: */community/u/*
Disallow: */community/user-api-key/*
Disallow: */community/users/*
# Unbounce-related
Disallow: /*/clkg/*
Disallow: /*/clkn/*
# S3 replication source content
Disallow: *.json
Disallow: *.php
Disallow: *.srt
Disallow: */premium/search/*
Disallow: */us-west-2/*
Disallow: *?*no_definitions*=*
Disallow: *?*no_subcategories*=*
Disallow: *?TB_iframe*=*
Disallow: *?from*=*
Disallow: *?gclid*=*
Disallow: *?gp*
Disallow: *?period*=*
Disallow: *?publisher*=*
Disallow: *?q*=*
Disallow: *?replytocom*=*
Disallow: *?source*=*
Disallow: *?time*=*
Disallow: *?topic*=*
Disallow: *?u*=*
Disallow: *utm*=*
Allow: */forums/*
Allow: */premium/*