Add crawl delays

parameter1 · Jul 25, 2024 · eb486d6 · eb486d6
1 parent 38af074
commit eb486d6
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/sites/bizbash.com/server/public/robots.txt b/sites/bizbash.com/server/public/robots.txt
@@ -5,5 +5,11 @@ User-agent: *
 Disallow: /__
 Disallow: /print/content
 
+User-agent: AhrefsBot
+Crawl-Delay: 10
+
+User-agent: DataForSeoBot
+Crawl-Delay: 10
+
 Sitemap: https://www.bizbash.com/sitemap.xml
 Sitemap: https://www.bizbash.com/sitemap-google-news.xml