fix: update list of crawlers

openfoodfacts · Aug 23, 2023 · d5f5fc0 · d5f5fc0
1 parent 8d12768
commit d5f5fc0
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/app/main.py b/app/main.py
@@ -72,7 +72,9 @@
     r"Googlebot|Googlebot-Image|bingbot|Applebot|YandexBot|"
     r"YandexRenderResourcesBot|DuckDuckBot|DotBot|SeekportBot|AhrefsBot|"
     r"DataForSeoBot|SeznamBot|ZoomBot|MojeekBot|QRbot|www\.qwant\.com|"
-    r"facebookexternalhit"
+    r"facebookexternalhit|Bytespider|GPTBot|SEOkicks-Robot|SearchmetricsBot"
+    r"MJ12bot|SurveyBot|SEOdiver|wotbox|Cliqzbot|Paracrawl|VelenPublicWebCrawler"
+    r"SemrushBot|MegaIndex\.ru|YandexMarket|Amazonbot"
 )