From 8f5c8876a3ed8f3207a8296fdc2f697ceefdd578 Mon Sep 17 00:00:00 2001 From: Giuliano Riccio Date: Mon, 27 Nov 2023 10:52:28 +0100 Subject: [PATCH 1/2] Add Bytespider to bad user agents Crawling bot that doesn't respect the robots.txt rules and continues to make requests even on overloaded servers. Makes requests through Amazon AWS or Bytedance owned network. --- web/bad_user_agents.regex.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/web/bad_user_agents.regex.txt b/web/bad_user_agents.regex.txt index 2ff6cc9e..ab1484b0 100644 --- a/web/bad_user_agents.regex.txt +++ b/web/bad_user_agents.regex.txt @@ -86,6 +86,7 @@ \bBullseye\b \bBunnySlippers\b \bBuzzSumo\b +\bBytespider\b \bCalculon\b \bCATExplorador\b \bCazoodleBot\b From d7a60aad55ce25166282b65e81a6f409030b14db Mon Sep 17 00:00:00 2001 From: Giuliano Riccio Date: Mon, 27 Nov 2023 11:24:51 +0100 Subject: [PATCH 2/2] Add Bytespider to bad user agents Crawling bot that doesn't respect the robots.txt rules and continues to make requests even on overloaded servers. Makes requests through Amazon AWS or Bytedance owned network. --- web/bad_user_agents.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/web/bad_user_agents.txt b/web/bad_user_agents.txt index f3463b92..7200dd15 100644 --- a/web/bad_user_agents.txt +++ b/web/bad_user_agents.txt @@ -86,6 +86,7 @@ BuiltWith Bullseye BunnySlippers BuzzSumo +Bytespider Calculon CATExplorador CazoodleBot