1 files changed, 43 insertions(+), 0 deletions(-)

M static/robots.txt

@@ 2,3 2,46 @@ User-agent: gptbot
 Disallow: /
 User-agent: robots
 Disallow: /
+
+# taken from https://codeberg.org/benjaminhollon/robots.txt-deny-llm/src/branch/main/robots.txt
+
+# from https://neil-clarke.com/block-the-bots-that-feed-ai-models-by-scraping-your-website/
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: Omgili
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+# from https://github.com/healsdata/ai-training-opt-out
+
+# may not work, needs more research (see https://github.com/rom1504/img2dataset/issues/48)
+User-agent: img2dataset
+Disallow: /
+
+User-agent: Claude-Web
+Disallow: /
+
+User-agent: magpie-crawler
+Disallow: /