~comcloudway/pages

d3cb10ca07795f4c1fbb548a8f3e150b25a08078 — Jakob Meier 8 months ago 5bb2751
Added a bunch of AI crawlers to the robot file so f off
2 files changed, 8 insertions(+), 0 deletions(-)

A res/robots.txt
M templates/page.html
A res/robots.txt => res/robots.txt +4 -0
@@ 0,0 1,4 @@
User-agent: gptbot
Disallow: /
User-agent: robots
Disallow: /

M templates/page.html => templates/page.html +4 -0
@@ 10,6 10,10 @@
    $if(date-meta)$
    <meta name="dcterms.date" content="$date-meta$">
    $endif$
    <meta name="robots" content="noml">
    <meta name="gptbot" content="noml">
    <meta name="bingbot" content="noml">
    <meta name="googlebot" content="noml">
    <title>$if(title-prefix)$$title-prefix$ - $endif$$pagetitle$</title>
    <style type="text/css">code{white-space: pre;}</style>
         $if(quotes)$