diff --git a/public/robots.txt b/public/robots.txt index cccfa58f2..809f570d6 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,9 +1,9 @@ # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -User-agent: GPTBot -Disallow: / - -User-agent: * +User-agent: archive.org_bot Disallow: /media_proxy/ Disallow: /interact/ Disallow: /api/v1/instance/domain_blocks + +User-agent: * +Disallow: /