diff --git a/static/html/robots.txt b/static/html/robots.txt index 483e53c..aea46ec 100644 --- a/static/html/robots.txt +++ b/static/html/robots.txt @@ -1,10 +1,12 @@ -# prevent bots from signing up for demo accounts +# prevent bots from signing up for demo accounts, etc. User-agent: * Disallow: /users/demo Disallow: /users/thanks Disallow: /users/thanks_download Disallow: /notebooks/export_csv/ Disallow: /notebooks/export_html/ +Disallow: /forums/general/create_thread +Disallow: /forums/support/create_thread # this crawler is completely broken and requests many invalid URLs User-agent: panscient.com