witten
/
luminotes
Archived
1
0
Fork 0

Web spiders should not be creating forum threads.

This commit is contained in:
Dan Helfman 2008-11-12 22:44:25 -08:00
parent 69987e08d9
commit 994e31884d
1 changed files with 3 additions and 1 deletions

View File

@ -1,10 +1,12 @@
# prevent bots from signing up for demo accounts
# prevent bots from signing up for demo accounts, etc.
User-agent: *
Disallow: /users/demo
Disallow: /users/thanks
Disallow: /users/thanks_download
Disallow: /notebooks/export_csv/
Disallow: /notebooks/export_html/
Disallow: /forums/general/create_thread
Disallow: /forums/support/create_thread
# this crawler is completely broken and requests many invalid URLs
User-agent: panscient.com