Web spiders should not be creating forum threads.
This commit is contained in:
parent
69987e08d9
commit
994e31884d
|
@ -1,10 +1,12 @@
|
|||
# prevent bots from signing up for demo accounts
|
||||
# prevent bots from signing up for demo accounts, etc.
|
||||
User-agent: *
|
||||
Disallow: /users/demo
|
||||
Disallow: /users/thanks
|
||||
Disallow: /users/thanks_download
|
||||
Disallow: /notebooks/export_csv/
|
||||
Disallow: /notebooks/export_html/
|
||||
Disallow: /forums/general/create_thread
|
||||
Disallow: /forums/support/create_thread
|
||||
|
||||
# this crawler is completely broken and requests many invalid URLs
|
||||
User-agent: panscient.com
|
||||
|
|
Reference in New Issue