14 lines
401 B
Plaintext
14 lines
401 B
Plaintext
# prevent bots from signing up for demo accounts, etc.
|
|
User-agent: *
|
|
Disallow: /users/demo
|
|
Disallow: /users/thanks
|
|
Disallow: /users/thanks_download
|
|
Disallow: /notebooks/export_csv/
|
|
Disallow: /notebooks/export_html/
|
|
Disallow: /forums/general/create_thread
|
|
Disallow: /forums/support/create_thread
|
|
|
|
# this crawler is completely broken and requests many invalid URLs
|
|
User-agent: panscient.com
|
|
Disallow: /
|