This repository has been archived on 2023-12-16. You can view files and clone it, but cannot push or open issues or pull requests.
2008-11-13 06:44:25 +00:00
|
|
|
# prevent bots from signing up for demo accounts, etc.
|
2007-10-19 22:53:41 +00:00
|
|
|
User-agent: *
|
|
|
|
Disallow: /users/demo
|
2008-01-11 22:55:04 +00:00
|
|
|
Disallow: /users/thanks
|
2008-10-15 19:25:48 +00:00
|
|
|
Disallow: /users/thanks_download
|
|
|
|
Disallow: /notebooks/export_csv/
|
|
|
|
Disallow: /notebooks/export_html/
|
2008-11-13 06:44:25 +00:00
|
|
|
Disallow: /forums/general/create_thread
|
|
|
|
Disallow: /forums/support/create_thread
|
2007-11-07 19:43:09 +00:00
|
|
|
|
|
|
|
# this crawler is completely broken and requests many invalid URLs
|
|
|
|
User-agent: panscient.com
|
|
|
|
Disallow: /
|