This repository has been archived on 2023-12-16. You can view files and clone it, but cannot push or open issues or pull requests.
|
# prevent bots from signing up for demo accounts
|
|
User-agent: *
|
|
Disallow: /users/demo
|
|
Disallow: /users/thanks
|
|
Disallow: /notebooks/download_html/
|
|
|
|
# this crawler is completely broken and requests many invalid URLs
|
|
User-agent: panscient.com
|
|
Disallow: /
|