10 lines
251 B
Plaintext
10 lines
251 B
Plaintext
# prevent bots from signing up for demo accounts
|
|
User-agent: *
|
|
Disallow: /users/demo
|
|
Disallow: /users/thanks
|
|
Disallow: /notebooks/download_html/
|
|
|
|
# this crawler is completely broken and requests many invalid URLs
|
|
User-agent: panscient.com
|
|
Disallow: /
|