Prevent web crawlers from spidering the "download as html" version of a notebook.
This commit is contained in:
parent
e7ae481eea
commit
c4a2cf6713
|
@ -1,6 +1,7 @@
|
||||||
# prevent bots from signing up for demo accounts
|
# prevent bots from signing up for demo accounts
|
||||||
User-agent: *
|
User-agent: *
|
||||||
Disallow: /users/demo
|
Disallow: /users/demo
|
||||||
|
Disallow: /notebooks/download_html/
|
||||||
|
|
||||||
# this crawler is completely broken and requests many invalid URLs
|
# this crawler is completely broken and requests many invalid URLs
|
||||||
User-agent: panscient.com
|
User-agent: panscient.com
|
||||||
|
|
Reference in New Issue