From c4a2cf6713e69b77f55f8352165acc1868df0c45 Mon Sep 17 00:00:00 2001
From: Dan Helfman <dan@luminotes.com>
Date: Wed, 2 Jan 2008 03:27:31 +0000
Subject: [PATCH] Prevent web crawlers from spidering the "download as html"
 version of a notebook.

---
 static/html/robots.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/static/html/robots.txt b/static/html/robots.txt
index 8c4f43e..b79cb91 100644
--- a/static/html/robots.txt
+++ b/static/html/robots.txt
@@ -1,6 +1,7 @@
 # prevent bots from signing up for demo accounts
 User-agent: *
 Disallow: /users/demo
+Disallow: /notebooks/download_html/
 
 # this crawler is completely broken and requests many invalid URLs
 User-agent: panscient.com