diff --git a/NEWS b/NEWS index 97129fc..a602afa 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,13 @@ +1.5.1: + * Implemented CSV exporting, so now you can export all of your notes to a + CSV spreadsheet file. This currently doesn't include revision history or + attached files. + * Improved CSV importing so you can export a CSV of your notes from one + Luminotes installation and import that CSV into a different Luminotes + installation. + * Fixed a bug in which the image preview page didn't correctly handle + filenames containing special characters. + 1.5.0: September 12, 2008 * Initial release of Luminotes Desktop! * Fixed a Luminotes Desktop Internet Explorer bug in which note links within diff --git a/config/Common.py b/config/Common.py index 5530415..ee8e358 100644 --- a/config/Common.py +++ b/config/Common.py @@ -132,6 +132,22 @@ settings = { "stream_response": True, "encoding_filter.on": False, }, + "/files/download_product": { + "stream_response": True, + "encoding_filter.on": False, + }, + "/files/thumbnail": { + "stream_response": True, + "encoding_filter.on": False, + }, + "/files/image": { + "stream_response": True, + "encoding_filter.on": False, + }, + "/notebooks/export_csv": { + "stream_response": True, + "encoding_filter.on": False, + }, "/files/upload": { "server.max_request_body_size": 505 * MEGABYTE, # maximum upload size }, diff --git a/controller/Notebooks.py b/controller/Notebooks.py index 423d4c5..5b8be92 100644 --- a/controller/Notebooks.py +++ b/controller/Notebooks.py @@ -1,6 +1,8 @@ import re import cgi +import csv import cherrypy +from cStringIO import StringIO from datetime import datetime from Expose import expose from Validate import validate, Valid_string, Validation_error, Valid_bool, Valid_int @@ -1182,7 +1184,7 @@ class Notebooks( object ): notebook_id = Valid_id(), user_id = Valid_id( none_okay = True ), ) - def download_html( self, notebook_id, user_id ): + def export_html( self, notebook_id, user_id ): """ Download the entire contents of the given notebook as a stand-alone HTML page (no JavaScript). @@ -1191,7 +1193,7 @@ class Notebooks( object ): @type user_id: unicode @param user_id: id of current logged-in user (if any), determined by @grab_user_id @rtype: unicode - @return: rendered HTML page + @return: rendered HTML page with appropriate headers to trigger a download @raise Access_error: the current user doesn't have access to the given notebook @raise Validation_error: one of the arguments is invalid """ @@ -1211,6 +1213,69 @@ class Notebooks( object ): notes = startup_notes + other_notes, ) + @expose() + @weakly_expire + @end_transaction + @grab_user_id + @validate( + notebook_id = Valid_id(), + user_id = Valid_id( none_okay = True ), + ) + def export_csv( self, notebook_id, user_id ): + """ + Download the entire contents of the given notebook as a CSV file. + + @type notebook_id: unicode + @param notebook_id: id of notebook to download + @type user_id: unicode + @param user_id: id of current logged-in user (if any), determined by @grab_user_id + @rtype: unicode + @return: CSV file with appropriate headers to trigger a download + @raise Access_error: the current user doesn't have access to the given notebook + @raise Validation_error: one of the arguments is invalid + """ + if not self.__users.check_access( user_id, notebook_id ): + raise Access_error() + + notebook = self.__database.load( Notebook, notebook_id ) + + if not notebook: + raise Access_error() + + startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() ) + other_notes = self.__database.select_many( Note, notebook.sql_load_non_startup_notes() ) + notes = startup_notes + other_notes + + buffer = StringIO() + writer = csv.writer( buffer ) + + cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=wiki.csv" + cherrypy.response.headerMap[ u"Content-Type" ] = u"text/csv" + + def stream(): + writer.writerow( ( u"contents", u"title", u"note_id", u"startup", u"username", u"revision_date" ) ) + yield buffer.getvalue() + buffer.truncate( 0 ) + + for note in notes: + user = None + if note.user_id: + user = self.__database.load( User, note.user_id ) + + writer.writerow( ( + note.contents.encode( "utf8" ), # TODO: should this try to remove the title? + note.title.encode( "utf8" ), + note.object_id, + note.startup and 1 or 0, + note.user_id and user and user.username.encode( "utf8" ) or u"", + note.revision, + ) ) + + yield buffer.getvalue() + buffer.truncate( 0 ) + + return stream() + @expose( view = Json ) @end_transaction @grab_user_id @@ -1704,7 +1769,8 @@ class Notebooks( object ): @type title_column: int or NoneType @param title_column: zero-based index of the column containing note titles (None indicates the lack of any such column, in which case titles are derived from the - first few words of each note's contents) + first few words of each note's contents if no title is already present + in the note's contents) @type plaintext: bool @param plaintext: True if the note contents are plaintext, or False if they're HTML @type import_button: unicode @@ -1743,11 +1809,13 @@ class Notebooks( object ): if title_column is not None and title_column >= row_length: raise Import_error() - # if there is a title column, use it. otherwise, use the first line of the content column as - # the title + title = None + + # if there is a title column, use it. otherwise, if the note doesn't already contain a title, + # use the first line of the content column as the title if title_column and title_column != content_column and len( row[ title_column ].strip() ) > 0: title = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = plaintext )( row[ title_column ].strip() ) ) - else: + elif plaintext or not Note.TITLE_PATTERN.search( row[ content_column ] ): content_text = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = plaintext )( row[ content_column ].strip() ) ) content_lines = [ line for line in self.NEWLINE_PATTERN.split( content_text ) if line.strip() ] @@ -1769,16 +1837,18 @@ class Notebooks( object ): else: break - contents = u"

%s

%s" % ( - title, - Valid_string( max = 25000, escape_html = plaintext, require_link_target = True )( row[ content_column ] ), - ) + contents = Valid_string( max = 25000, escape_html = plaintext, require_link_target = True )( row[ content_column ] ) if plaintext: contents = contents.replace( u"\n", u"
" ) note_id = self.__database.next_id( Note, commit = False ) note = Note.create( note_id, contents, notebook_id = notebook.object_id, startup = False, rank = None, user_id = user_id ) + + # if the note doesn't have a title yet, then tack the given title onto the start of the contents + if title and note.title is None: + note.contents = u"

%s

%s" % ( title, note.contents ) + self.__database.save( note, commit = False ) # delete the CSV file now that it's been imported diff --git a/controller/test/Test_notebooks.py b/controller/test/Test_notebooks.py index 0449bd2..7f8df51 100644 --- a/controller/test/Test_notebooks.py +++ b/controller/test/Test_notebooks.py @@ -3471,14 +3471,14 @@ class Test_notebooks( Test_controller ): assert result.get( "error" ) - def test_download_html( self ): + def test_export_html( self ): self.login() note3 = Note.create( "55", u"

blah

foo", notebook_id = self.notebook.object_id ) self.database.save( note3 ) result = self.http_get( - "/notebooks/download_html/%s" % self.notebook.object_id, + "/notebooks/export_html/%s" % self.notebook.object_id, session_id = self.session_id, ) assert result.get( "notebook_name" ) == self.notebook.name @@ -3500,11 +3500,11 @@ class Test_notebooks( Test_controller ): previous_revision = note.revision - def test_download_html( self ): + def test_export_html( self ): note3 = Note.create( "55", u"

blah

foo", notebook_id = self.notebook.object_id ) self.database.save( note3 ) - path = "/notebooks/download_html/%s" % self.notebook.object_id + path = "/notebooks/export_html/%s" % self.notebook.object_id result = self.http_get( path, session_id = self.session_id, @@ -3514,11 +3514,11 @@ class Test_notebooks( Test_controller ): assert headers assert headers.get( "Location" ) == u"http:///login?after_login=%s" % urllib.quote( path ) - def test_download_html_with_unknown_notebook( self ): + def test_export_html_with_unknown_notebook( self ): self.login() result = self.http_get( - "/notebooks/download_html/%s" % self.unknown_notebook_id, + "/notebooks/export_html/%s" % self.unknown_notebook_id, session_id = self.session_id, ) @@ -4379,7 +4379,8 @@ class Test_notebooks( Test_controller ): assert note.title == title if plaintext is True: contents = contents.replace( u"\n", u"
" ) - contents = u"

%s

%s" % ( title, contents ) + if plaintext is True or u"

" not in contents: + contents = u"

%s

%s" % ( title, contents ) assert note.contents == contents # make sure the CSV data file has been deleted from the database and filesystem @@ -4390,6 +4391,70 @@ class Test_notebooks( Test_controller ): user = self.database.load( User, self.user.object_id ) assert user.storage_bytes > 0 + def test_import_csv_title_already_in_contents( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","

yay

3.3"\n"8","whee","hmm\n

my title

foo"\n3,4,5' + expected_notes = [ + ( "yay", "

yay

3.3" ), # ( title, contents ) + ( "my title", "hmm\n

my title

foo" ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) + + def test_import_csv_title_already_in_plaintext_contents( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","hi\n

yay

3.3"\n"8","whee","hmm\n

my title

foo"\n3,4,5' + expected_notes = [ + ( "blah and stuff", "hi
<h3>yay</h3>3.3" ), # ( title, contents ) + ( "whee", "hmm
<h3>my title</h3>foo" ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = True ) + def test_import_csv_unknown_file_id( self ): self.login() @@ -4553,6 +4618,70 @@ class Test_notebooks( Test_controller ): self.__assert_imported_notebook( expected_notes, result ) + def test_import_csv_no_title_column_and_title_already_in_contents( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","

yay

3.3"\n"8","whee","hmm\n

my title

foo"\n3,4,5' + expected_notes = [ + ( "yay", "

yay

3.3" ), # ( title, contents ) + ( "my title", "hmm\n

my title

foo" ), + ( "5", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) + + def test_import_csv_no_title_column_and_title_already_in_plaintext_contents( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","hi\n

yay

3.3"\n"8","whee","hmm\n

my title

foo"\n3,4,5' + expected_notes = [ + ( "hi", "hi
<h3>yay</h3>3.3" ), # ( title, contents ) + ( "hmm", "hmm
<h3>my title</h3>foo" ), + ( "5", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = True ) + def test_import_csv_no_title_column_and_html_first_line( self ): self.login() diff --git a/static/css/style.css b/static/css/style.css index 02e5a6d..b76c141 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -580,6 +580,7 @@ h1 { .pulldown_label { color: #000000; + text-decoration: none; } .pulldown_label:hover { diff --git a/static/js/Wiki.js b/static/js/Wiki.js index ca83b59..fcb31e7 100644 --- a/static/js/Wiki.js +++ b/static/js/Wiki.js @@ -340,10 +340,13 @@ Wiki.prototype.populate = function ( startup_notes, current_notes, note_read_wri save_button.disabled = true; } - var download_html_link = getElement( "download_html_link" ); - if ( download_html_link ) { - connect( download_html_link, "onclick", function ( event ) { - self.save_editor( null, true ); + var export_link = getElement( "export_link" ); + if ( export_link ) { + connect( export_link, "onclick", function ( event ) { + self.save_editor( null, true, function () { + self.export_clicked(); + } ); + event.stop(); } ); } @@ -2595,6 +2598,18 @@ Wiki.prototype.zero_total_notes_count = function () { signal( this, "total_notes_count_updated", this.total_notes_count ); } +Wiki.prototype.export_clicked = function () { + var pulldown_id = "export_pulldown"; + var existing_div = getElement( pulldown_id ); + if ( existing_div ) { + existing_div.pulldown.shutdown(); + existing_div.pulldown = null; + return; + } + + new Export_pulldown( this, this.notebook_id, this.invoker, getElement( "export_link" ) ); +} + Wiki.prototype.import_clicked = function () { var pulldown_id = "import_pulldown"; var existing_div = getElement( pulldown_id ); @@ -3358,6 +3373,39 @@ Upload_pulldown.prototype.shutdown = function () { } +function Export_pulldown( wiki, notebook_id, invoker, anchor ) { + Pulldown.call( this, wiki, notebook_id, "export_pulldown", anchor, null, false ); + + this.invoker = invoker; + this.html_link = createDOM( "a", { + "href": "/notebooks/export_html/" + notebook_id, + "class": "pulldown_label", + "title": "Download this notebook as a stand-alone HTML web page." + }, + "HTML web page" + ); + this.csv_link = createDOM( "a", { + "href": "/notebooks/export_csv/" + notebook_id, + "class": "pulldown_label", + "title": "Download this notebook as a CSV spreadsheet file." + }, + "CSV spreadsheet" + ); + + appendChildNodes( this.div, createDOM( "div", {}, this.html_link ) ); + appendChildNodes( this.div, createDOM( "div", {}, this.csv_link ) ); + + Pulldown.prototype.finish_init.call( this ); +} + +Export_pulldown.prototype = new function () { this.prototype = Pulldown.prototype; }; +Export_pulldown.prototype.constructor = Export_pulldown; + +Export_pulldown.prototype.shutdown = function () { + Pulldown.prototype.shutdown.call( this ); +} + + function Import_pulldown( wiki, notebook_id, invoker, anchor ) { anchor.pulldown = this; diff --git a/view/Link_area.py b/view/Link_area.py index 68b4c06..e2022a6 100644 --- a/view/Link_area.py +++ b/view/Link_area.py @@ -32,16 +32,6 @@ class Link_area( Div ): class_ = u"link_area_item", ), - ( notebook.name != u"Luminotes" ) and Div( - A( - u"download as html", - href = u"/notebooks/download_html/%s" % notebook.object_id, - id = u"download_html_link", - title = u"Download a stand-alone copy of the entire wiki notebook.", - ), - class_ = u"link_area_item", - ) or None, - ( rate_plan.get( u"notebook_sharing" ) and notebook.name == u"Luminotes blog" ) and Div( A( u"subscribe to rss", @@ -70,24 +60,41 @@ class Link_area( Div ): class_ = u"link_area_item", ) or None ), + notebook.read_write and Div( + A( + u"nothing but notes", + href = u"#", + id = u"declutter_link", + title = u"Focus on just your notes without any distractions.", + ), + class_ = u"link_area_item", + ) or None, + + ( not notebook.read_write and notebook.name != u"Luminotes" ) and Div( + A( + u"export", + href = u"#", + id = u"export_link", + title = u"Download a stand-alone copy of the entire wiki notebook.", + ), + class_ = u"link_area_item", + ) or None, + notebook.read_write and Span( Div( - A( - u"nothing but notes", - href = u"#", - id = u"declutter_link", - title = u"Focus on just your notes without any distractions.", - ), - class_ = u"link_area_item", - ), - - ( notebook.name != u"Luminotes" ) and Div( A( u"import", href = u"#", id = u"import_link", title = u"Import notes from other software into Luminotes.", ), + u"|", + A( + u"export", + href = u"#", + id = u"export_link", + title = u"Download a stand-alone copy of the entire wiki notebook.", + ), class_ = u"link_area_item", ) or None,