From 26722eee93f4ad5021f3a630363f2a519c8b1767 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 18 Feb 2009 23:52:18 -0800 Subject: [PATCH] Converting export features to work as separate plugins. Made exported filenames based on the name of the exported notebook. --- NEWS | 7 +- config/Common.py | 2 +- controller/Expose.py | 21 +++- controller/Notebooks.py | 91 ++++---------- controller/test/Test_notebooks.py | 53 ++------ plugins/Invoke.py | 10 ++ plugins/__init__.py | 0 plugins/export_csv/__init__.py | 38 ++++++ plugins/export_csv/test/Test_export_csv.py | 122 +++++++++++++++++++ {view => plugins/export_html}/Html_file.py | 10 +- plugins/export_html/__init__.py | 14 +++ plugins/export_html/test/Test_export_html.py | 75 ++++++++++++ static/js/Wiki.js | 4 +- 13 files changed, 324 insertions(+), 123 deletions(-) create mode 100644 plugins/Invoke.py create mode 100644 plugins/__init__.py create mode 100644 plugins/export_csv/__init__.py create mode 100644 plugins/export_csv/test/Test_export_csv.py rename {view => plugins/export_html}/Html_file.py (85%) create mode 100644 plugins/export_html/__init__.py create mode 100644 plugins/export_html/test/Test_export_html.py diff --git a/NEWS b/NEWS index 81c10bf..25596f7 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,10 @@ 1.6.7: ? - * + * When you export your notebook as an HTML or CSV file, the saved filename is + now based on the name of your notebook, for instance "my-to-do-list.html" + or "ideas-for-my-novel.csv". + * Converted the existing HTML and CSV export features to work as separate + export plugins. This means that a new export format can be implemented as + a new plugin. 1.6.6: February 16, 2009 * Luminotes now recognizes "mailto:" links as external links, so you can diff --git a/config/Common.py b/config/Common.py index 83789eb..6883c88 100644 --- a/config/Common.py +++ b/config/Common.py @@ -143,7 +143,7 @@ settings = { "stream_response": True, "encoding_filter.on": False, }, - "/notebooks/export_csv": { + "/notebooks/export": { "stream_response": True, "encoding_filter.on": False, }, diff --git a/controller/Expose.py b/controller/Expose.py index b257443..487054a 100644 --- a/controller/Expose.py +++ b/controller/Expose.py @@ -70,18 +70,31 @@ def expose( view = None, rss = None ): if isinstance( result, types.GeneratorType ): return result - redirect = result.get( u"redirect", None ) + redirect = result.get( u"redirect" ) + encoding = result.get( u"manual_encode" ) + if encoding: + del( result[ u"manual_encode" ] ) + + def render( view, result, encoding = None ): + output = unicode( view( **result ) ) + if not encoding: + return output + return output.encode( encoding ) # try using the supplied view to render the result try: if view_override is None: if rss and use_rss: cherrypy.response.headers[ u"Content-Type" ] = u"application/xml" - return unicode( rss( **result ) ).encode( "utf8" ) + return render( rss, result, encoding or "utf8" ) elif view: - return unicode( view( **result ) ) + return render( view, result, encoding ) + elif result.get( "view" ): + result_view = result.get( "view" ) + del( result[ "view" ] ) + return render( result_view, result, encoding ) else: - return unicode( view_override( **result ) ) + return render( view_override, result, encoding ) except: if redirect is None: if original_error: diff --git a/controller/Notebooks.py b/controller/Notebooks.py index b22e023..3ae0dd9 100644 --- a/controller/Notebooks.py +++ b/controller/Notebooks.py @@ -1,8 +1,6 @@ import re import cgi -import csv import cherrypy -from cStringIO import StringIO from datetime import datetime from Expose import expose from Validate import validate, Valid_string, Validation_error, Valid_bool, Valid_int @@ -21,7 +19,6 @@ from model.File import File from model.Tag import Tag from view.Main_page import Main_page from view.Json import Json -from view.Html_file import Html_file from view.Note_tree_area import Note_tree_area from view.Notebook_rss import Notebook_rss from view.Updates_rss import Updates_rss @@ -1191,61 +1188,33 @@ class Notebooks( object ): notes = notes, ) - @expose( view = Html_file ) - @weakly_expire - @end_transaction - @grab_user_id - @validate( - notebook_id = Valid_id(), - user_id = Valid_id( none_okay = True ), - ) - def export_html( self, notebook_id, user_id ): - """ - Download the entire contents of the given notebook as a stand-alone HTML page (no JavaScript). - - @type notebook_id: unicode - @param notebook_id: id of notebook to download - @type user_id: unicode - @param user_id: id of current logged-in user (if any), determined by @grab_user_id - @rtype: unicode - @return: rendered HTML page with appropriate headers to trigger a download - @raise Access_error: the current user doesn't have access to the given notebook - @raise Validation_error: one of the arguments is invalid - """ - notebook = self.__users.load_notebook( user_id, notebook_id ) - - if not notebook: - raise Access_error() - - startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() ) - other_notes = self.__database.select_many( Note, notebook.sql_load_non_startup_notes() ) - - return dict( - notebook_name = notebook.name, - notes = startup_notes + other_notes, - ) - @expose() @weakly_expire @end_transaction @grab_user_id @validate( notebook_id = Valid_id(), + format = Valid_string( min = 1, max = 100 ), user_id = Valid_id( none_okay = True ), ) - def export_csv( self, notebook_id, user_id ): + def export( self, notebook_id, format, user_id ): """ - Download the entire contents of the given notebook as a CSV file. + Download the entire contents of the given notebook as a stand-alone file. @type notebook_id: unicode @param notebook_id: id of notebook to download + @type format: unicode + @param format: string indicating the export plugin to use, currently one of: "html", "csv" @type user_id: unicode @param user_id: id of current logged-in user (if any), determined by @grab_user_id - @rtype: unicode - @return: CSV file with appropriate headers to trigger a download + @rtype: unicode or generator (for streaming files) + @return: exported file with appropriate headers to trigger a download @raise Access_error: the current user doesn't have access to the given notebook @raise Validation_error: one of the arguments is invalid """ + if format not in ( "html", "csv" ): + raise Access_error() + notebook = self.__users.load_notebook( user_id, notebook_id ) if not notebook: @@ -1253,37 +1222,19 @@ class Notebooks( object ): startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() ) other_notes = self.__database.select_many( Note, notebook.sql_load_non_startup_notes() ) - notes = startup_notes + other_notes + notes = startup_notes + other_notes - buffer = StringIO() - writer = csv.writer( buffer ) + import imp + from plugins.Invoke import invoke - cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=wiki.csv" - cherrypy.response.headerMap[ u"Content-Type" ] = u"text/csv;charset=utf-8" - - def stream(): - writer.writerow( ( u"contents", u"title", u"note_id", u"startup", u"username", u"revision_date" ) ) - yield buffer.getvalue() - buffer.truncate( 0 ) - - for note in notes: - user = None - if note.user_id: - user = self.__database.load( User, note.user_id ) - - writer.writerow( ( - note.contents and note.contents.strip().encode( "utf8" ) or None, - note.title and note.title.strip().encode( "utf8" ) or None, - note.object_id, - note.startup and 1 or 0, - note.user_id and user and user.username and user.username.encode( "utf8" ) or u"", - note.revision, - ) ) - - yield buffer.getvalue() - buffer.truncate( 0 ) - - return stream() + return invoke( + plugin_type = u"export", + plugin_name = format, + database = self.__database, + notebook = notebook, + notes = startup_notes + other_notes, + response_headers = cherrypy.response.headerMap, + ) @expose( view = Json ) @end_transaction diff --git a/controller/test/Test_notebooks.py b/controller/test/Test_notebooks.py index 8cb0fe6..14039cb 100644 --- a/controller/test/Test_notebooks.py +++ b/controller/test/Test_notebooks.py @@ -4478,10 +4478,12 @@ class Test_notebooks( Test_controller ): self.database.save( note3 ) result = self.http_get( - "/notebooks/export_html/%s" % self.notebook.object_id, + "/notebooks/export?notebook_id=%s&format=html" % self.notebook.object_id, session_id = self.session_id, ) - assert result.get( "notebook_name" ) == self.notebook.name + + assert result.get( "notebook" ).object_id == self.notebook.object_id + assert result.get( "view" ) notes = result.get( "notes" ) assert len( notes ) == self.database.select_one( int, self.notebook.sql_count_notes() ) @@ -4517,7 +4519,7 @@ class Test_notebooks( Test_controller ): note3 = Note.create( "55", u"

blah

foo", notebook_id = self.notebook.object_id ) self.database.save( note3 ) - path = "/notebooks/export_html/%s" % self.notebook.object_id + path = "/notebooks/export?notebook_id=%s&format=html" % self.notebook.object_id result = self.http_get( path, session_id = self.session_id, @@ -4537,37 +4539,26 @@ class Test_notebooks( Test_controller ): assert result.get( "error" ) - def test_export_csv( self, note_contents = None ): + def test_export_csv( self ): self.login() - if not note_contents: - note_contents = u"

blah

foo" + note_contents = u"

blah

foo" note3 = Note.create( "55", note_contents, notebook_id = self.notebook.object_id ) self.database.save( note3 ) result = self.http_get( - "/notebooks/export_csv/%s" % self.notebook.object_id, + "/notebooks/export?notebook_id=%s&format=csv" % self.notebook.object_id, session_id = self.session_id, ) headers = result[ u"headers" ] assert headers assert headers[ u"Content-Type" ] == u"text/csv;charset=utf-8" - assert headers[ u"Content-Disposition" ] == 'attachment; filename=wiki.csv' + assert headers[ u"Content-Disposition" ] == 'attachment; filename=%s.csv' % self.notebook.friendly_id - gen = result[ u"body" ] - assert isinstance( gen, types.GeneratorType ) - pieces = [] - - try: - for piece in gen: - pieces.append( piece ) - except AttributeError, exc: - if u"session_storage" not in str( exc ): - raise exc - - csv_data = "".join( pieces ) + assert result[ u"body" ] + csv_data = result[ u"body" ][ 0 ] reader = csv.reader( StringIO( csv_data ) ) row = reader.next() @@ -4612,29 +4603,11 @@ class Test_notebooks( Test_controller ): assert note_count == expected_note_count - def test_export_csv_with_unicode( self ): - self.test_export_csv( note_contents = u"

blah

ümlaut.png" ) - - def test_export_csv_without_note_title( self ): - self.test_export_csv( note_contents = u"there's no title" ) - - def test_export_csv_with_trailing_newline_in_title( self ): - self.test_export_csv( note_contents = u"

blah\n

foo" ) - - def test_export_csv_with_trailing_newline_in_contents( self ): - self.test_export_csv( note_contents = u"

blah

foo\n" ) - - def test_export_csv_with_blank_username( self ): - self.user._User__username = None - self.database.save( self.user ) - - self.test_export_csv( note_contents = u"

blah

foo" ) - def test_export_csv_without_login( self ): note3 = Note.create( "55", u"

blah

foo", notebook_id = self.notebook.object_id ) self.database.save( note3 ) - path = "/notebooks/export_csv/%s" % self.notebook.object_id + path = "/notebooks/export?notebook_id=%s&format=csv" % self.notebook.object_id result = self.http_get( path, session_id = self.session_id, @@ -4648,7 +4621,7 @@ class Test_notebooks( Test_controller ): self.login() result = self.http_get( - "/notebooks/export_csv/%s" % self.unknown_notebook_id, + "/notebooks/export?notebook_id=%s&format=csv" % self.unknown_notebook_id, session_id = self.session_id, ) diff --git a/plugins/Invoke.py b/plugins/Invoke.py new file mode 100644 index 0000000..80beb63 --- /dev/null +++ b/plugins/Invoke.py @@ -0,0 +1,10 @@ +import imp +import plugins + +def invoke( plugin_type, plugin_name, *args, **kwargs ): + plugin_name = u"%s_%s" % ( plugin_type, plugin_name ) + plugin_location = imp.find_module( plugin_name, plugins.__path__ ) + plugin_module = imp.load_module( plugin_name, *plugin_location ) + + function = getattr( plugin_module, plugin_type ) + return apply( function, args, kwargs ) diff --git a/plugins/__init__.py b/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/export_csv/__init__.py b/plugins/export_csv/__init__.py new file mode 100644 index 0000000..7a971d8 --- /dev/null +++ b/plugins/export_csv/__init__.py @@ -0,0 +1,38 @@ +import csv +from cStringIO import StringIO +from model.User import User + + +def export( database, notebook, notes, response_headers ): + """ + Format the given notes as a CSV file and return it as a streaming generator. + """ + buffer = StringIO() + writer = csv.writer( buffer ) + + response_headers[ u"Content-Disposition" ] = u"attachment; filename=%s.csv" % notebook.friendly_id + response_headers[ u"Content-Type" ] = u"text/csv;charset=utf-8" + + def stream(): + writer.writerow( ( u"contents", u"title", u"note_id", u"startup", u"username", u"revision_date" ) ) + yield buffer.getvalue() + buffer.truncate( 0 ) + + for note in notes: + user = None + if note.user_id: + user = database.load( User, note.user_id ) + + writer.writerow( ( + note.contents and note.contents.strip().encode( "utf8" ) or None, + note.title and note.title.strip().encode( "utf8" ) or None, + note.object_id, + note.startup and 1 or 0, + note.user_id and user and user.username and user.username.encode( "utf8" ) or u"", + note.revision, + ) ) + + yield buffer.getvalue() + buffer.truncate( 0 ) + + return stream() diff --git a/plugins/export_csv/test/Test_export_csv.py b/plugins/export_csv/test/Test_export_csv.py new file mode 100644 index 0000000..9110d49 --- /dev/null +++ b/plugins/export_csv/test/Test_export_csv.py @@ -0,0 +1,122 @@ +# -*- coding: utf8 -*- + +import csv +import types +import cherrypy +from cStringIO import StringIO +from pysqlite2 import dbapi2 as sqlite + +from model.User import User +from model.Note import Note +from model.Notebook import Notebook +from controller.Database import Database, Connection_wrapper +from controller.test.Stub_cache import Stub_cache +from plugins.Invoke import invoke + + +class Test_export_csv( object ): + def setUp( self ): + self.database = Database( + Connection_wrapper( sqlite.connect( ":memory:", detect_types = sqlite.PARSE_DECLTYPES, check_same_thread = False ) ), + cache = Stub_cache(), + ) + self.database.execute_script( file( "model/schema.sqlite" ).read(), commit = True ) + + self.username = u"mulder" + self.password = u"trustno1" + self.email_address = u"outthere@example.com" + self.user = User.create( self.database.next_id( User ), self.username, self.password, self.email_address ) + self.database.save( self.user, commit = False ) + + self.trash = Notebook.create( self.database.next_id( Notebook ), u"trash" ) + self.database.save( self.trash, commit = False ) + self.notebook = Notebook.create( self.database.next_id( Notebook ), u"notebook", self.trash.object_id, user_id = self.user.object_id ) + self.database.save( self.notebook, commit = False ) + + note_id = self.database.next_id( Note ) + self.note1 = Note.create( note_id, u"

my title

blah", notebook_id = self.notebook.object_id, startup = True, user_id = self.user.object_id ) + self.database.save( self.note1, commit = False ) + + note_id = self.database.next_id( Note ) + self.note2 = Note.create( note_id, u"

other title

whee", notebook_id = self.notebook.object_id, user_id = self.user.object_id ) + self.database.save( self.note2, commit = False ) + + def test_export_csv( self, note_contents = None ): + if not note_contents: + note_contents = u"

blah

foo" + + note3 = Note.create( self.database.next_id( Note ), note_contents, notebook_id = self.notebook.object_id, user_id = self.user.object_id ) + self.database.save( note3 ) + response_headers = {} + expected_notes = ( self.note1, self.note2, note3 ) + + result = invoke( + "export", + "csv", + self.database, + self.notebook, + expected_notes, + response_headers, + ) + + assert response_headers + assert response_headers[ u"Content-Type" ] == u"text/csv;charset=utf-8" + assert response_headers[ u"Content-Disposition" ] == 'attachment; filename=%s.csv' % self.notebook.friendly_id + + assert isinstance( result, types.GeneratorType ) + pieces = [] + + for piece in result: + pieces.append( piece ) + + csv_data = "".join( pieces ) + reader = csv.reader( StringIO( csv_data ) ) + + row = reader.next() + expected_header = [ u"contents", u"title", u"note_id", u"startup", u"username", u"revision_date" ] + assert row == expected_header + + note_count = 0 + + # assert that startup notes come first, then normal notes in descending revision order + for row in reader: + assert len( row ) == len( expected_header ) + ( contents, title, note_id, startup, username, revision_date ) = row + + assert note_count < len( expected_notes ) + expected_note = expected_notes[ note_count ] + + assert expected_note + assert contents.decode( "utf8" ) == expected_note.contents.strip() + + if expected_note.title: + assert title.decode( "utf8" ) == expected_note.title.strip() + else: + assert not title + + assert note_id.decode( "utf8" ) == expected_note.object_id + assert startup.decode( "utf8" ) == expected_note.startup and u"1" or "0" + assert username.decode( "utf8" ) == ( expected_note.user_id and self.user.username or u"" ) + assert revision_date.decode( "utf8" ) == unicode( expected_note.revision ) + + note_count += 1 + + assert note_count == len( expected_notes ) + + def test_export_csv_with_unicode( self ): + self.test_export_csv( note_contents = u"

blah

ümlaut.png" ) + + def test_export_csv_without_note_title( self ): + self.test_export_csv( note_contents = u"there's no title" ) + + def test_export_csv_with_trailing_newline_in_title( self ): + self.test_export_csv( note_contents = u"

blah\n

foo" ) + + def test_export_csv_with_trailing_newline_in_contents( self ): + self.test_export_csv( note_contents = u"

blah

foo\n" ) + + def test_export_csv_with_blank_username( self ): + self.user._User__username = None + self.database.save( self.user ) + + self.test_export_csv( note_contents = u"

blah

foo" ) diff --git a/view/Html_file.py b/plugins/export_html/Html_file.py similarity index 85% rename from view/Html_file.py rename to plugins/export_html/Html_file.py index 271bc43..b160d48 100644 --- a/view/Html_file.py +++ b/plugins/export_html/Html_file.py @@ -1,13 +1,13 @@ import re import cherrypy -from Tags import Html, Head, Title, Style, Meta, Body, H1, Div, Span, Hr, A +from view.Tags import Html, Head, Title, Style, Meta, Body, H1, Div, Span, Hr, A class Html_file( Html ): NOTE_LINK_PATTERN = re.compile( u']+[?&]note_id=([a-z0-9]*)"[^>]*>', re.IGNORECASE ) IMAGE_PATTERN = re.compile( u']* ?/?>', re.IGNORECASE ) - def __init__( self, notebook_name, notes ): + def __init__( self, notebook, notes ): relinked_notes = {} # map from note id to relinked note contents # relink all note links so they point to named anchors within the page. also, for now, remove all @@ -17,18 +17,18 @@ class Html_file( Html ): contents = self.IMAGE_PATTERN.sub( '', contents ) relinked_notes[ note.object_id ] = contents - cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=wiki.html" + cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=%s.html" % notebook.friendly_id Html.__init__( self, Head( Style( file( u"static/css/download.css" ).read(), type = u"text/css" ), Meta( content = u"text/html; charset=UTF-8", http_equiv = u"content-type" ), - Title( notebook_name ), + Title( notebook.name ), ), Body( Div( - H1( notebook_name ), + H1( notebook.name ), [ Span( A( name = u"note_%s" % note.object_id ), Div( diff --git a/plugins/export_html/__init__.py b/plugins/export_html/__init__.py new file mode 100644 index 0000000..fd62b0d --- /dev/null +++ b/plugins/export_html/__init__.py @@ -0,0 +1,14 @@ +from Html_file import Html_file + + +def export( database, notebook, notes, response_headers ): + """ + Format the given notes as an HTML file by relying on controller.Expose.expose() to use Html_file + as the view. + """ + return dict( + notebook = notebook, + notes = notes, + view = Html_file, + manual_encode = u"utf8", + ) diff --git a/plugins/export_html/test/Test_export_html.py b/plugins/export_html/test/Test_export_html.py new file mode 100644 index 0000000..456071d --- /dev/null +++ b/plugins/export_html/test/Test_export_html.py @@ -0,0 +1,75 @@ +# -*- coding: utf8 -*- + +import types +import cherrypy +from cStringIO import StringIO +from pysqlite2 import dbapi2 as sqlite + +from model.User import User +from model.Note import Note +from model.Notebook import Notebook +from controller.Database import Database, Connection_wrapper +from controller.test.Stub_cache import Stub_cache +from plugins.Invoke import invoke + + +class Test_export_html( object ): + def setUp( self ): + self.database = Database( + Connection_wrapper( sqlite.connect( ":memory:", detect_types = sqlite.PARSE_DECLTYPES, check_same_thread = False ) ), + cache = Stub_cache(), + ) + self.database.execute_script( file( "model/schema.sqlite" ).read(), commit = True ) + + self.username = u"mulder" + self.password = u"trustno1" + self.email_address = u"outthere@example.com" + self.user = User.create( self.database.next_id( User ), self.username, self.password, self.email_address ) + self.database.save( self.user, commit = False ) + + self.trash = Notebook.create( self.database.next_id( Notebook ), u"trash" ) + self.database.save( self.trash, commit = False ) + self.notebook = Notebook.create( self.database.next_id( Notebook ), u"notebook", self.trash.object_id, user_id = self.user.object_id ) + self.database.save( self.notebook, commit = False ) + + note_id = self.database.next_id( Note ) + self.note1 = Note.create( note_id, u"

my title

blah", notebook_id = self.notebook.object_id, startup = True, user_id = self.user.object_id ) + self.database.save( self.note1, commit = False ) + + note_id = self.database.next_id( Note ) + self.note2 = Note.create( note_id, u"

other title

whee", notebook_id = self.notebook.object_id, user_id = self.user.object_id ) + self.database.save( self.note2, commit = False ) + + def test_export_html( self ): + note3 = Note.create( "55", u"

blah

foo", notebook_id = self.notebook.object_id ) + self.database.save( note3 ) + response_headers = {} + expected_notes = ( self.note1, self.note2, note3 ) + + result = invoke( + "export", + "html", + self.database, + self.notebook, + expected_notes, + response_headers, + ) + + # response headers should be unchanged + assert response_headers == {} + + notes = result.get( "notes" ) + assert len( notes ) == len( expected_notes ) + + # assert that the notes are in the expected order + for ( note, expected_note ) in zip( notes, expected_notes ): + assert note.object_id == expected_note.object_id + assert note.revision == expected_note.revision + assert note.title == expected_note.title + assert note.contents == expected_note.contents + assert note.notebook_id == expected_note.notebook_id + assert note.startup == expected_note.startup + assert note.deleted_from_id == expected_note.deleted_from_id + assert note.rank == expected_note.rank + assert note.user_id == expected_note.user_id + assert note.creation == expected_note.creation diff --git a/static/js/Wiki.js b/static/js/Wiki.js index 1c182e9..3b14ef2 100644 --- a/static/js/Wiki.js +++ b/static/js/Wiki.js @@ -3695,14 +3695,14 @@ function Export_pulldown( wiki, notebook_id, invoker, anchor ) { this.invoker = invoker; this.html_link = createDOM( "a", { - "href": "/notebooks/export_html/" + notebook_id, + "href": "/notebooks/export?notebook_id=" + notebook_id + "&format=html", "class": "pulldown_label", "title": "Download this notebook as a stand-alone HTML web page." }, "HTML web page" ); this.csv_link = createDOM( "a", { - "href": "/notebooks/export_csv/" + notebook_id, + "href": "/notebooks/export?notebook_id=" + notebook_id + "&format=csv", "class": "pulldown_label", "title": "Download this notebook as a CSV spreadsheet file." },