witten
/
luminotes
Archived
1
0
Fork 0
Code Releases Activity

Implemented CSV exporting, and improved CSV importing to better handle the exported CSV files. 

Importing still needs work on properly handling internal note links.
This commit is contained in:
Dan Helfman 2008-09-17 23:39:11 -07:00
parent 6831fe6d89
commit 968ef22bc4
7 changed files with 322 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
NEWS
View File

@ -1,3 +1,13 @@
1.5.1:
* Implemented CSV exporting, so now you can export all of your notes to a
CSV spreadsheet file. This currently doesn't include revision history or
attached files.
* Improved CSV importing so you can export a CSV of your notes from one
Luminotes installation and import that CSV into a different Luminotes
installation.
* Fixed a bug in which the image preview page didn't correctly handle
filenames containing special characters.
1.5.0: September 12, 2008
* Initial release of Luminotes Desktop!
* Fixed a Luminotes Desktop Internet Explorer bug in which note links within

16
config/Common.py
View File

@ -132,6 +132,22 @@ settings = {
"stream_response": True,
"encoding_filter.on": False,
},
"/files/download_product": {
"stream_response": True,
"encoding_filter.on": False,
},
"/files/thumbnail": {
"stream_response": True,
"encoding_filter.on": False,
},
"/files/image": {
"stream_response": True,
"encoding_filter.on": False,
},
"/notebooks/export_csv": {
"stream_response": True,
"encoding_filter.on": False,
},
"/files/upload": {
"server.max_request_body_size": 505 * MEGABYTE, # maximum upload size
},

90
controller/Notebooks.py
View File

@ -1,6 +1,8 @@
import re
import cgi
import csv
import cherrypy
from cStringIO import StringIO
from datetime import datetime
from Expose import expose
from Validate import validate, Valid_string, Validation_error, Valid_bool, Valid_int
@ -1182,7 +1184,7 @@ class Notebooks( object ):
notebook_id = Valid_id(),
user_id = Valid_id( none_okay = True ),
)
def download_html( self, notebook_id, user_id ):
def export_html( self, notebook_id, user_id ):
"""
Download the entire contents of the given notebook as a stand-alone HTML page (no JavaScript).
@ -1191,7 +1193,7 @@ class Notebooks( object ):
@type user_id: unicode
@param user_id: id of current logged-in user (if any), determined by @grab_user_id
@rtype: unicode
@return: rendered HTML page
@return: rendered HTML page with appropriate headers to trigger a download
@raise Access_error: the current user doesn't have access to the given notebook
@raise Validation_error: one of the arguments is invalid
"""
@ -1211,6 +1213,69 @@ class Notebooks( object ):
notes = startup_notes + other_notes,
)
@expose()
@weakly_expire
@end_transaction
@grab_user_id
@validate(
notebook_id = Valid_id(),
user_id = Valid_id( none_okay = True ),
)
def export_csv( self, notebook_id, user_id ):
"""
Download the entire contents of the given notebook as a CSV file.
@type notebook_id: unicode
@param notebook_id: id of notebook to download
@type user_id: unicode
@param user_id: id of current logged-in user (if any), determined by @grab_user_id
@rtype: unicode
@return: CSV file with appropriate headers to trigger a download
@raise Access_error: the current user doesn't have access to the given notebook
@raise Validation_error: one of the arguments is invalid
"""
if not self.__users.check_access( user_id, notebook_id ):
raise Access_error()
notebook = self.__database.load( Notebook, notebook_id )
if not notebook:
raise Access_error()
startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() )
other_notes = self.__database.select_many( Note, notebook.sql_load_non_startup_notes() )
notes = startup_notes + other_notes
buffer = StringIO()
writer = csv.writer( buffer )
cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=wiki.csv"
cherrypy.response.headerMap[ u"Content-Type" ] = u"text/csv"
def stream():
writer.writerow( ( u"contents", u"title", u"note_id", u"startup", u"username", u"revision_date" ) )
yield buffer.getvalue()
buffer.truncate( 0 )
for note in notes:
user = None
if note.user_id:
user = self.__database.load( User, note.user_id )
writer.writerow( (
note.contents.encode( "utf8" ), # TODO: should this try to remove the title?
note.title.encode( "utf8" ),
note.object_id,
note.startup and 1 or 0,
note.user_id and user and user.username.encode( "utf8" ) or u"",
note.revision,
) )
yield buffer.getvalue()
buffer.truncate( 0 )
return stream()
@expose( view = Json )
@end_transaction
@grab_user_id
@ -1704,7 +1769,8 @@ class Notebooks( object ):
@type title_column: int or NoneType
@param title_column: zero-based index of the column containing note titles (None indicates
the lack of any such column, in which case titles are derived from the
first few words of each note's contents)
first few words of each note's contents if no title is already present
in the note's contents)
@type plaintext: bool
@param plaintext: True if the note contents are plaintext, or False if they're HTML
@type import_button: unicode
@ -1743,11 +1809,13 @@ class Notebooks( object ):
if title_column is not None and title_column >= row_length:
raise Import_error()
# if there is a title column, use it. otherwise, use the first line of the content column as
# the title
title = None
# if there is a title column, use it. otherwise, if the note doesn't already contain a title,
# use the first line of the content column as the title
if title_column and title_column != content_column and len( row[ title_column ].strip() ) > 0:
title = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = plaintext )( row[ title_column ].strip() ) )
else:
elif plaintext or not Note.TITLE_PATTERN.search( row[ content_column ] ):
content_text = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = plaintext )( row[ content_column ].strip() ) )
content_lines = [ line for line in self.NEWLINE_PATTERN.split( content_text ) if line.strip() ]
@ -1769,16 +1837,18 @@ class Notebooks( object ):
else:
break
contents = u"<h3>%s</h3>%s" % (
title,
Valid_string( max = 25000, escape_html = plaintext, require_link_target = True )( row[ content_column ] ),
)
contents = Valid_string( max = 25000, escape_html = plaintext, require_link_target = True )( row[ content_column ] )
if plaintext:
contents = contents.replace( u"\n", u"<br />" )
note_id = self.__database.next_id( Note, commit = False )
note = Note.create( note_id, contents, notebook_id = notebook.object_id, startup = False, rank = None, user_id = user_id )
# if the note doesn't have a title yet, then tack the given title onto the start of the contents
if title and note.title is None:
note.contents = u"<h3>%s</h3>%s" % ( title, note.contents )
self.__database.save( note, commit = False )
# delete the CSV file now that it's been imported

143
controller/test/Test_notebooks.py
View File

@ -3471,14 +3471,14 @@ class Test_notebooks( Test_controller ):
assert result.get( "error" )
def test_download_html( self ):
def test_export_html( self ):
self.login()
note3 = Note.create( "55", u"<h3>blah</h3>foo", notebook_id = self.notebook.object_id )
self.database.save( note3 )
result = self.http_get(
"/notebooks/download_html/%s" % self.notebook.object_id,
"/notebooks/export_html/%s" % self.notebook.object_id,
session_id = self.session_id,
)
assert result.get( "notebook_name" ) == self.notebook.name
@ -3500,11 +3500,11 @@ class Test_notebooks( Test_controller ):
previous_revision = note.revision
def test_download_html( self ):
def test_export_html( self ):
note3 = Note.create( "55", u"<h3>blah</h3>foo", notebook_id = self.notebook.object_id )
self.database.save( note3 )
path = "/notebooks/download_html/%s" % self.notebook.object_id
path = "/notebooks/export_html/%s" % self.notebook.object_id
result = self.http_get(
path,
session_id = self.session_id,
@ -3514,11 +3514,11 @@ class Test_notebooks( Test_controller ):
assert headers
assert headers.get( "Location" ) == u"http:///login?after_login=%s" % urllib.quote( path )
def test_download_html_with_unknown_notebook( self ):
def test_export_html_with_unknown_notebook( self ):
self.login()
result = self.http_get(
"/notebooks/download_html/%s" % self.unknown_notebook_id,
"/notebooks/export_html/%s" % self.unknown_notebook_id,
session_id = self.session_id,
)
@ -4379,7 +4379,8 @@ class Test_notebooks( Test_controller ):
assert note.title == title
if plaintext is True:
contents = contents.replace( u"\n", u"<br />" )
contents = u"<h3>%s</h3>%s" % ( title, contents )
if plaintext is True or u"<h3>" not in contents:
contents = u"<h3>%s</h3>%s" % ( title, contents )
assert note.contents == contents
# make sure the CSV data file has been deleted from the database and filesystem
@ -4390,6 +4391,70 @@ class Test_notebooks( Test_controller ):
user = self.database.load( User, self.user.object_id )
assert user.storage_bytes > 0
def test_import_csv_title_already_in_contents( self ):
self.login()
csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","<h3>yay</h3>3.3"\n"8","whee","hmm\n<h3>my title</h3>foo"\n3,4,5'
expected_notes = [
( "yay", "<h3>yay</h3>3.3" ), # ( title, contents )
( "my title", "hmm\n<h3>my title</h3>foo" ),
( "4", "5" ),
]
self.http_upload(
"/files/upload?file_id=%s" % self.file_id,
dict(
notebook_id = self.notebook.object_id,
note_id = self.note.object_id,
),
filename = self.filename,
file_data = csv_data,
content_type = self.content_type,
session_id = self.session_id,
)
result = self.http_post( "/notebooks/import_csv/", dict(
file_id = self.file_id,
content_column = 2,
title_column = 1,
plaintext = False,
import_button = u"import",
), session_id = self.session_id )
self.__assert_imported_notebook( expected_notes, result, plaintext = False )
def test_import_csv_title_already_in_plaintext_contents( self ):
self.login()
csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","hi\n<h3>yay</h3>3.3"\n"8","whee","hmm\n<h3>my title</h3>foo"\n3,4,5'
expected_notes = [
( "blah and stuff", "hi<br />&lt;h3&gt;yay&lt;/h3&gt;3.3" ), # ( title, contents )
( "whee", "hmm<br />&lt;h3&gt;my title&lt;/h3&gt;foo" ),
( "4", "5" ),
]
self.http_upload(
"/files/upload?file_id=%s" % self.file_id,
dict(
notebook_id = self.notebook.object_id,
note_id = self.note.object_id,
),
filename = self.filename,
file_data = csv_data,
content_type = self.content_type,
session_id = self.session_id,
)
result = self.http_post( "/notebooks/import_csv/", dict(
file_id = self.file_id,
content_column = 2,
title_column = 1,
plaintext = True,
import_button = u"import",
), session_id = self.session_id )
self.__assert_imported_notebook( expected_notes, result, plaintext = True )
def test_import_csv_unknown_file_id( self ):
self.login()
@ -4553,6 +4618,70 @@ class Test_notebooks( Test_controller ):
self.__assert_imported_notebook( expected_notes, result )
def test_import_csv_no_title_column_and_title_already_in_contents( self ):
self.login()
csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","<h3>yay</h3>3.3"\n"8","whee","hmm\n<h3>my title</h3>foo"\n3,4,5'
expected_notes = [
( "yay", "<h3>yay</h3>3.3" ), # ( title, contents )
( "my title", "hmm\n<h3>my title</h3>foo" ),
( "5", "5" ),
]
self.http_upload(
"/files/upload?file_id=%s" % self.file_id,
dict(
notebook_id = self.notebook.object_id,
note_id = self.note.object_id,
),
filename = self.filename,
file_data = csv_data,
content_type = self.content_type,
session_id = self.session_id,
)
result = self.http_post( "/notebooks/import_csv/", dict(
file_id = self.file_id,
content_column = 2,
title_column = None,
plaintext = False,
import_button = u"import",
), session_id = self.session_id )
self.__assert_imported_notebook( expected_notes, result, plaintext = False )
def test_import_csv_no_title_column_and_title_already_in_plaintext_contents( self ):
self.login()
csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","hi\n<h3>yay</h3>3.3"\n"8","whee","hmm\n<h3>my title</h3>foo"\n3,4,5'
expected_notes = [
( "hi", "hi<br />&lt;h3&gt;yay&lt;/h3&gt;3.3" ), # ( title, contents )
( "hmm", "hmm<br />&lt;h3&gt;my title&lt;/h3&gt;foo" ),
( "5", "5" ),
]
self.http_upload(
"/files/upload?file_id=%s" % self.file_id,
dict(
notebook_id = self.notebook.object_id,
note_id = self.note.object_id,
),
filename = self.filename,
file_data = csv_data,
content_type = self.content_type,
session_id = self.session_id,
)
result = self.http_post( "/notebooks/import_csv/", dict(
file_id = self.file_id,
content_column = 2,
title_column = None,
plaintext = True,
import_button = u"import",
), session_id = self.session_id )
self.__assert_imported_notebook( expected_notes, result, plaintext = True )
def test_import_csv_no_title_column_and_html_first_line( self ):
self.login()

1
static/css/style.css
View File

@ -580,6 +580,7 @@ h1 {
.pulldown_label {
color: #000000;
text-decoration: none;
}
.pulldown_label:hover {

56
static/js/Wiki.js
View File

@ -340,10 +340,13 @@ Wiki.prototype.populate = function ( startup_notes, current_notes, note_read_wri
save_button.disabled = true;
}
var download_html_link = getElement( "download_html_link" );
if ( download_html_link ) {
connect( download_html_link, "onclick", function ( event ) {
self.save_editor( null, true );
var export_link = getElement( "export_link" );
if ( export_link ) {
connect( export_link, "onclick", function ( event ) {
self.save_editor( null, true, function () {
self.export_clicked();
} );
event.stop();
} );
}
@ -2595,6 +2598,18 @@ Wiki.prototype.zero_total_notes_count = function () {
signal( this, "total_notes_count_updated", this.total_notes_count );
}
Wiki.prototype.export_clicked = function () {
var pulldown_id = "export_pulldown";
var existing_div = getElement( pulldown_id );
if ( existing_div ) {
existing_div.pulldown.shutdown();
existing_div.pulldown = null;
return;
}
new Export_pulldown( this, this.notebook_id, this.invoker, getElement( "export_link" ) );
}
Wiki.prototype.import_clicked = function () {
var pulldown_id = "import_pulldown";
var existing_div = getElement( pulldown_id );
@ -3358,6 +3373,39 @@ Upload_pulldown.prototype.shutdown = function () {
}
function Export_pulldown( wiki, notebook_id, invoker, anchor ) {
Pulldown.call( this, wiki, notebook_id, "export_pulldown", anchor, null, false );
this.invoker = invoker;
this.html_link = createDOM( "a", {
"href": "/notebooks/export_html/" + notebook_id,
"class": "pulldown_label",
"title": "Download this notebook as a stand-alone HTML web page."
},
"HTML web page"
);
this.csv_link = createDOM( "a", {
"href": "/notebooks/export_csv/" + notebook_id,
"class": "pulldown_label",
"title": "Download this notebook as a CSV spreadsheet file."
},
"CSV spreadsheet"
);
appendChildNodes( this.div, createDOM( "div", {}, this.html_link ) );
appendChildNodes( this.div, createDOM( "div", {}, this.csv_link ) );
Pulldown.prototype.finish_init.call( this );
}
Export_pulldown.prototype = new function () { this.prototype = Pulldown.prototype; };
Export_pulldown.prototype.constructor = Export_pulldown;
Export_pulldown.prototype.shutdown = function () {
Pulldown.prototype.shutdown.call( this );
}
function Import_pulldown( wiki, notebook_id, invoker, anchor ) {
anchor.pulldown = this;

47
view/Link_area.py
View File

@ -32,16 +32,6 @@ class Link_area( Div ):
class_ = u"link_area_item",
),
( notebook.name != u"Luminotes" ) and Div(
A(
u"download as html",
href = u"/notebooks/download_html/%s" % notebook.object_id,
id = u"download_html_link",
title = u"Download a stand-alone copy of the entire wiki notebook.",
),
class_ = u"link_area_item",
) or None,
( rate_plan.get( u"notebook_sharing" ) and notebook.name == u"Luminotes blog" ) and Div(
A(
u"subscribe to rss",
@ -70,24 +60,41 @@ class Link_area( Div ):
class_ = u"link_area_item",
) or None ),
notebook.read_write and Div(
A(
u"nothing but notes",
href = u"#",
id = u"declutter_link",
title = u"Focus on just your notes without any distractions.",
),
class_ = u"link_area_item",
) or None,
( not notebook.read_write and notebook.name != u"Luminotes" ) and Div(
A(
u"export",
href = u"#",
id = u"export_link",
title = u"Download a stand-alone copy of the entire wiki notebook.",
),
class_ = u"link_area_item",
) or None,
notebook.read_write and Span(
Div(
A(
u"nothing but notes",
href = u"#",
id = u"declutter_link",
title = u"Focus on just your notes without any distractions.",
),
class_ = u"link_area_item",
),
( notebook.name != u"Luminotes" ) and Div(
A(
u"import",
href = u"#",
id = u"import_link",
title = u"Import notes from other software into Luminotes.",
),
u"|",
A(
u"export",
href = u"#",
id = u"export_link",
title = u"Download a stand-alone copy of the entire wiki notebook.",
),
class_ = u"link_area_item",
) or None,