From 6e8e1a31c9970f9935307b4eccd0347012c44afb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 11 Aug 2008 13:21:12 -0700 Subject: [PATCH] Improved escaping of titles based on value of plaintext parameter. Finished unit for import_csv(). --- controller/Notebooks.py | 12 +- controller/test/Test_notebooks.py | 407 ++++++++++++++++++++++++++++-- 2 files changed, 399 insertions(+), 20 deletions(-) diff --git a/controller/Notebooks.py b/controller/Notebooks.py index 9e2c272..c1c39c0 100644 --- a/controller/Notebooks.py +++ b/controller/Notebooks.py @@ -1722,10 +1722,16 @@ class Notebooks( object ): # if there is a title column, use it. otherwise, use the first line of the content column as # the title if title_column and title_column != content_column and len( row[ title_column ].strip() ) > 0: - title = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = True )( row[ title_column ].strip() ) ) + title = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = plaintext )( row[ title_column ].strip() ) ) else: - title = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = True )( row[ content_column ].strip() ) ) - title = [ line for line in self.NEWLINE_PATTERN.split( title ) if line.strip() ][ 0 ] + content_text = Html_nuker( allow_refs = True ).nuke( Valid_string( escape_html = plaintext )( row[ content_column ].strip() ) ) + content_lines = [ line for line in self.NEWLINE_PATTERN.split( content_text ) if line.strip() ] + + # skip notes with empty contents + if len( content_lines ) == 0: + continue + + title = content_lines[ 0 ] # truncate the makeshift title to a reasonable length, but truncate on a word boundary if len( title ) > TRUNCATED_TITLE_CHAR_LENGTH: diff --git a/controller/test/Test_notebooks.py b/controller/test/Test_notebooks.py index 8f0c9bf..9de725b 100644 --- a/controller/test/Test_notebooks.py +++ b/controller/test/Test_notebooks.py @@ -4348,7 +4348,7 @@ class Test_notebooks( Test_controller ): self.__assert_imported_notebook( expected_notes, result ) - def __assert_imported_notebook( self, expected_notes, result ): + def __assert_imported_notebook( self, expected_notes, result, plaintext = True ): assert result[ u"redirect" ].startswith( u"/notebooks/" ) # make sure that a notebook has been created with the imported notes @@ -4377,7 +4377,9 @@ class Test_notebooks( Test_controller ): for ( note, ( title, contents ) ) in zip( recent_notes, expected_notes ): assert note.title == title - contents = u"

%s

%s" % ( title, contents.replace( u"\n", u"
" ) ) + if plaintext is True: + contents = contents.replace( u"\n", u"
" ) + contents = u"

%s

%s" % ( title, contents ) assert note.contents == contents # make sure the CSV data file has been deleted from the database and filesystem @@ -4585,40 +4587,411 @@ class Test_notebooks( Test_controller ): self.__assert_imported_notebook( expected_notes, result ) def test_import_csv_no_title_column_and_long_first_line( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","Ten percent of nuthin\' is...let me do the math here...nuthin\' into nuthin\'...carry the nuthin\'..."\n"8","whee","I brought you some supper but if you\'d prefer a lecture, I\'ve a few very catchy ones prepped...sin and hellfire... one has lepers.\n--Book"\n3,4,5' + + # expect the long titles to be truncated on a word boundary + expected_notes = [ + ( "Ten percent of nuthin' is...let me do the math here...nuthin' into", "Ten percent of nuthin' is...let me do the math here...nuthin' into nuthin'...carry the nuthin'..." ), + ( "I brought you some supper but if you'd prefer a lecture, I've a few very catchy", "I brought you some supper but if you'd prefer a lecture, I've a few very catchy ones prepped...sin and hellfire... one has lepers.\n--Book" ), + ( "5", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result ) def test_import_csv_no_title_column_and_long_first_line_without_spaces( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"\n"8","whee","ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ\nfoo"\n3,4,5' + + # expect the long titles not to be truncated since there are no spaces + expected_notes = [ + ( "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" ), + ( "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ", "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ\nfoo" ), + ( "5", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result ) def test_import_csv_no_title_column_and_blank_first_line( self ): - raise NotImplementedError() + self.login() - def test_import_csv_plaintext_content_with_newline( self ): - raise NotImplementedError() + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","\n\n3.3"\n"8","whee","\nfoo"\n3,4,5' + expected_notes = [ + ( "3.3", "3.3" ), # ( title, contents ) + ( "foo", "foo" ), + ( "5", "5" ), + ] - def test_import_csv_plaintext_content_with_html( self ): - raise NotImplementedError() + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) - def test_import_csv_long_content( self ): - raise NotImplementedError() + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result ) + + def test_import_csv_no_title_column_and_empty_contents( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","\n\n\n \n"\n"8","whee","foo"\n3,4,5' + expected_notes = [ + ( "foo", "foo" ), + ( "5", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result ) + + def test_import_csv_plaintext_content_as_html( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff",3.3\n"8","whee","hmm\nfoo"\n3,4,5' + expected_notes = [ + ( "blah and stuff", "3.3" ), # ( title, contents ) + ( "whee", "hmm\nfoo" ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) + + def test_import_csv_html_title( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff",3.3\n"8","wh ee","hmm\nfoo"\n3,4,5' + expected_notes = [ + ( "blah and stuff", "3.3" ), # ( title, contents ) + ( "wh ee", "hmm\nfoo" ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) def test_import_csv_html_content( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","3.3  "\n"8","whee","hmm\nfoo"\n3,4,5' + expected_notes = [ + ( "blah and stuff", "3.3  " ), # ( title, contents ) + ( "whee", "hmm\nfoo" ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) + + def test_import_csv_html_content_without_title( self ): + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","3.3  "\n"8","whee","hmm\nfoo"\n3,4,5' + expected_notes = [ + ( "3.3  ", "3.3  " ), # ( title, contents ) + ( "hmm", "hmm\nfoo" ), + ( "5", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = None, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) def test_import_csv_html_content_with_link( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","3.3  "\n"8","whee","hmm\nfoo"\n3,4,5' + expected_notes = [ + ( "blah and stuff", "3.3  " ), # ( title, contents ) + ( "whee", 'hmm\nfoo' ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) def test_import_csv_html_content_with_link_and_target( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff","3.3  "\n"8","whee","hmm\nfoo"\n3,4,5' + expected_notes = [ + ( "blah and stuff", "3.3  " ), # ( title, contents ) + ( "whee", 'hmm\nfoo' ), + ( "4", "5" ), + ] + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = False, + import_button = u"import", + ), session_id = self.session_id ) + + self.__assert_imported_notebook( expected_notes, result, plaintext = False ) def test_import_csv_without_login( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff",3.3\n"8","whee","hmm\nfoo"\n3,4,5' + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = True, + import_button = u"import", + ) ) + + assert u"access" in result[ u"error" ] def test_import_csv_without_access( self ): - raise NotImplementedError() + self.login() + self.make_extra_notebooks() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff",3.3\n"8","whee","hmm\nfoo"\n3,4,5' + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook2.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + self.login2() + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + assert u"access" in result[ u"error" ] def test_import_csv_invalid( self ): - raise NotImplementedError() + self.login() + + csv_data = '"label 1","label 2","label 3"\n5,"blah and stuff",,,,,,3.3\n"8","whee","hmm\nfoo"\n3,4,5' + + self.http_upload( + "/files/upload?file_id=%s" % self.file_id, + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = csv_data, + content_type = self.content_type, + session_id = self.session_id, + ) + + result = self.http_post( "/notebooks/import_csv/", dict( + file_id = self.file_id, + content_column = 2, + title_column = 1, + plaintext = True, + import_button = u"import", + ), session_id = self.session_id ) + + assert result[ u"error" ] def login( self ): result = self.http_post( "/users/login", dict(