diff --git a/NEWS b/NEWS index 0ef224b..227d1e0 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.2.0: February ??, 2008 + * Users can now upload files to attach to their notes. + 1.1.3: January 28, 2008 * Now, if you delete a notebook and the only remaining notebook is read-only, then a new read-write notebook is created for you automatically. diff --git a/config/Common.py b/config/Common.py index 461d714..c7d76b4 100644 --- a/config/Common.py +++ b/config/Common.py @@ -63,7 +63,11 @@ settings = { """ """, }, - "/files/upload": { + "/files/download": { + "stream_response": True, + "encoding_filter.on": False, + }, + "/files/progress": { "stream_response": True }, } diff --git a/controller/Files.py b/controller/Files.py index ecb49da..2734ee5 100644 --- a/controller/Files.py +++ b/controller/Files.py @@ -1,11 +1,17 @@ import cgi +import time +import tempfile import cherrypy +from threading import Lock, Event from Expose import expose -from Validate import validate +from Validate import validate, Valid_int, Validation_error from Database import Valid_id from Users import grab_user_id from Expire import strongly_expire +from model.File import File from view.Upload_page import Upload_page +from view.Blank_page import Blank_page +from view.Json import Json class Access_error( Exception ): @@ -36,6 +42,153 @@ class Upload_error( Exception ): ) +# map of upload id to Upload_file +current_uploads = {} +current_uploads_lock = Lock() + + +class Upload_file( object ): + """ + File-like object for storing file uploads. + """ + def __init__( self, file_id, filename, content_length ): + self.__file = file( self.make_server_filename( file_id ), "w+" ) + self.__file_id = file_id + self.__filename = filename + self.__content_length = content_length + self.__file_received_bytes = 0 + self.__total_received_bytes = cherrypy.request.rfile.bytes_read + self.__total_received_bytes_updated = Event() + self.__complete = Event() + + def write( self, data ): + self.__file.write( data ) + self.__file_received_bytes += len( data ) + self.__total_received_bytes = cherrypy.request.rfile.bytes_read + self.__total_received_bytes_updated.set() + + def tell( self ): + return self.__file.tell() + + def seek( self, position ): + self.__file.seek( position ) + + def read( self, size = None ): + if size is None: + return self.__file.read() + + return self.__file.read( size ) + + def wait_for_total_received_bytes( self ): + self.__total_received_bytes_updated.wait( timeout = cherrypy.server.socket_timeout ) + self.__total_received_bytes_updated.clear() + return self.__total_received_bytes + + def close( self ): + self.__file.close() + self.__complete.set() + + def wait_for_complete( self ): + self.__complete.wait( timeout = cherrypy.server.socket_timeout ) + + @staticmethod + def make_server_filename( file_id ): + return u"files/%s" % file_id + + filename = property( lambda self: self.__filename ) + + # expected byte count of the entire form upload, including the file and other form parameters + content_length = property( lambda self: self.__content_length ) + + # count of bytes received thus far for this file upload only + file_received_bytes = property( lambda self: self.__file_received_bytes ) + + # count of bytes received thus far for the form upload, including the file and other form + # parameters + total_received_bytes = property( lambda self: self.__total_received_bytes ) + + +class FieldStorage( cherrypy._cpcgifs.FieldStorage ): + """ + Derived from cherrypy._cpcgifs.FieldStorage, which is in turn derived from cgi.FieldStorage, which + calls make_file() to create a temporary file where file uploads are stored. By wrapping this file + object, we can track its progress as its written. Inspired by: + http://www.cherrypy.org/attachment/ticket/546/uploadfilter.py + + This method relies on a file_id parameter being present in the HTTP query string. + + @type binary: NoneType + @param binary: ignored + @type user_id: unicode or NoneType + @param user_id: id of current logged-in user (if any) + @rtype: Upload_file + @return: wrapped temporary file used to store the upload + @raise Upload_error: the provided file_id value is invalid, or the filename or Content-Length is + missing + """ + def make_file( self, binary = None, user_id = None ): + global current_uploads, current_uploads_lock + + cherrypy.server.max_request_body_size = 0 # remove CherryPy default file size limit of 100 MB + cherrypy.response.timeout = 3600 * 2 # increase upload timeout to 2 hours (default is 5 min) + cherrypy.server.socket_timeout = 60 # increase socket timeout to one minute (default is 10 sec) + DASHES_AND_NEWLINES = 6 # four dashes and two newlines + + # release the cherrypy session lock so that the user can issue other commands while the file is + # uploading + cherrypy.session.release_lock() + + # pluck the file id out of the query string. it would be preferable to grab it out of parsed + # form variables instead, but at this point in the processing, all the form variables might not + # be parsed + file_id = cgi.parse_qs( cherrypy.request.query_string ).get( u"file_id", [ None ] )[ 0 ] + try: + file_id = Valid_id()( file_id ) + except ValueError: + raise Upload_error( "The file_id is invalid." ) + + if not self.filename: + raise Upload_error( "Please provide a filename." ) + + content_length = cherrypy.request.headers.get( "content-length", 0 ) + try: + content_length = Valid_int( min = 0 )( content_length ) - len( self.outerboundary ) - DASHES_AND_NEWLINES + except ValueError: + raise Upload_error( "The Content-Length header value is invalid." ) + + # file size is the entire content length of the POST, minus the size of the other form + # parameters and boundaries. note: this assumes that the uploaded file is sent as the last + # form parameter in the POST + # TODO: verify that the uploaded file is always sent as the last parameter + existing_file = current_uploads.get( file_id ) + if existing_file: + existing_file.close() + + upload_file = Upload_file( file_id, self.filename.strip(), content_length ) + + current_uploads_lock.acquire() + try: + current_uploads[ file_id ] = upload_file + finally: + current_uploads_lock.release() + + return upload_file + + def __write( self, line ): + """ + This implementation of __write() is different than that of the base class, because it calls + make_file() whenever there is a filename instead of only for large enough files. + """ + if self.__file is not None and self.filename: + self.file = self.make_file( '' ) + self.file.write( self.__file.getvalue() ) + self.__file = None + + self.file.write( line ) + +cherrypy._cpcgifs.FieldStorage = FieldStorage + + class Files( object ): """ Controller for dealing with uploaded files, corresponding to the "/files" URL. @@ -54,39 +207,96 @@ class Files( object ): self.__database = database self.__users = users + @expose() + @grab_user_id + @validate( + file_id = Valid_id(), + user_id = Valid_id( none_okay = True ), + ) + def download( self, file_id, user_id = None ): + """ + Return the contents of file that a user has previously uploaded. + + @type file_id: unicode + @param file_id: id of the file to download + @type user_id: unicode or NoneType + @param user_id: id of current logged-in user (if any) + @rtype: unicode + @return: file data + @raise Access_error: the current user doesn't have access to the notebook that the file is in + """ + db_file = self.__database.load( File, file_id ) + + if not db_file or not self.__users.check_access( user_id, db_file.notebook_id ): + raise Access_error() + + db_file = self.__database.load( File, file_id ) + + cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=%s" % db_file.filename + cherrypy.response.headerMap[ u"Content-Length" ] = db_file.size_bytes +# TODO: send content type +# cherrypy.response.headerMap[ u"Content-Type" ] = u"image/png" + + def stream(): + CHUNK_SIZE = 8192 + local_file = file( Upload_file.make_server_filename( file_id ) ) + + while True: + data = local_file.read( CHUNK_SIZE ) + if len( data ) == 0: break + yield data + + return stream() + + @expose( view = Upload_page ) + @strongly_expire + @grab_user_id @validate( notebook_id = Valid_id(), note_id = Valid_id(), + user_id = Valid_id( none_okay = True ), ) - def upload_page( self, notebook_id, note_id ): + def upload_page( self, notebook_id, note_id, user_id ): """ - Provide the information necessary to display the file upload page. + Provide the information necessary to display the file upload page, including the generation of a + unique file id. @type notebook_id: unicode @param notebook_id: id of the notebook that the upload will be to @type note_id: unicode @param note_id: id of the note that the upload will be to + @type user_id: unicode or NoneType + @param user_id: id of current logged-in user (if any) @rtype: unicode @return: rendered HTML page + @raise Access_error: the current user doesn't have access to the given notebook """ + if not self.__users.check_access( user_id, notebook_id, read_write = True ): + raise Access_error() + + file_id = self.__database.next_id( File ) + return dict( notebook_id = notebook_id, note_id = note_id, + file_id = file_id, ) - @expose() + @expose( view = Blank_page ) @strongly_expire @grab_user_id @validate( upload = (), notebook_id = Valid_id(), note_id = Valid_id(), + file_id = Valid_id(), user_id = Valid_id( none_okay = True ), ) - def upload( self, upload, notebook_id, note_id, user_id ): + def upload( self, upload, notebook_id, note_id, file_id, user_id ): """ - Upload a file from the client for attachment to a particular note. + Upload a file from the client for attachment to a particular note. The file_id must be provided + as part of the query string, even if the other values are submitted as form data. @type upload: cgi.FieldStorage @param upload: file handle to uploaded file @@ -94,40 +304,90 @@ class Files( object ): @param notebook_id: id of the notebook that the upload is to @type note_id: unicode @param note_id: id of the note that the upload is to - @raise Access_error: the current user doesn't have access to the given notebook or note - @raise Upload_error: an error occurred when processing the uploaded file + @type file_id: unicode + @param file_id: id of the file being uploaded @type user_id: unicode or NoneType @param user_id: id of current logged-in user (if any) @rtype: unicode @return: rendered HTML page + @raise Access_error: the current user doesn't have access to the given notebook or note + @raise Upload_error: the Content-Length header value is invalid """ - if not self.__users.check_access( user_id, notebook_id ): + global current_uploads, current_uploads_lock + + if not self.__users.check_access( user_id, notebook_id, read_write = True ): raise Access_error() - cherrypy.server.max_request_body_size = 0 # remove file size limit of 100 MB - cherrypy.response.timeout = 3600 # increase upload timeout to one hour (default is 5 min) - cherrypy.server.socket_timeout = 60 # increase socket timeout to one minute (default is 10 sec) - CHUNK_SIZE = 8 * 1024 # 8 Kb + # write the file to the database + uploaded_file = current_uploads.get( file_id ) + if not uploaded_file: + raise Upload_error() - headers = {} - for key, val in cherrypy.request.headers.iteritems(): - headers[ key.lower() ] = val +# TODO: grab content type and store it + #print upload.headers.get( "content-type", "MISSING" ) +# TODO: somehow detect when upload is canceled and abort + + db_file = File.create( file_id, notebook_id, note_id, uploaded_file.filename, uploaded_file.file_received_bytes ) + self.__database.save( db_file ) + uploaded_file.close() + + current_uploads_lock.acquire() try: - file_size = int( headers.get( "content-length", 0 ) ) - except ValueError: - raise Upload_error() - if file_size <= 0: - raise Upload_error() + del( current_uploads[ file_id ] ) + finally: + current_uploads_lock.release() - filename = upload.filename.strip() + return dict() - def process_upload(): + @expose() + @strongly_expire + @validate( + file_id = Valid_id(), + filename = unicode, + ) + def progress( self, file_id, filename ): + """ + Stream information on a file that is in the process of being uploaded. This method does not + perform any access checks, but the only information streamed is a progress bar and upload + percentage. + + @type file_id: unicode + @param file_id: id of a currently uploading file + @type filename: unicode + @param filename: name of the file to report on + @rtype: unicode + @return: streaming HTML progress bar + """ + # release the session lock before beginning to stream the upload report. otherwise, if the + # upload is cancelled before it's done, the lock won't be released + cherrypy.session.release_lock() + + # poll until the file is uploading (as determined by current_uploads) or completely uploaded (in + # the database with a filename) + while True: + uploading_file = current_uploads.get( file_id ) + db_file = None + + if uploading_file: + fraction_reported = 0.0 + break + + db_file = self.__database.load( File, file_id ) + if not db_file: + raise Upload_error( u"The file id is unknown" ) + if db_file.filename is None: + time.sleep( 0.1 ) + continue + fraction_reported = 1.0 + break + + # TODO: maybe move this to the view/ directory + def report( uploading_file, fraction_reported ): """ - Process the file upload while streaming a progress meter as it uploads. + Stream a progress meter as it uploads. """ progress_bytes = 0 - fraction_reported = 0.0 progress_width_em = 20 tick_increment = 0.01 progress_bar = u'' % \ @@ -144,15 +404,6 @@ class Files( object ): """ - if not filename: - yield \ - u""" -
upload error:
- Please check that the filename is valid. - - """ - return - base_filename = filename.split( u"/" )[ -1 ].split( u"\\" )[ -1 ] yield \ u""" @@ -180,46 +431,64 @@ class Files( object ): """ % ( cgi.escape( base_filename ), progress_bar, progress_width_em ) - while True: - chunk = upload.file.read( CHUNK_SIZE ) - if not chunk: break - progress_bytes += len( chunk ) - fraction_done = float( progress_bytes ) / float( file_size ) + if uploading_file: + received_bytes = 0 + while received_bytes < uploading_file.content_length: + received_bytes = uploading_file.wait_for_total_received_bytes() + fraction_done = float( received_bytes ) / float( uploading_file.content_length ) - if fraction_done > fraction_reported + tick_increment: - yield '' % fraction_reported - fraction_reported = fraction_done + if fraction_done == 1.0 or fraction_done > fraction_reported + tick_increment: + fraction_reported = fraction_done + yield '' % fraction_reported - # TODO: write to the database + uploading_file.wait_for_complete() - if fraction_reported == 0: + if fraction_reported < 1.0: yield "An error occurred when uploading the file." return - # the file finished uploading, so fill out the progress meter to 100% - if fraction_reported < 1.0: - yield '' - - # the setTimeout() below ensures that the 100% progress bar is displayed for at least a moment yield \ u""" """ - upload.file.close() + return report( uploading_file, fraction_reported ) - # release the session lock before beginning the upload, because if the upload is cancelled - # before it's done, the lock won't be released - cherrypy.session.release_lock() + @expose( view = Json ) + @strongly_expire + @grab_user_id + @validate( + file_id = Valid_id(), + user_id = Valid_id( none_okay = True ), + ) + def stats( self, file_id, user_id = None ): + """ + Return information on a file that has been completely uploaded and is stored in the database. - return process_upload() + @type file_id: unicode + @param file_id: id of the file to report on + @type user_id: unicode or NoneType + @param user_id: id of current logged-in user (if any) + @rtype: dict + @return: { + 'filename': filename, + 'size_bytes': filesize, + } + @raise Access_error: the current user doesn't have access to the notebook that the file is in + """ + db_file = self.__database.load( File, file_id ) - def stats( file_id ): - pass + if not db_file or not self.__users.check_access( user_id, db_file.notebook_id ): + raise Access_error() + + return dict( + filename = db_file.filename, + size_bytes = db_file.size_bytes, + ) def rename( file_id, filename ): - pass + pass # TODO diff --git a/controller/Users.py b/controller/Users.py index 19c41b7..ee59c90 100644 --- a/controller/Users.py +++ b/controller/Users.py @@ -125,7 +125,8 @@ def grab_user_id( function ): arg_names = list( function.func_code.co_varnames ) if "user_id" in arg_names: arg_index = arg_names.index( "user_id" ) - args[ arg_index ] = cherrypy.session.get( "user_id" ) + args = list( args ) + args[ arg_index - 1 ] = cherrypy.session.get( "user_id" ) else: kwargs[ "user_id" ] = cherrypy.session.get( "user_id" ) diff --git a/controller/Validate.py b/controller/Validate.py index e136b6c..7e8681e 100644 --- a/controller/Validate.py +++ b/controller/Validate.py @@ -146,10 +146,12 @@ class Valid_int( object ): def __call__( self, value ): value = int( value ) - if self.min is not None and value < min: + if self.min is not None and value < self.min: self.message = "is too small" - if self.max is not None and value > max: + raise ValueError() + if self.max is not None and value > self.max: self.message = "is too large" + raise ValueError() return value diff --git a/controller/test/Test_controller.py b/controller/test/Test_controller.py index 5cbdcbd..55ec61b 100644 --- a/controller/test/Test_controller.py +++ b/controller/test/Test_controller.py @@ -7,6 +7,19 @@ from StringIO import StringIO from copy import copy +class Truncated_StringIO( StringIO ): + """ + A wrapper for StringIO that forcibly closes the file when only some of it has been read. Used + for simulating an upload that is canceled part of the way through. + """ + def readline( self, size = None ): + if self.tell() >= len( self.getvalue() ) * 0.25: + self.close() + return "" + + return StringIO.readline( self, 256 ) + + class Test_controller( object ): def __init__( self ): from model.User import User @@ -427,7 +440,7 @@ class Test_controller( object ): finally: request.close() - def http_upload( self, http_path, form_args, filename, file_data, headers = None, session_id = None ): + def http_upload( self, http_path, form_args, filename, file_data, simulate_cancel = False, headers = None, session_id = None ): """ Perform an HTTP POST with the given path on the test server, sending the provided form_args and file_data as a multipart form file upload. Return the result dict as returned by the @@ -452,16 +465,21 @@ class Test_controller( object ): headers = [] post_data = str( "".join( post_data ) ) - headers.extend( [ - ( "Content-Type", "multipart/form-data; boundary=%s" % boundary ), - ( "Content-Length", str( len( post_data ) ) ), - ] ) + headers.append( ( "Content-Type", "multipart/form-data; boundary=%s" % boundary ) ) + + if "Content-Length" not in [ name for ( name, value ) in headers ]: + headers.append( ( "Content-Length", str( len( post_data ) ) ) ) if session_id: headers.append( ( u"Cookie", "session_id=%s" % session_id ) ) # will break if unicode is used for the value + if simulate_cancel: + file_wrapper = Truncated_StringIO( post_data ) + else: + file_wrapper = StringIO( post_data ) + request = cherrypy.server.request( ( u"127.0.0.1", 1234 ), u"127.0.0.5" ) - response = request.run( "POST %s HTTP/1.0" % str( http_path ), headers = headers, rfile = StringIO( post_data ) ) + response = request.run( "POST %s HTTP/1.0" % str( http_path ), headers = headers, rfile = file_wrapper ) session_id = response.simple_cookie.get( u"session_id" ) if session_id: session_id = session_id.value diff --git a/controller/test/Test_files.py b/controller/test/Test_files.py index 61842b3..c8e1f55 100644 --- a/controller/test/Test_files.py +++ b/controller/test/Test_files.py @@ -68,7 +68,7 @@ class Test_files( Test_controller ): assert result.get( u"notebook_id" ) == self.notebook.object_id assert result.get( u"note_id" ) == self.note.object_id - def test_upload_file( self ): + def test_upload( self ): self.login() result = self.http_upload( @@ -101,12 +101,12 @@ class Test_files( Test_controller ): raise exc # assert that the progress bar is moving, and then completes - assert tick_count >= 3 + assert tick_count >= 2 assert tick_done # TODO: assert that the uploaded file actually got stored somewhere - def test_upload_file_without_login( self ): + def test_upload_without_login( self ): result = self.http_upload( "/files/upload", dict( @@ -120,7 +120,7 @@ class Test_files( Test_controller ): assert u"access" in result.get( u"body" )[ 0 ] - def test_upload_file_without_access( self ): + def test_upload_without_access( self ): self.login2() result = self.http_upload( @@ -136,7 +136,7 @@ class Test_files( Test_controller ): assert u"access" in result.get( u"body" )[ 0 ] - def assert_inline_error( self, result ): + def assert_streaming_error( self, result ): gen = result[ u"body" ] assert isinstance( gen, types.GeneratorType ) @@ -152,7 +152,7 @@ class Test_files( Test_controller ): assert found_error - def test_upload_file_unnamed( self ): + def test_upload_unnamed( self ): self.login() result = self.http_upload( @@ -166,9 +166,9 @@ class Test_files( Test_controller ): session_id = self.session_id, ) - self.assert_inline_error( result ) + self.assert_streaming_error( result ) - def test_upload_file_empty( self ): + def test_upload_empty( self ): self.login() result = self.http_upload( @@ -182,12 +182,43 @@ class Test_files( Test_controller ): session_id = self.session_id, ) - self.assert_inline_error( result ) + self.assert_streaming_error( result ) - def test_upload_file_cancel( self ): - raise NotImplementError() + def test_upload_invalid_content_length( self ): + self.login() - def test_upload_file_over_quota( self ): + result = self.http_upload( + "/files/upload", + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = self.file_data, + headers = [ ( "Content-Length", "-10" ) ], + session_id = self.session_id, + ) + + assert "invalid" in result[ "body" ][ 0 ] + + def test_upload_cancel( self ): + self.login() + + result = self.http_upload( + "/files/upload", + dict( + notebook_id = self.notebook.object_id, + note_id = self.note.object_id, + ), + filename = self.filename, + file_data = self.file_data, + simulate_cancel = True, + session_id = self.session_id, + ) + + self.assert_streaming_error( result ) + + def test_upload_over_quota( self ): raise NotImplementError() def login( self ): diff --git a/model/File.py b/model/File.py new file mode 100644 index 0000000..d209897 --- /dev/null +++ b/model/File.py @@ -0,0 +1,109 @@ +from Persistent import Persistent, quote +from psycopg2 import Binary +from StringIO import StringIO + + +class File( Persistent ): + """ + Metadata about an uploaded file. The actual file data is stored on the filesystem instead of in + the database. (Binary conversion to/from PostgreSQL's bytea is too slow, and the version of + psycopg2 I'm using doesn't have large object support.) + """ + def __init__( self, object_id, revision = None, notebook_id = None, note_id = None, + filename = None, size_bytes = None ): + """ + Create a File with the given id. + + @type object_id: unicode + @param object_id: id of the File + @type revision: datetime or NoneType + @param revision: revision timestamp of the object (optional, defaults to now) + @type notebook_id: unicode or NoneType + @param notebook_id: id of the notebook containing the file + @type note_id: unicode or NoneType + @param note_id: id of the note linking to the file + @type filename: unicode + @param filename: name of the file on the client + @type size_bytes: int + @param size_bytes: length of the file data in bytes + @rtype: File + @return: newly constructed File + """ + Persistent.__init__( self, object_id, revision ) + self.__notebook_id = notebook_id + self.__note_id = note_id + self.__filename = filename + self.__size_bytes = size_bytes + + @staticmethod + def create( object_id, notebook_id = None, note_id = None, filename = None, size_bytes = None ): + """ + Convenience constructor for creating a new File. + + @type object_id: unicode + @param object_id: id of the File + @type notebook_id: unicode or NoneType + @param notebook_id: id of the notebook containing the file + @type note_id: unicode or NoneType + @param note_id: id of the note linking to the file + @type filename: unicode + @param filename: name of the file on the client + @type size_bytes: int + @param size_bytes: length of the file data in bytes + @rtype: File + @return: newly constructed File + """ + return File( object_id, notebook_id = notebook_id, note_id = note_id, filename = filename, + size_bytes = size_bytes ) + + @staticmethod + def sql_load( object_id, revision = None ): + # Files don't store old revisions + if revision: + raise NotImplementedError() + + return \ + """ + select + file.id, file.revision, file.notebook_id, file.note_id, file.filename, size_bytes + from + file + where + file.id = %s; + """ % quote( object_id ) + + @staticmethod + def sql_id_exists( object_id, revision = None ): + if revision: + raise NotImplementedError() + + return "select id from file where id = %s;" % quote( object_id ) + + def sql_exists( self ): + return File.sql_id_exists( self.object_id ) + + def sql_create( self ): + return "insert into file ( id, revision, notebook_id, note_id, filename, size_bytes ) values ( %s, %s, %s, %s, %s, %s );" % \ + ( quote( self.object_id ), quote( self.revision ), quote( self.__notebook_id ), quote( self.__note_id ), + quote( self.__filename ), self.__size_bytes or 'null' ) + + def sql_update( self ): + return "update file set revision = %s, notebook_id = %s, note_id = %s, filename = %s, size_bytes = %s where id = %s;" % \ + ( quote( self.revision ), quote( self.__notebook_id ), quote( self.__note_id ), quote( self.__filename ), + self.__size_bytes or 'null', quote( self.object_id ) ) + + def to_dict( self ): + d = Persistent.to_dict( self ) + d.update( dict( + notebook_id = self.__notebook_id, + note_id = self.__note_id, + filename = self.__filename, + size_bytes = self.__size_bytes, + ) ) + + return d + + notebook_id = property( lambda self: self.__notebook_id ) + note_id = property( lambda self: self.__note_id ) + filename = property( lambda self: self.__filename ) + size_bytes = property( lambda self: self.__size_bytes ) diff --git a/model/Invite.py b/model/Invite.py index e1aa24a..8dc6f70 100644 --- a/model/Invite.py +++ b/model/Invite.py @@ -66,7 +66,7 @@ class Invite( Persistent ): @staticmethod def sql_load( object_id, revision = None ): - # password resets don't store old revisions + # invites don't store old revisions if revision: raise NotImplementedError() diff --git a/model/delta/1.2.0.sql b/model/delta/1.2.0.sql new file mode 100644 index 0000000..25f72d2 --- /dev/null +++ b/model/delta/1.2.0.sql @@ -0,0 +1,9 @@ +create table file ( + id text, + revision timestamp with time zone, + notebook_id text, + note_id text, + filename text, + size_bytes integer +); +alter table file add primary key ( id ); diff --git a/model/drop.sql b/model/drop.sql index 4998536..861170a 100644 --- a/model/drop.sql +++ b/model/drop.sql @@ -6,3 +6,5 @@ DROP VIEW notebook_current; DROP TABLE notebook; DROP TABLE password_reset; DROP TABLE user_notebook; +DROP TABLE invite; +DROP TABLE file; diff --git a/model/schema.sql b/model/schema.sql index 7fd6987..77d4b38 100644 --- a/model/schema.sql +++ b/model/schema.sql @@ -31,6 +31,22 @@ CREATE FUNCTION drop_html_tags(text) RETURNS text ALTER FUNCTION public.drop_html_tags(text) OWNER TO luminotes; +-- +-- Name: file; Type: TABLE; Schema: public; Owner: luminotes; Tablespace: +-- + +CREATE TABLE file ( + id text NOT NULL, + revision timestamp with time zone, + notebook_id text, + note_id text, + filename text, + size_bytes integer +); + + +ALTER TABLE public.file OWNER TO luminotes; + -- -- Name: invite; Type: TABLE; Schema: public; Owner: luminotes; Tablespace: -- @@ -161,6 +177,14 @@ CREATE TABLE user_notebook ( ALTER TABLE public.user_notebook OWNER TO luminotes; + +-- Name: file_pkey; Type: CONSTRAINT; Schema: public; Owner: luminotes; Tablespace: +-- + +ALTER TABLE ONLY file + ADD CONSTRAINT file_pkey PRIMARY KEY (id); + + -- -- Name: invite_pkey; Type: CONSTRAINT; Schema: public; Owner: luminotes; Tablespace: -- diff --git a/model/test/Test_file.py b/model/test/Test_file.py new file mode 100644 index 0000000..c091bef --- /dev/null +++ b/model/test/Test_file.py @@ -0,0 +1,33 @@ +from pytz import utc +from datetime import datetime, timedelta +from model.File import File + + +class Test_file( object ): + def setUp( self ): + self.object_id = u"17" + self.notebook_id = u"18" + self.note_id = u"19" + self.filename = u"foo.png" + self.size_bytes = 2888 + self.delta = timedelta( seconds = 1 ) + + self.file = File.create( self.object_id, self.notebook_id, self.note_id, self.filename, + self.size_bytes ) + + def test_create( self ): + assert self.file.object_id == self.object_id + assert self.file.notebook_id == self.notebook_id + assert self.file.note_id == self.note_id + assert self.file.filename == self.filename + assert self.file.size_bytes == self.size_bytes + + def test_to_dict( self ): + d = self.file.to_dict() + + assert d.get( "object_id" ) == self.object_id + assert datetime.now( tz = utc ) - d.get( "revision" ) < self.delta + assert d.get( "notebook_id" ) == self.notebook_id + assert d.get( "note_id" ) == self.note_id + assert d.get( "filename" ) == self.filename + assert d.get( "size_bytes" ) == self.size_bytes diff --git a/model/test/Test_invite.py b/model/test/Test_invite.py index 8dbe2a7..837fd88 100644 --- a/model/test/Test_invite.py +++ b/model/test/Test_invite.py @@ -1,6 +1,5 @@ from pytz import utc from datetime import datetime, timedelta -from model.User import User from model.Invite import Invite diff --git a/model/test/Test_password_reset.py b/model/test/Test_password_reset.py index 13bf801..08406a4 100644 --- a/model/test/Test_password_reset.py +++ b/model/test/Test_password_reset.py @@ -1,4 +1,3 @@ -from model.User import User from model.Password_reset import Password_reset diff --git a/static/css/style.css b/static/css/style.css index 80c1427..76edfa7 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -615,7 +615,7 @@ img { color: #ff6600; } -#upload_frame { +.upload_frame { padding: 0; margin: 0; width: 40em; diff --git a/static/js/Editor.js b/static/js/Editor.js index 7d30e9a..aafa02f 100644 --- a/static/js/Editor.js +++ b/static/js/Editor.js @@ -378,6 +378,12 @@ Editor.prototype.mouse_clicked = function ( event ) { return; } + // special case for links to uploaded files + if ( !link.target && /\/files\//.test( link.href ) ) { + location.href = link.href; + return; + } + event.stop(); // load the note corresponding to the clicked link diff --git a/static/js/Wiki.js b/static/js/Wiki.js index f2ffc62..6cc95a2 100644 --- a/static/js/Wiki.js +++ b/static/js/Wiki.js @@ -113,9 +113,13 @@ Wiki.prototype.update_next_id = function ( result ) { var KILOBYTE = 1024; var MEGABYTE = 1024 * KILOBYTE; -function bytes_to_megabytes( bytes, or_kilobytes ) { - if ( or_kilobytes && bytes < MEGABYTE ) - return Math.round( bytes / KILOBYTE ) + " KB"; +function bytes_to_megabytes( bytes, choose_units ) { + if ( choose_units ) { + if ( bytes < KILOBYTE ) + return bytes + " bytes"; + if ( bytes < MEGABYTE ) + return Math.round( bytes / KILOBYTE ) + " KB"; + } return Math.round( bytes / MEGABYTE ) + " MB"; } @@ -731,7 +735,7 @@ Wiki.prototype.display_link_pulldown = function ( editor, link ) { if ( link_title( link ).length > 0 ) { if ( !pulldown ) { this.clear_pulldowns(); - // display a different pulldown dependong on whether the link is a note link or a file link + // display a different pulldown depending on whether the link is a note link or a file link if ( link.target || !/\/files\//.test( link.href ) ) new Link_pulldown( this, this.notebook_id, this.invoker, editor, link ); else @@ -2246,9 +2250,11 @@ function Upload_pulldown( wiki, notebook_id, invoker, editor ) { "frameBorder": "0", "scrolling": "no", "id": "upload_frame", - "name": "upload_frame" + "name": "upload_frame", + "class": "upload_frame" } ); this.iframe.pulldown = this; + this.file_id = null; var self = this; connect( this.iframe, "onload", function ( event ) { self.init_frame(); } ); @@ -2266,26 +2272,47 @@ Upload_pulldown.prototype.init_frame = function () { withDocument( doc, function () { connect( "upload_button", "onclick", function ( event ) { withDocument( doc, function () { - self.upload_started( getElement( "upload" ).value ); + self.upload_started( getElement( "file_id" ).value, getElement( "upload" ).value ); } ); } ); } ); } -Upload_pulldown.prototype.upload_started = function ( filename ) { +Upload_pulldown.prototype.upload_started = function ( file_id, filename ) { + this.file_id = file_id; + + // make the upload iframe invisible but still present so that the upload continues + addElementClass( this.iframe, "invisible" ); + setElementDimensions( this.iframe, { "h": "0" } ); + // get the basename of the file var pieces = filename.split( "/" ); filename = pieces[ pieces.length - 1 ]; pieces = filename.split( "\\" ); filename = pieces[ pieces.length - 1 ]; - // the current title is blank, replace the title with the upload's filename + // if the current title is blank, replace the title with the upload's filename if ( link_title( this.link ) == "" ) replaceChildNodes( this.link, this.editor.document.createTextNode( filename ) ); - // TODO: set the link's href to the file + + // FIXME: this call might occur before upload() is even called + var progress_iframe = createDOM( "iframe", { + "src": "/files/progress?file_id=" + file_id + "&filename=" + escape( filename ), + "frameBorder": "0", + "scrolling": "no", + "id": "progress_frame", + "name": "progress_frame", + "class": "upload_frame" + } ); + + appendChildNodes( this.div, progress_iframe ); } Upload_pulldown.prototype.upload_complete = function () { + // now that the upload is done, the file link should point to the uploaded file + this.link.href = "/files/download?file_id=" + this.file_id + +// FIXME: the upload pulldown is sometimes being closed here before the upload is complete, thereby truncating the upload new File_link_pulldown( this.wiki, this.notebook_id, this.invoker, this.editor, this.link ); this.shutdown(); } diff --git a/view/Upload_page.py b/view/Upload_page.py index 861ab4b..18b33de 100644 --- a/view/Upload_page.py +++ b/view/Upload_page.py @@ -2,7 +2,7 @@ from Tags import Html, Head, Link, Meta, Body, P, Form, Span, Input class Upload_page( Html ): - def __init__( self, notebook_id, note_id ): + def __init__( self, notebook_id, note_id, file_id ): Html.__init__( self, Head( @@ -12,15 +12,16 @@ class Upload_page( Html ): Body( Form( Span( u"attach file: ", class_ = u"field_label" ), - Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ), - Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ), Input( type = u"hidden", id = u"notebook_id", name = u"notebook_id", value = notebook_id ), Input( type = u"hidden", id = u"note_id", name = u"note_id", value = note_id ), - action = u"/files/upload", + Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ), + Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ), + action = u"/files/upload?file_id=%s" % file_id, method = u"post", enctype = u"multipart/form-data", ), P( u"Please select a file to upload." ), Span( id = u"tick_preload" ), + Input( type = u"hidden", id = u"file_id", value = file_id ), ), )