More work on file uploading. Unit tests need to be fixed.
This commit is contained in:
parent
43f098cda0
commit
731dc52623
3
NEWS
3
NEWS
|
@ -1,3 +1,6 @@
|
|||
1.2.0: February ??, 2008
|
||||
* Users can now upload files to attach to their notes.
|
||||
|
||||
1.1.3: January 28, 2008
|
||||
* Now, if you delete a notebook and the only remaining notebook is read-only,
|
||||
then a new read-write notebook is created for you automatically.
|
||||
|
|
|
@ -63,7 +63,11 @@ settings = {
|
|||
"""
|
||||
""",
|
||||
},
|
||||
"/files/upload": {
|
||||
"/files/download": {
|
||||
"stream_response": True,
|
||||
"encoding_filter.on": False,
|
||||
},
|
||||
"/files/progress": {
|
||||
"stream_response": True
|
||||
},
|
||||
}
|
||||
|
|
|
@ -1,11 +1,17 @@
|
|||
import cgi
|
||||
import time
|
||||
import tempfile
|
||||
import cherrypy
|
||||
from threading import Lock, Event
|
||||
from Expose import expose
|
||||
from Validate import validate
|
||||
from Validate import validate, Valid_int, Validation_error
|
||||
from Database import Valid_id
|
||||
from Users import grab_user_id
|
||||
from Expire import strongly_expire
|
||||
from model.File import File
|
||||
from view.Upload_page import Upload_page
|
||||
from view.Blank_page import Blank_page
|
||||
from view.Json import Json
|
||||
|
||||
|
||||
class Access_error( Exception ):
|
||||
|
@ -36,6 +42,153 @@ class Upload_error( Exception ):
|
|||
)
|
||||
|
||||
|
||||
# map of upload id to Upload_file
|
||||
current_uploads = {}
|
||||
current_uploads_lock = Lock()
|
||||
|
||||
|
||||
class Upload_file( object ):
|
||||
"""
|
||||
File-like object for storing file uploads.
|
||||
"""
|
||||
def __init__( self, file_id, filename, content_length ):
|
||||
self.__file = file( self.make_server_filename( file_id ), "w+" )
|
||||
self.__file_id = file_id
|
||||
self.__filename = filename
|
||||
self.__content_length = content_length
|
||||
self.__file_received_bytes = 0
|
||||
self.__total_received_bytes = cherrypy.request.rfile.bytes_read
|
||||
self.__total_received_bytes_updated = Event()
|
||||
self.__complete = Event()
|
||||
|
||||
def write( self, data ):
|
||||
self.__file.write( data )
|
||||
self.__file_received_bytes += len( data )
|
||||
self.__total_received_bytes = cherrypy.request.rfile.bytes_read
|
||||
self.__total_received_bytes_updated.set()
|
||||
|
||||
def tell( self ):
|
||||
return self.__file.tell()
|
||||
|
||||
def seek( self, position ):
|
||||
self.__file.seek( position )
|
||||
|
||||
def read( self, size = None ):
|
||||
if size is None:
|
||||
return self.__file.read()
|
||||
|
||||
return self.__file.read( size )
|
||||
|
||||
def wait_for_total_received_bytes( self ):
|
||||
self.__total_received_bytes_updated.wait( timeout = cherrypy.server.socket_timeout )
|
||||
self.__total_received_bytes_updated.clear()
|
||||
return self.__total_received_bytes
|
||||
|
||||
def close( self ):
|
||||
self.__file.close()
|
||||
self.__complete.set()
|
||||
|
||||
def wait_for_complete( self ):
|
||||
self.__complete.wait( timeout = cherrypy.server.socket_timeout )
|
||||
|
||||
@staticmethod
|
||||
def make_server_filename( file_id ):
|
||||
return u"files/%s" % file_id
|
||||
|
||||
filename = property( lambda self: self.__filename )
|
||||
|
||||
# expected byte count of the entire form upload, including the file and other form parameters
|
||||
content_length = property( lambda self: self.__content_length )
|
||||
|
||||
# count of bytes received thus far for this file upload only
|
||||
file_received_bytes = property( lambda self: self.__file_received_bytes )
|
||||
|
||||
# count of bytes received thus far for the form upload, including the file and other form
|
||||
# parameters
|
||||
total_received_bytes = property( lambda self: self.__total_received_bytes )
|
||||
|
||||
|
||||
class FieldStorage( cherrypy._cpcgifs.FieldStorage ):
|
||||
"""
|
||||
Derived from cherrypy._cpcgifs.FieldStorage, which is in turn derived from cgi.FieldStorage, which
|
||||
calls make_file() to create a temporary file where file uploads are stored. By wrapping this file
|
||||
object, we can track its progress as its written. Inspired by:
|
||||
http://www.cherrypy.org/attachment/ticket/546/uploadfilter.py
|
||||
|
||||
This method relies on a file_id parameter being present in the HTTP query string.
|
||||
|
||||
@type binary: NoneType
|
||||
@param binary: ignored
|
||||
@type user_id: unicode or NoneType
|
||||
@param user_id: id of current logged-in user (if any)
|
||||
@rtype: Upload_file
|
||||
@return: wrapped temporary file used to store the upload
|
||||
@raise Upload_error: the provided file_id value is invalid, or the filename or Content-Length is
|
||||
missing
|
||||
"""
|
||||
def make_file( self, binary = None, user_id = None ):
|
||||
global current_uploads, current_uploads_lock
|
||||
|
||||
cherrypy.server.max_request_body_size = 0 # remove CherryPy default file size limit of 100 MB
|
||||
cherrypy.response.timeout = 3600 * 2 # increase upload timeout to 2 hours (default is 5 min)
|
||||
cherrypy.server.socket_timeout = 60 # increase socket timeout to one minute (default is 10 sec)
|
||||
DASHES_AND_NEWLINES = 6 # four dashes and two newlines
|
||||
|
||||
# release the cherrypy session lock so that the user can issue other commands while the file is
|
||||
# uploading
|
||||
cherrypy.session.release_lock()
|
||||
|
||||
# pluck the file id out of the query string. it would be preferable to grab it out of parsed
|
||||
# form variables instead, but at this point in the processing, all the form variables might not
|
||||
# be parsed
|
||||
file_id = cgi.parse_qs( cherrypy.request.query_string ).get( u"file_id", [ None ] )[ 0 ]
|
||||
try:
|
||||
file_id = Valid_id()( file_id )
|
||||
except ValueError:
|
||||
raise Upload_error( "The file_id is invalid." )
|
||||
|
||||
if not self.filename:
|
||||
raise Upload_error( "Please provide a filename." )
|
||||
|
||||
content_length = cherrypy.request.headers.get( "content-length", 0 )
|
||||
try:
|
||||
content_length = Valid_int( min = 0 )( content_length ) - len( self.outerboundary ) - DASHES_AND_NEWLINES
|
||||
except ValueError:
|
||||
raise Upload_error( "The Content-Length header value is invalid." )
|
||||
|
||||
# file size is the entire content length of the POST, minus the size of the other form
|
||||
# parameters and boundaries. note: this assumes that the uploaded file is sent as the last
|
||||
# form parameter in the POST
|
||||
# TODO: verify that the uploaded file is always sent as the last parameter
|
||||
existing_file = current_uploads.get( file_id )
|
||||
if existing_file:
|
||||
existing_file.close()
|
||||
|
||||
upload_file = Upload_file( file_id, self.filename.strip(), content_length )
|
||||
|
||||
current_uploads_lock.acquire()
|
||||
try:
|
||||
current_uploads[ file_id ] = upload_file
|
||||
finally:
|
||||
current_uploads_lock.release()
|
||||
|
||||
return upload_file
|
||||
|
||||
def __write( self, line ):
|
||||
"""
|
||||
This implementation of __write() is different than that of the base class, because it calls
|
||||
make_file() whenever there is a filename instead of only for large enough files.
|
||||
"""
|
||||
if self.__file is not None and self.filename:
|
||||
self.file = self.make_file( '' )
|
||||
self.file.write( self.__file.getvalue() )
|
||||
self.__file = None
|
||||
|
||||
self.file.write( line )
|
||||
|
||||
cherrypy._cpcgifs.FieldStorage = FieldStorage
|
||||
|
||||
|
||||
class Files( object ):
|
||||
"""
|
||||
Controller for dealing with uploaded files, corresponding to the "/files" URL.
|
||||
|
@ -54,39 +207,96 @@ class Files( object ):
|
|||
self.__database = database
|
||||
self.__users = users
|
||||
|
||||
@expose()
|
||||
@grab_user_id
|
||||
@validate(
|
||||
file_id = Valid_id(),
|
||||
user_id = Valid_id( none_okay = True ),
|
||||
)
|
||||
def download( self, file_id, user_id = None ):
|
||||
"""
|
||||
Return the contents of file that a user has previously uploaded.
|
||||
|
||||
@type file_id: unicode
|
||||
@param file_id: id of the file to download
|
||||
@type user_id: unicode or NoneType
|
||||
@param user_id: id of current logged-in user (if any)
|
||||
@rtype: unicode
|
||||
@return: file data
|
||||
@raise Access_error: the current user doesn't have access to the notebook that the file is in
|
||||
"""
|
||||
db_file = self.__database.load( File, file_id )
|
||||
|
||||
if not db_file or not self.__users.check_access( user_id, db_file.notebook_id ):
|
||||
raise Access_error()
|
||||
|
||||
db_file = self.__database.load( File, file_id )
|
||||
|
||||
cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=%s" % db_file.filename
|
||||
cherrypy.response.headerMap[ u"Content-Length" ] = db_file.size_bytes
|
||||
# TODO: send content type
|
||||
# cherrypy.response.headerMap[ u"Content-Type" ] = u"image/png"
|
||||
|
||||
def stream():
|
||||
CHUNK_SIZE = 8192
|
||||
local_file = file( Upload_file.make_server_filename( file_id ) )
|
||||
|
||||
while True:
|
||||
data = local_file.read( CHUNK_SIZE )
|
||||
if len( data ) == 0: break
|
||||
yield data
|
||||
|
||||
return stream()
|
||||
|
||||
|
||||
@expose( view = Upload_page )
|
||||
@strongly_expire
|
||||
@grab_user_id
|
||||
@validate(
|
||||
notebook_id = Valid_id(),
|
||||
note_id = Valid_id(),
|
||||
user_id = Valid_id( none_okay = True ),
|
||||
)
|
||||
def upload_page( self, notebook_id, note_id ):
|
||||
def upload_page( self, notebook_id, note_id, user_id ):
|
||||
"""
|
||||
Provide the information necessary to display the file upload page.
|
||||
Provide the information necessary to display the file upload page, including the generation of a
|
||||
unique file id.
|
||||
|
||||
@type notebook_id: unicode
|
||||
@param notebook_id: id of the notebook that the upload will be to
|
||||
@type note_id: unicode
|
||||
@param note_id: id of the note that the upload will be to
|
||||
@type user_id: unicode or NoneType
|
||||
@param user_id: id of current logged-in user (if any)
|
||||
@rtype: unicode
|
||||
@return: rendered HTML page
|
||||
@raise Access_error: the current user doesn't have access to the given notebook
|
||||
"""
|
||||
if not self.__users.check_access( user_id, notebook_id, read_write = True ):
|
||||
raise Access_error()
|
||||
|
||||
file_id = self.__database.next_id( File )
|
||||
|
||||
return dict(
|
||||
notebook_id = notebook_id,
|
||||
note_id = note_id,
|
||||
file_id = file_id,
|
||||
)
|
||||
|
||||
@expose()
|
||||
@expose( view = Blank_page )
|
||||
@strongly_expire
|
||||
@grab_user_id
|
||||
@validate(
|
||||
upload = (),
|
||||
notebook_id = Valid_id(),
|
||||
note_id = Valid_id(),
|
||||
file_id = Valid_id(),
|
||||
user_id = Valid_id( none_okay = True ),
|
||||
)
|
||||
def upload( self, upload, notebook_id, note_id, user_id ):
|
||||
def upload( self, upload, notebook_id, note_id, file_id, user_id ):
|
||||
"""
|
||||
Upload a file from the client for attachment to a particular note.
|
||||
Upload a file from the client for attachment to a particular note. The file_id must be provided
|
||||
as part of the query string, even if the other values are submitted as form data.
|
||||
|
||||
@type upload: cgi.FieldStorage
|
||||
@param upload: file handle to uploaded file
|
||||
|
@ -94,40 +304,90 @@ class Files( object ):
|
|||
@param notebook_id: id of the notebook that the upload is to
|
||||
@type note_id: unicode
|
||||
@param note_id: id of the note that the upload is to
|
||||
@raise Access_error: the current user doesn't have access to the given notebook or note
|
||||
@raise Upload_error: an error occurred when processing the uploaded file
|
||||
@type file_id: unicode
|
||||
@param file_id: id of the file being uploaded
|
||||
@type user_id: unicode or NoneType
|
||||
@param user_id: id of current logged-in user (if any)
|
||||
@rtype: unicode
|
||||
@return: rendered HTML page
|
||||
@raise Access_error: the current user doesn't have access to the given notebook or note
|
||||
@raise Upload_error: the Content-Length header value is invalid
|
||||
"""
|
||||
if not self.__users.check_access( user_id, notebook_id ):
|
||||
global current_uploads, current_uploads_lock
|
||||
|
||||
if not self.__users.check_access( user_id, notebook_id, read_write = True ):
|
||||
raise Access_error()
|
||||
|
||||
cherrypy.server.max_request_body_size = 0 # remove file size limit of 100 MB
|
||||
cherrypy.response.timeout = 3600 # increase upload timeout to one hour (default is 5 min)
|
||||
cherrypy.server.socket_timeout = 60 # increase socket timeout to one minute (default is 10 sec)
|
||||
CHUNK_SIZE = 8 * 1024 # 8 Kb
|
||||
# write the file to the database
|
||||
uploaded_file = current_uploads.get( file_id )
|
||||
if not uploaded_file:
|
||||
raise Upload_error()
|
||||
|
||||
headers = {}
|
||||
for key, val in cherrypy.request.headers.iteritems():
|
||||
headers[ key.lower() ] = val
|
||||
# TODO: grab content type and store it
|
||||
#print upload.headers.get( "content-type", "MISSING" )
|
||||
|
||||
# TODO: somehow detect when upload is canceled and abort
|
||||
|
||||
db_file = File.create( file_id, notebook_id, note_id, uploaded_file.filename, uploaded_file.file_received_bytes )
|
||||
self.__database.save( db_file )
|
||||
uploaded_file.close()
|
||||
|
||||
current_uploads_lock.acquire()
|
||||
try:
|
||||
file_size = int( headers.get( "content-length", 0 ) )
|
||||
except ValueError:
|
||||
raise Upload_error()
|
||||
if file_size <= 0:
|
||||
raise Upload_error()
|
||||
del( current_uploads[ file_id ] )
|
||||
finally:
|
||||
current_uploads_lock.release()
|
||||
|
||||
filename = upload.filename.strip()
|
||||
return dict()
|
||||
|
||||
def process_upload():
|
||||
@expose()
|
||||
@strongly_expire
|
||||
@validate(
|
||||
file_id = Valid_id(),
|
||||
filename = unicode,
|
||||
)
|
||||
def progress( self, file_id, filename ):
|
||||
"""
|
||||
Stream information on a file that is in the process of being uploaded. This method does not
|
||||
perform any access checks, but the only information streamed is a progress bar and upload
|
||||
percentage.
|
||||
|
||||
@type file_id: unicode
|
||||
@param file_id: id of a currently uploading file
|
||||
@type filename: unicode
|
||||
@param filename: name of the file to report on
|
||||
@rtype: unicode
|
||||
@return: streaming HTML progress bar
|
||||
"""
|
||||
# release the session lock before beginning to stream the upload report. otherwise, if the
|
||||
# upload is cancelled before it's done, the lock won't be released
|
||||
cherrypy.session.release_lock()
|
||||
|
||||
# poll until the file is uploading (as determined by current_uploads) or completely uploaded (in
|
||||
# the database with a filename)
|
||||
while True:
|
||||
uploading_file = current_uploads.get( file_id )
|
||||
db_file = None
|
||||
|
||||
if uploading_file:
|
||||
fraction_reported = 0.0
|
||||
break
|
||||
|
||||
db_file = self.__database.load( File, file_id )
|
||||
if not db_file:
|
||||
raise Upload_error( u"The file id is unknown" )
|
||||
if db_file.filename is None:
|
||||
time.sleep( 0.1 )
|
||||
continue
|
||||
fraction_reported = 1.0
|
||||
break
|
||||
|
||||
# TODO: maybe move this to the view/ directory
|
||||
def report( uploading_file, fraction_reported ):
|
||||
"""
|
||||
Process the file upload while streaming a progress meter as it uploads.
|
||||
Stream a progress meter as it uploads.
|
||||
"""
|
||||
progress_bytes = 0
|
||||
fraction_reported = 0.0
|
||||
progress_width_em = 20
|
||||
tick_increment = 0.01
|
||||
progress_bar = u'<img src="/static/images/tick.png" style="width: %sem; height: 1em;" id="progress_bar" />' % \
|
||||
|
@ -144,15 +404,6 @@ class Files( object ):
|
|||
<body>
|
||||
"""
|
||||
|
||||
if not filename:
|
||||
yield \
|
||||
u"""
|
||||
<div class="field_label">upload error: </div>
|
||||
Please check that the filename is valid.
|
||||
</body></html>
|
||||
"""
|
||||
return
|
||||
|
||||
base_filename = filename.split( u"/" )[ -1 ].split( u"\\" )[ -1 ]
|
||||
yield \
|
||||
u"""
|
||||
|
@ -180,46 +431,64 @@ class Files( object ):
|
|||
</script>
|
||||
""" % ( cgi.escape( base_filename ), progress_bar, progress_width_em )
|
||||
|
||||
while True:
|
||||
chunk = upload.file.read( CHUNK_SIZE )
|
||||
if not chunk: break
|
||||
progress_bytes += len( chunk )
|
||||
fraction_done = float( progress_bytes ) / float( file_size )
|
||||
if uploading_file:
|
||||
received_bytes = 0
|
||||
while received_bytes < uploading_file.content_length:
|
||||
received_bytes = uploading_file.wait_for_total_received_bytes()
|
||||
fraction_done = float( received_bytes ) / float( uploading_file.content_length )
|
||||
|
||||
if fraction_done > fraction_reported + tick_increment:
|
||||
yield '<script type="text/javascript">tick(%s);</script>' % fraction_reported
|
||||
fraction_reported = fraction_done
|
||||
if fraction_done == 1.0 or fraction_done > fraction_reported + tick_increment:
|
||||
fraction_reported = fraction_done
|
||||
yield '<script type="text/javascript">tick(%s);</script>' % fraction_reported
|
||||
|
||||
# TODO: write to the database
|
||||
uploading_file.wait_for_complete()
|
||||
|
||||
if fraction_reported == 0:
|
||||
if fraction_reported < 1.0:
|
||||
yield "An error occurred when uploading the file.</body></html>"
|
||||
return
|
||||
|
||||
# the file finished uploading, so fill out the progress meter to 100%
|
||||
if fraction_reported < 1.0:
|
||||
yield '<script type="text/javascript">tick(1.0);</script>'
|
||||
|
||||
# the setTimeout() below ensures that the 100% progress bar is displayed for at least a moment
|
||||
yield \
|
||||
u"""
|
||||
<script type="text/javascript">
|
||||
setTimeout( 'withDocument( window.parent.document, function () { getElement( "upload_frame" ).pulldown.upload_complete(); } );', 10 );
|
||||
withDocument( window.parent.document, function () { getElement( "upload_frame" ).pulldown.upload_complete(); } );
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
upload.file.close()
|
||||
return report( uploading_file, fraction_reported )
|
||||
|
||||
# release the session lock before beginning the upload, because if the upload is cancelled
|
||||
# before it's done, the lock won't be released
|
||||
cherrypy.session.release_lock()
|
||||
@expose( view = Json )
|
||||
@strongly_expire
|
||||
@grab_user_id
|
||||
@validate(
|
||||
file_id = Valid_id(),
|
||||
user_id = Valid_id( none_okay = True ),
|
||||
)
|
||||
def stats( self, file_id, user_id = None ):
|
||||
"""
|
||||
Return information on a file that has been completely uploaded and is stored in the database.
|
||||
|
||||
return process_upload()
|
||||
@type file_id: unicode
|
||||
@param file_id: id of the file to report on
|
||||
@type user_id: unicode or NoneType
|
||||
@param user_id: id of current logged-in user (if any)
|
||||
@rtype: dict
|
||||
@return: {
|
||||
'filename': filename,
|
||||
'size_bytes': filesize,
|
||||
}
|
||||
@raise Access_error: the current user doesn't have access to the notebook that the file is in
|
||||
"""
|
||||
db_file = self.__database.load( File, file_id )
|
||||
|
||||
def stats( file_id ):
|
||||
pass
|
||||
if not db_file or not self.__users.check_access( user_id, db_file.notebook_id ):
|
||||
raise Access_error()
|
||||
|
||||
return dict(
|
||||
filename = db_file.filename,
|
||||
size_bytes = db_file.size_bytes,
|
||||
)
|
||||
|
||||
def rename( file_id, filename ):
|
||||
pass
|
||||
pass # TODO
|
||||
|
|
|
@ -125,7 +125,8 @@ def grab_user_id( function ):
|
|||
arg_names = list( function.func_code.co_varnames )
|
||||
if "user_id" in arg_names:
|
||||
arg_index = arg_names.index( "user_id" )
|
||||
args[ arg_index ] = cherrypy.session.get( "user_id" )
|
||||
args = list( args )
|
||||
args[ arg_index - 1 ] = cherrypy.session.get( "user_id" )
|
||||
else:
|
||||
kwargs[ "user_id" ] = cherrypy.session.get( "user_id" )
|
||||
|
||||
|
|
|
@ -146,10 +146,12 @@ class Valid_int( object ):
|
|||
def __call__( self, value ):
|
||||
value = int( value )
|
||||
|
||||
if self.min is not None and value < min:
|
||||
if self.min is not None and value < self.min:
|
||||
self.message = "is too small"
|
||||
if self.max is not None and value > max:
|
||||
raise ValueError()
|
||||
if self.max is not None and value > self.max:
|
||||
self.message = "is too large"
|
||||
raise ValueError()
|
||||
|
||||
return value
|
||||
|
||||
|
|
|
@ -7,6 +7,19 @@ from StringIO import StringIO
|
|||
from copy import copy
|
||||
|
||||
|
||||
class Truncated_StringIO( StringIO ):
|
||||
"""
|
||||
A wrapper for StringIO that forcibly closes the file when only some of it has been read. Used
|
||||
for simulating an upload that is canceled part of the way through.
|
||||
"""
|
||||
def readline( self, size = None ):
|
||||
if self.tell() >= len( self.getvalue() ) * 0.25:
|
||||
self.close()
|
||||
return ""
|
||||
|
||||
return StringIO.readline( self, 256 )
|
||||
|
||||
|
||||
class Test_controller( object ):
|
||||
def __init__( self ):
|
||||
from model.User import User
|
||||
|
@ -427,7 +440,7 @@ class Test_controller( object ):
|
|||
finally:
|
||||
request.close()
|
||||
|
||||
def http_upload( self, http_path, form_args, filename, file_data, headers = None, session_id = None ):
|
||||
def http_upload( self, http_path, form_args, filename, file_data, simulate_cancel = False, headers = None, session_id = None ):
|
||||
"""
|
||||
Perform an HTTP POST with the given path on the test server, sending the provided form_args
|
||||
and file_data as a multipart form file upload. Return the result dict as returned by the
|
||||
|
@ -452,16 +465,21 @@ class Test_controller( object ):
|
|||
headers = []
|
||||
|
||||
post_data = str( "".join( post_data ) )
|
||||
headers.extend( [
|
||||
( "Content-Type", "multipart/form-data; boundary=%s" % boundary ),
|
||||
( "Content-Length", str( len( post_data ) ) ),
|
||||
] )
|
||||
headers.append( ( "Content-Type", "multipart/form-data; boundary=%s" % boundary ) )
|
||||
|
||||
if "Content-Length" not in [ name for ( name, value ) in headers ]:
|
||||
headers.append( ( "Content-Length", str( len( post_data ) ) ) )
|
||||
|
||||
if session_id:
|
||||
headers.append( ( u"Cookie", "session_id=%s" % session_id ) ) # will break if unicode is used for the value
|
||||
|
||||
if simulate_cancel:
|
||||
file_wrapper = Truncated_StringIO( post_data )
|
||||
else:
|
||||
file_wrapper = StringIO( post_data )
|
||||
|
||||
request = cherrypy.server.request( ( u"127.0.0.1", 1234 ), u"127.0.0.5" )
|
||||
response = request.run( "POST %s HTTP/1.0" % str( http_path ), headers = headers, rfile = StringIO( post_data ) )
|
||||
response = request.run( "POST %s HTTP/1.0" % str( http_path ), headers = headers, rfile = file_wrapper )
|
||||
session_id = response.simple_cookie.get( u"session_id" )
|
||||
if session_id: session_id = session_id.value
|
||||
|
||||
|
|
|
@ -68,7 +68,7 @@ class Test_files( Test_controller ):
|
|||
assert result.get( u"notebook_id" ) == self.notebook.object_id
|
||||
assert result.get( u"note_id" ) == self.note.object_id
|
||||
|
||||
def test_upload_file( self ):
|
||||
def test_upload( self ):
|
||||
self.login()
|
||||
|
||||
result = self.http_upload(
|
||||
|
@ -101,12 +101,12 @@ class Test_files( Test_controller ):
|
|||
raise exc
|
||||
|
||||
# assert that the progress bar is moving, and then completes
|
||||
assert tick_count >= 3
|
||||
assert tick_count >= 2
|
||||
assert tick_done
|
||||
|
||||
# TODO: assert that the uploaded file actually got stored somewhere
|
||||
|
||||
def test_upload_file_without_login( self ):
|
||||
def test_upload_without_login( self ):
|
||||
result = self.http_upload(
|
||||
"/files/upload",
|
||||
dict(
|
||||
|
@ -120,7 +120,7 @@ class Test_files( Test_controller ):
|
|||
|
||||
assert u"access" in result.get( u"body" )[ 0 ]
|
||||
|
||||
def test_upload_file_without_access( self ):
|
||||
def test_upload_without_access( self ):
|
||||
self.login2()
|
||||
|
||||
result = self.http_upload(
|
||||
|
@ -136,7 +136,7 @@ class Test_files( Test_controller ):
|
|||
|
||||
assert u"access" in result.get( u"body" )[ 0 ]
|
||||
|
||||
def assert_inline_error( self, result ):
|
||||
def assert_streaming_error( self, result ):
|
||||
gen = result[ u"body" ]
|
||||
assert isinstance( gen, types.GeneratorType )
|
||||
|
||||
|
@ -152,7 +152,7 @@ class Test_files( Test_controller ):
|
|||
|
||||
assert found_error
|
||||
|
||||
def test_upload_file_unnamed( self ):
|
||||
def test_upload_unnamed( self ):
|
||||
self.login()
|
||||
|
||||
result = self.http_upload(
|
||||
|
@ -166,9 +166,9 @@ class Test_files( Test_controller ):
|
|||
session_id = self.session_id,
|
||||
)
|
||||
|
||||
self.assert_inline_error( result )
|
||||
self.assert_streaming_error( result )
|
||||
|
||||
def test_upload_file_empty( self ):
|
||||
def test_upload_empty( self ):
|
||||
self.login()
|
||||
|
||||
result = self.http_upload(
|
||||
|
@ -182,12 +182,43 @@ class Test_files( Test_controller ):
|
|||
session_id = self.session_id,
|
||||
)
|
||||
|
||||
self.assert_inline_error( result )
|
||||
self.assert_streaming_error( result )
|
||||
|
||||
def test_upload_file_cancel( self ):
|
||||
raise NotImplementError()
|
||||
def test_upload_invalid_content_length( self ):
|
||||
self.login()
|
||||
|
||||
def test_upload_file_over_quota( self ):
|
||||
result = self.http_upload(
|
||||
"/files/upload",
|
||||
dict(
|
||||
notebook_id = self.notebook.object_id,
|
||||
note_id = self.note.object_id,
|
||||
),
|
||||
filename = self.filename,
|
||||
file_data = self.file_data,
|
||||
headers = [ ( "Content-Length", "-10" ) ],
|
||||
session_id = self.session_id,
|
||||
)
|
||||
|
||||
assert "invalid" in result[ "body" ][ 0 ]
|
||||
|
||||
def test_upload_cancel( self ):
|
||||
self.login()
|
||||
|
||||
result = self.http_upload(
|
||||
"/files/upload",
|
||||
dict(
|
||||
notebook_id = self.notebook.object_id,
|
||||
note_id = self.note.object_id,
|
||||
),
|
||||
filename = self.filename,
|
||||
file_data = self.file_data,
|
||||
simulate_cancel = True,
|
||||
session_id = self.session_id,
|
||||
)
|
||||
|
||||
self.assert_streaming_error( result )
|
||||
|
||||
def test_upload_over_quota( self ):
|
||||
raise NotImplementError()
|
||||
|
||||
def login( self ):
|
||||
|
|
|
@ -0,0 +1,109 @@
|
|||
from Persistent import Persistent, quote
|
||||
from psycopg2 import Binary
|
||||
from StringIO import StringIO
|
||||
|
||||
|
||||
class File( Persistent ):
|
||||
"""
|
||||
Metadata about an uploaded file. The actual file data is stored on the filesystem instead of in
|
||||
the database. (Binary conversion to/from PostgreSQL's bytea is too slow, and the version of
|
||||
psycopg2 I'm using doesn't have large object support.)
|
||||
"""
|
||||
def __init__( self, object_id, revision = None, notebook_id = None, note_id = None,
|
||||
filename = None, size_bytes = None ):
|
||||
"""
|
||||
Create a File with the given id.
|
||||
|
||||
@type object_id: unicode
|
||||
@param object_id: id of the File
|
||||
@type revision: datetime or NoneType
|
||||
@param revision: revision timestamp of the object (optional, defaults to now)
|
||||
@type notebook_id: unicode or NoneType
|
||||
@param notebook_id: id of the notebook containing the file
|
||||
@type note_id: unicode or NoneType
|
||||
@param note_id: id of the note linking to the file
|
||||
@type filename: unicode
|
||||
@param filename: name of the file on the client
|
||||
@type size_bytes: int
|
||||
@param size_bytes: length of the file data in bytes
|
||||
@rtype: File
|
||||
@return: newly constructed File
|
||||
"""
|
||||
Persistent.__init__( self, object_id, revision )
|
||||
self.__notebook_id = notebook_id
|
||||
self.__note_id = note_id
|
||||
self.__filename = filename
|
||||
self.__size_bytes = size_bytes
|
||||
|
||||
@staticmethod
|
||||
def create( object_id, notebook_id = None, note_id = None, filename = None, size_bytes = None ):
|
||||
"""
|
||||
Convenience constructor for creating a new File.
|
||||
|
||||
@type object_id: unicode
|
||||
@param object_id: id of the File
|
||||
@type notebook_id: unicode or NoneType
|
||||
@param notebook_id: id of the notebook containing the file
|
||||
@type note_id: unicode or NoneType
|
||||
@param note_id: id of the note linking to the file
|
||||
@type filename: unicode
|
||||
@param filename: name of the file on the client
|
||||
@type size_bytes: int
|
||||
@param size_bytes: length of the file data in bytes
|
||||
@rtype: File
|
||||
@return: newly constructed File
|
||||
"""
|
||||
return File( object_id, notebook_id = notebook_id, note_id = note_id, filename = filename,
|
||||
size_bytes = size_bytes )
|
||||
|
||||
@staticmethod
|
||||
def sql_load( object_id, revision = None ):
|
||||
# Files don't store old revisions
|
||||
if revision:
|
||||
raise NotImplementedError()
|
||||
|
||||
return \
|
||||
"""
|
||||
select
|
||||
file.id, file.revision, file.notebook_id, file.note_id, file.filename, size_bytes
|
||||
from
|
||||
file
|
||||
where
|
||||
file.id = %s;
|
||||
""" % quote( object_id )
|
||||
|
||||
@staticmethod
|
||||
def sql_id_exists( object_id, revision = None ):
|
||||
if revision:
|
||||
raise NotImplementedError()
|
||||
|
||||
return "select id from file where id = %s;" % quote( object_id )
|
||||
|
||||
def sql_exists( self ):
|
||||
return File.sql_id_exists( self.object_id )
|
||||
|
||||
def sql_create( self ):
|
||||
return "insert into file ( id, revision, notebook_id, note_id, filename, size_bytes ) values ( %s, %s, %s, %s, %s, %s );" % \
|
||||
( quote( self.object_id ), quote( self.revision ), quote( self.__notebook_id ), quote( self.__note_id ),
|
||||
quote( self.__filename ), self.__size_bytes or 'null' )
|
||||
|
||||
def sql_update( self ):
|
||||
return "update file set revision = %s, notebook_id = %s, note_id = %s, filename = %s, size_bytes = %s where id = %s;" % \
|
||||
( quote( self.revision ), quote( self.__notebook_id ), quote( self.__note_id ), quote( self.__filename ),
|
||||
self.__size_bytes or 'null', quote( self.object_id ) )
|
||||
|
||||
def to_dict( self ):
|
||||
d = Persistent.to_dict( self )
|
||||
d.update( dict(
|
||||
notebook_id = self.__notebook_id,
|
||||
note_id = self.__note_id,
|
||||
filename = self.__filename,
|
||||
size_bytes = self.__size_bytes,
|
||||
) )
|
||||
|
||||
return d
|
||||
|
||||
notebook_id = property( lambda self: self.__notebook_id )
|
||||
note_id = property( lambda self: self.__note_id )
|
||||
filename = property( lambda self: self.__filename )
|
||||
size_bytes = property( lambda self: self.__size_bytes )
|
|
@ -66,7 +66,7 @@ class Invite( Persistent ):
|
|||
|
||||
@staticmethod
|
||||
def sql_load( object_id, revision = None ):
|
||||
# password resets don't store old revisions
|
||||
# invites don't store old revisions
|
||||
if revision:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
create table file (
|
||||
id text,
|
||||
revision timestamp with time zone,
|
||||
notebook_id text,
|
||||
note_id text,
|
||||
filename text,
|
||||
size_bytes integer
|
||||
);
|
||||
alter table file add primary key ( id );
|
|
@ -6,3 +6,5 @@ DROP VIEW notebook_current;
|
|||
DROP TABLE notebook;
|
||||
DROP TABLE password_reset;
|
||||
DROP TABLE user_notebook;
|
||||
DROP TABLE invite;
|
||||
DROP TABLE file;
|
||||
|
|
|
@ -31,6 +31,22 @@ CREATE FUNCTION drop_html_tags(text) RETURNS text
|
|||
|
||||
ALTER FUNCTION public.drop_html_tags(text) OWNER TO luminotes;
|
||||
|
||||
--
|
||||
-- Name: file; Type: TABLE; Schema: public; Owner: luminotes; Tablespace:
|
||||
--
|
||||
|
||||
CREATE TABLE file (
|
||||
id text NOT NULL,
|
||||
revision timestamp with time zone,
|
||||
notebook_id text,
|
||||
note_id text,
|
||||
filename text,
|
||||
size_bytes integer
|
||||
);
|
||||
|
||||
|
||||
ALTER TABLE public.file OWNER TO luminotes;
|
||||
|
||||
--
|
||||
-- Name: invite; Type: TABLE; Schema: public; Owner: luminotes; Tablespace:
|
||||
--
|
||||
|
@ -161,6 +177,14 @@ CREATE TABLE user_notebook (
|
|||
|
||||
ALTER TABLE public.user_notebook OWNER TO luminotes;
|
||||
|
||||
|
||||
-- Name: file_pkey; Type: CONSTRAINT; Schema: public; Owner: luminotes; Tablespace:
|
||||
--
|
||||
|
||||
ALTER TABLE ONLY file
|
||||
ADD CONSTRAINT file_pkey PRIMARY KEY (id);
|
||||
|
||||
|
||||
--
|
||||
-- Name: invite_pkey; Type: CONSTRAINT; Schema: public; Owner: luminotes; Tablespace:
|
||||
--
|
||||
|
|
|
@ -0,0 +1,33 @@
|
|||
from pytz import utc
|
||||
from datetime import datetime, timedelta
|
||||
from model.File import File
|
||||
|
||||
|
||||
class Test_file( object ):
|
||||
def setUp( self ):
|
||||
self.object_id = u"17"
|
||||
self.notebook_id = u"18"
|
||||
self.note_id = u"19"
|
||||
self.filename = u"foo.png"
|
||||
self.size_bytes = 2888
|
||||
self.delta = timedelta( seconds = 1 )
|
||||
|
||||
self.file = File.create( self.object_id, self.notebook_id, self.note_id, self.filename,
|
||||
self.size_bytes )
|
||||
|
||||
def test_create( self ):
|
||||
assert self.file.object_id == self.object_id
|
||||
assert self.file.notebook_id == self.notebook_id
|
||||
assert self.file.note_id == self.note_id
|
||||
assert self.file.filename == self.filename
|
||||
assert self.file.size_bytes == self.size_bytes
|
||||
|
||||
def test_to_dict( self ):
|
||||
d = self.file.to_dict()
|
||||
|
||||
assert d.get( "object_id" ) == self.object_id
|
||||
assert datetime.now( tz = utc ) - d.get( "revision" ) < self.delta
|
||||
assert d.get( "notebook_id" ) == self.notebook_id
|
||||
assert d.get( "note_id" ) == self.note_id
|
||||
assert d.get( "filename" ) == self.filename
|
||||
assert d.get( "size_bytes" ) == self.size_bytes
|
|
@ -1,6 +1,5 @@
|
|||
from pytz import utc
|
||||
from datetime import datetime, timedelta
|
||||
from model.User import User
|
||||
from model.Invite import Invite
|
||||
|
||||
|
||||
|
|
|
@ -1,4 +1,3 @@
|
|||
from model.User import User
|
||||
from model.Password_reset import Password_reset
|
||||
|
||||
|
||||
|
|
|
@ -615,7 +615,7 @@ img {
|
|||
color: #ff6600;
|
||||
}
|
||||
|
||||
#upload_frame {
|
||||
.upload_frame {
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
width: 40em;
|
||||
|
|
|
@ -378,6 +378,12 @@ Editor.prototype.mouse_clicked = function ( event ) {
|
|||
return;
|
||||
}
|
||||
|
||||
// special case for links to uploaded files
|
||||
if ( !link.target && /\/files\//.test( link.href ) ) {
|
||||
location.href = link.href;
|
||||
return;
|
||||
}
|
||||
|
||||
event.stop();
|
||||
|
||||
// load the note corresponding to the clicked link
|
||||
|
|
|
@ -113,9 +113,13 @@ Wiki.prototype.update_next_id = function ( result ) {
|
|||
|
||||
var KILOBYTE = 1024;
|
||||
var MEGABYTE = 1024 * KILOBYTE;
|
||||
function bytes_to_megabytes( bytes, or_kilobytes ) {
|
||||
if ( or_kilobytes && bytes < MEGABYTE )
|
||||
return Math.round( bytes / KILOBYTE ) + " KB";
|
||||
function bytes_to_megabytes( bytes, choose_units ) {
|
||||
if ( choose_units ) {
|
||||
if ( bytes < KILOBYTE )
|
||||
return bytes + " bytes";
|
||||
if ( bytes < MEGABYTE )
|
||||
return Math.round( bytes / KILOBYTE ) + " KB";
|
||||
}
|
||||
|
||||
return Math.round( bytes / MEGABYTE ) + " MB";
|
||||
}
|
||||
|
@ -731,7 +735,7 @@ Wiki.prototype.display_link_pulldown = function ( editor, link ) {
|
|||
if ( link_title( link ).length > 0 ) {
|
||||
if ( !pulldown ) {
|
||||
this.clear_pulldowns();
|
||||
// display a different pulldown dependong on whether the link is a note link or a file link
|
||||
// display a different pulldown depending on whether the link is a note link or a file link
|
||||
if ( link.target || !/\/files\//.test( link.href ) )
|
||||
new Link_pulldown( this, this.notebook_id, this.invoker, editor, link );
|
||||
else
|
||||
|
@ -2246,9 +2250,11 @@ function Upload_pulldown( wiki, notebook_id, invoker, editor ) {
|
|||
"frameBorder": "0",
|
||||
"scrolling": "no",
|
||||
"id": "upload_frame",
|
||||
"name": "upload_frame"
|
||||
"name": "upload_frame",
|
||||
"class": "upload_frame"
|
||||
} );
|
||||
this.iframe.pulldown = this;
|
||||
this.file_id = null;
|
||||
|
||||
var self = this;
|
||||
connect( this.iframe, "onload", function ( event ) { self.init_frame(); } );
|
||||
|
@ -2266,26 +2272,47 @@ Upload_pulldown.prototype.init_frame = function () {
|
|||
withDocument( doc, function () {
|
||||
connect( "upload_button", "onclick", function ( event ) {
|
||||
withDocument( doc, function () {
|
||||
self.upload_started( getElement( "upload" ).value );
|
||||
self.upload_started( getElement( "file_id" ).value, getElement( "upload" ).value );
|
||||
} );
|
||||
} );
|
||||
} );
|
||||
}
|
||||
|
||||
Upload_pulldown.prototype.upload_started = function ( filename ) {
|
||||
Upload_pulldown.prototype.upload_started = function ( file_id, filename ) {
|
||||
this.file_id = file_id;
|
||||
|
||||
// make the upload iframe invisible but still present so that the upload continues
|
||||
addElementClass( this.iframe, "invisible" );
|
||||
setElementDimensions( this.iframe, { "h": "0" } );
|
||||
|
||||
// get the basename of the file
|
||||
var pieces = filename.split( "/" );
|
||||
filename = pieces[ pieces.length - 1 ];
|
||||
pieces = filename.split( "\\" );
|
||||
filename = pieces[ pieces.length - 1 ];
|
||||
|
||||
// the current title is blank, replace the title with the upload's filename
|
||||
// if the current title is blank, replace the title with the upload's filename
|
||||
if ( link_title( this.link ) == "" )
|
||||
replaceChildNodes( this.link, this.editor.document.createTextNode( filename ) );
|
||||
// TODO: set the link's href to the file
|
||||
|
||||
// FIXME: this call might occur before upload() is even called
|
||||
var progress_iframe = createDOM( "iframe", {
|
||||
"src": "/files/progress?file_id=" + file_id + "&filename=" + escape( filename ),
|
||||
"frameBorder": "0",
|
||||
"scrolling": "no",
|
||||
"id": "progress_frame",
|
||||
"name": "progress_frame",
|
||||
"class": "upload_frame"
|
||||
} );
|
||||
|
||||
appendChildNodes( this.div, progress_iframe );
|
||||
}
|
||||
|
||||
Upload_pulldown.prototype.upload_complete = function () {
|
||||
// now that the upload is done, the file link should point to the uploaded file
|
||||
this.link.href = "/files/download?file_id=" + this.file_id
|
||||
|
||||
// FIXME: the upload pulldown is sometimes being closed here before the upload is complete, thereby truncating the upload
|
||||
new File_link_pulldown( this.wiki, this.notebook_id, this.invoker, this.editor, this.link );
|
||||
this.shutdown();
|
||||
}
|
||||
|
|
|
@ -2,7 +2,7 @@ from Tags import Html, Head, Link, Meta, Body, P, Form, Span, Input
|
|||
|
||||
|
||||
class Upload_page( Html ):
|
||||
def __init__( self, notebook_id, note_id ):
|
||||
def __init__( self, notebook_id, note_id, file_id ):
|
||||
Html.__init__(
|
||||
self,
|
||||
Head(
|
||||
|
@ -12,15 +12,16 @@ class Upload_page( Html ):
|
|||
Body(
|
||||
Form(
|
||||
Span( u"attach file: ", class_ = u"field_label" ),
|
||||
Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ),
|
||||
Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ),
|
||||
Input( type = u"hidden", id = u"notebook_id", name = u"notebook_id", value = notebook_id ),
|
||||
Input( type = u"hidden", id = u"note_id", name = u"note_id", value = note_id ),
|
||||
action = u"/files/upload",
|
||||
Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ),
|
||||
Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ),
|
||||
action = u"/files/upload?file_id=%s" % file_id,
|
||||
method = u"post",
|
||||
enctype = u"multipart/form-data",
|
||||
),
|
||||
P( u"Please select a file to upload." ),
|
||||
Span( id = u"tick_preload" ),
|
||||
Input( type = u"hidden", id = u"file_id", value = file_id ),
|
||||
),
|
||||
)
|
||||
|
|
Reference in New Issue