witten
/
luminotes
Archived
1
0
Fork 0

Rewrote upload code to do progress bar updating from the UI via polling, rather than streaming the progress bar ticks from the server.

Still to do: Rewrite import code to do something similar, and refactor unit tests accordingly.
This commit is contained in:
Dan Helfman 2009-03-19 16:52:53 -07:00
parent 25b1fe25f6
commit 23b3884e83
6 changed files with 217 additions and 331 deletions

View File

@ -5,11 +5,12 @@ import cgi
import time
import urllib
import os.path
import httplib
import tempfile
import cherrypy
from PIL import Image
from cStringIO import StringIO
from threading import Lock, Event
from threading import Lock
from chardet.universaldetector import UniversalDetector
from Expose import expose
from Validate import validate, Valid_int, Valid_bool, Validation_error
@ -20,10 +21,9 @@ from model.File import File
from model.User import User
from model.Notebook import Notebook
from model.Download_access import Download_access
from view.Upload_page import Upload_page
from view.Blank_page import Blank_page
from view.Json import Json
from view.Progress_bar import stream_progress, stream_quota_error, quota_error_script, general_error_script
from view.Progress_bar import quota_error_script, general_error_script
from view.File_preview_page import File_preview_page
@ -87,14 +87,11 @@ class Upload_file( object ):
self.__content_length = content_length
self.__file_received_bytes = 0
self.__total_received_bytes = cherrypy.request.rfile.bytes_read
self.__total_received_bytes_updated = Event()
self.__complete = Event()
def write( self, data ):
self.__file.write( data )
self.__file_received_bytes += len( data )
self.__total_received_bytes = cherrypy.request.rfile.bytes_read
self.__total_received_bytes_updated.set()
def tell( self ):
return self.__file.tell()
@ -108,25 +105,13 @@ class Upload_file( object ):
return self.__file.read( size )
def wait_for_total_received_bytes( self ):
self.__total_received_bytes_updated.wait( timeout = cherrypy.server.socket_timeout )
self.__total_received_bytes_updated.clear()
return self.__total_received_bytes
def close( self ):
self.__file.close()
self.complete()
def complete( self ):
self.__complete.set()
def delete( self ):
self.__file.close()
self.delete_file( self.__file_id )
def wait_for_complete( self ):
self.__complete.wait( timeout = cherrypy.server.socket_timeout )
@staticmethod
def make_server_filename( file_id ):
global files_dir
@ -522,7 +507,7 @@ class Files( object ):
return stream()
@expose( view = Upload_page )
@expose( view = Json )
@strongly_expire
@end_transaction
@grab_user_id
@ -531,10 +516,9 @@ class Files( object ):
note_id = Valid_id(),
user_id = Valid_id( none_okay = True ),
)
def upload_page( self, notebook_id, note_id, user_id ):
def upload_id( self, notebook_id, note_id, user_id ):
"""
Provide the information necessary to display the file upload page, including the generation of a
unique file id.
Generate and return a unique file id for use in an upload.
@type notebook_id: unicode
@param notebook_id: id of the notebook that the upload will be to
@ -543,7 +527,7 @@ class Files( object ):
@type user_id: unicode or NoneType
@param user_id: id of current logged-in user (if any)
@rtype: unicode
@return: rendered HTML page
@return: { 'file_id': file_id }
@raise Access_error: the current user doesn't have access to the given notebook
"""
notebook = self.__users.load_notebook( user_id, notebook_id, read_write = True, note_id = note_id )
@ -554,47 +538,7 @@ class Files( object ):
file_id = self.__database.next_id( File )
return dict(
notebook_id = notebook_id,
note_id = note_id,
file_id = file_id,
label_text = u"attach file",
instructions_text = u"Please select a file to upload.",
)
@expose( view = Upload_page )
@strongly_expire
@end_transaction
@grab_user_id
@validate(
notebook_id = Valid_id(),
user_id = Valid_id( none_okay = True ),
)
def import_page( self, notebook_id, user_id ):
"""
Provide the information necessary to display the file import page, including the generation of a
unique file id.
@type notebook_id: unicode
@param notebook_id: id of the notebook that the upload will be to
@type note_id: unicode
@param user_id: id of current logged-in user (if any)
@rtype: unicode
@return: rendered HTML page
@raise Access_error: the current user doesn't have access to the given notebook
"""
notebook = self.__users.load_notebook( user_id, notebook_id, read_write = True )
if not notebook or notebook.read_write == Notebook.READ_WRITE_FOR_OWN_NOTES:
raise Access_error()
file_id = self.__database.next_id( File )
return dict(
notebook_id = notebook_id,
note_id = None,
file_id = file_id,
label_text = u"import file",
instructions_text = u"Please select a CSV file of notes to import into a new notebook.",
)
@expose( view = Blank_page )
@ -652,7 +596,11 @@ class Files( object ):
# if we didn't receive all of the expected data, abort
if uploaded_file.total_received_bytes < uploaded_file.content_length:
uploaded_file.delete()
return dict() # hopefully, the call to progress() will report this to the user
return dict( script = general_error_script % u"The uploaded file was not fully received. Please try again or contact support." )
if uploaded_file.file_received_bytes == 0:
uploaded_file.delete()
return dict( script = general_error_script % u"The uploaded file was not received. Please make sure that the file exists." )
# if the uploaded file's size would put the user over quota, bail and inform the user
rate_plan = self.__users.rate_plan( user.rate_plan )
@ -671,67 +619,79 @@ class Files( object ):
return dict()
@expose()
@expose( view = Json )
@strongly_expire
@end_transaction
@grab_user_id
@validate(
file_id = Valid_id(),
filename = unicode,
user_id = Valid_id( none_okay = True ),
)
def progress( self, file_id, filename, user_id = None ):
def progress( self, file_id, user_id = None ):
"""
Stream information on a file that is in the process of being uploaded. This method does not
perform any access checks, but the only information streamed is a progress bar and upload
percentage.
Return information on a file that is in the process of being uploaded. This method does not
perform any access checks, but the only information revealed is the file's upload progress.
This method is intended to be polled while the file is uploading, and its returned data is
intended to mimic the API described here:
http://wiki.nginx.org//NginxHttpUploadProgressModule
@type file_id: unicode
@param file_id: id of a currently uploading file
@type filename: unicode
@param filename: name of the file to report on
@type user_id: unicode or NoneType
@param user_id: id of current logged-in user (if any)
@rtype: unicode
@return: streaming HTML progress bar
@rtype: dict
@return: one of the following:
{ 'state': 'starting' } // file_id is unknown
{ 'state': 'done' } // upload is complete
{ 'state': 'error', 'status': http_error_code } // upload generated an HTTP error
{ 'state': 'uploading', // upload is in progress
'received': bytes_received, 'size': total_bytes }
"""
global current_uploads
# poll until the file is uploading (as determined by current_uploads) or completely uploaded (in
# the database with a filename)
while True:
uploading_file = current_uploads.get( file_id )
db_file = None
uploading_file = current_uploads.get( file_id )
db_file = None
if uploading_file:
fraction_reported = 0.0
break
db_file = self.__database.load( File, file_id )
if not db_file:
raise Upload_error( u"The file id is unknown" )
if db_file.filename is None:
time.sleep( 0.1 )
continue
fraction_reported = 1.0
break
# if the uploaded file's size would put the user over quota, bail and inform the user
if uploading_file:
# if the uploaded file's size would put the user over quota, bail and inform the user
SOFT_QUOTA_FACTOR = 1.05 # fudge factor since content_length isn't really the file's actual size
user = self.__database.load( User, user_id )
if not user:
raise Access_error()
return dict(
state = "error",
stauts = httplib.FORBIDDEN,
)
rate_plan = self.__users.rate_plan( user.rate_plan )
storage_quota_bytes = rate_plan.get( u"storage_quota_bytes" )
if storage_quota_bytes and \
user.storage_bytes + uploading_file.content_length > storage_quota_bytes * SOFT_QUOTA_FACTOR:
return stream_quota_error()
return dict(
state = "error",
stauts = httplib.REQUEST_ENTITY_TOO_LARGE,
)
return stream_progress( uploading_file, filename, fraction_reported )
return dict(
state = u"uploading",
received = uploading_file.total_received_bytes,
size = uploading_file.content_length,
);
db_file = self.__database.load( File, file_id )
if not db_file:
return dict(
state = "error",
stauts = httplib.NOT_FOUND,
)
if db_file.filename is None:
return dict( state = u"starting" );
# the file is completely uploaded (in the database with a filename)
return dict( state = u"done" );
@expose( view = Json )
@strongly_expire

View File

@ -281,6 +281,12 @@ h1 {
background-image: url(/static/images/grabber_hover.png);
}
#tick_preload {
height: 0;
overflow: hidden;
background-image: url(/static/images/tick.png);
}
#note_tree_area {
position: fixed;
width: 20em;
@ -886,7 +892,7 @@ h1 {
max-height: 20em;
min-width: 10em;
overflow: auto;
padding: 0.5em;
padding: 0.75em;
border: 1px solid #000000;
background-color: #ffff99;
}
@ -1159,6 +1165,8 @@ h1 {
}
.button {
-moz-border-radius: 3px;
-webkit-border-radius: 3px;
border-style: outset;
border-width: 0px;
background-color: #d0e0f0;
@ -1183,13 +1191,6 @@ h1 {
color: #ff6600;
}
.upload_frame {
padding: 0;
margin: 0;
width: 50em;
height: 5em;
}
.file_thumbnail {
margin-right: 0.5em;
vertical-align: top;
@ -1243,3 +1244,24 @@ h1 {
padding: 0.5em;
font-size: 110%;
}
#progress_row {
margin-top: 0.75em;
}
#progress_border {
border: 1px solid #000000;
background-color: #ffffff;
width: 20em;
height: 1em;
}
#progress_bar {
width: 0;
height: 1em;
}
#progress_percent {
margin-left: 0.75em;
margin-right: 0.75em;
}

View File

@ -1,58 +0,0 @@
html, body {
padding: 0;
margin: 0;
line-height: 140%;
font-family: sans-serif;
background-color: #ffff99;
}
body {
font-size: 72%;
}
form {
margin-bottom: 0.5em;
}
div {
margin-bottom: 0.5em;
}
.field_label {
font-weight: bold;
}
.text_field {
border: #999999 1px solid;
}
.button {
border-style: outset;
border-width: 0px;
background-color: #d0e0f0;
font-size: 100%;
outline: none;
cursor: pointer;
margin-left: 0.25em;
}
.button:hover {
background-color: #ffcc66;
}
#progress_border {
border: 1px solid #000000;
background-color: #ffffff;
width: 20em;
height: 1em;
}
td {
vertical-align: top;
}
#tick_preload {
height: 0;
overflow: hidden;
background-image: url(/static/images/tick.png);
}

View File

@ -3609,80 +3609,160 @@ function Upload_pulldown( wiki, notebook_id, invoker, editor, link, ephemeral )
Pulldown.call( this, wiki, notebook_id, "upload_" + editor.id, this.link, editor.iframe, ephemeral );
wiki.down_image_button( "attachFile" );
var vaguely_random = new Date().getTime();
this.invoker = invoker;
this.editor = editor;
this.iframe = createDOM( "iframe", {
"src": "/files/upload_page?notebook_id=" + notebook_id + "&note_id=" + editor.id,
"frameBorder": "0",
"scrolling": "no",
// if a new iframe has an id/name that WebKit has already seen, then it will just use its
// previous src value and ignore our new src value here. workaround: don't use the same id!
"id": "upload_frame_" + vaguely_random,
"name": "upload_frame_" + vaguely_random,
"class": "upload_frame"
"src": "about:blank",
"id": "upload_frame",
"name": "upload_frame",
"class": "upload_frame undisplayed"
} );
this.iframe.pulldown = this;
this.file_id = null;
this.uploading = false;
this.poller = null;
this.POLL_INTERVAL = 500;
var self = this;
connect( this.iframe, "onload", function ( event ) { self.init_frame(); } );
appendChildNodes( this.div, this.iframe );
this.progress_iframe = createDOM( "iframe", {
"frameBorder": "0",
"scrolling": "no",
"id": "progress_frame_" + vaguely_random,
"name": "progress_frame_" + vaguely_random,
"class": "upload_frame"
} );
addElementClass( this.progress_iframe, "undisplayed" );
this.upload_area = createDOM( "span" );
this.upload_button = createDOM( "input", { "id": "upload_button", "type": "submit", "class": "button", "value": "upload" } );
appendChildNodes( this.upload_area, createDOM( "form",
{
"target": "upload_frame",
"action": "/files/upload?file_id=new",
"method": "post",
"enctype": "multipart/form-data",
"id": "upload_form"
},
createDOM( "span", { "class": "field_label" }, "attach file: " ), // TODO: or "import file"
createDOM( "input", { "name": "notebook_id", "id": "notebook_id", "type": "hidden", "value": notebook_id } ),
createDOM( "input", { "name": "note_id", "id": "note_id", "type": "hidden", "value": editor ? editor.id : "" } ),
createDOM( "input", { "name": "upload", "id": "upload", "type": "file", "class": "text_field", "size": "30" } ),
this.upload_button
) );
this.upload_button.disabled = true;
appendChildNodes( this.upload_area, createDOM( "p", {}, "Please select a file to upload." ) ); // TODO: or import CSV
appendChildNodes( this.upload_area, createDOM( "span", { "id": "tick_preload" } ) );
appendChildNodes( this.upload_area, createDOM( "input", { "name": "file_id", "id": "file_id", "type": "hidden", "value": "new" } ) );
appendChildNodes( this.div, this.upload_area );
connect( this.upload_button, "onclick", function ( event ) {
self.upload_started();
} );
// grab the next available file id
this.invoker.invoke( "/files/upload_id", "POST",
{ "notebook_id": notebook_id, "note_id": editor ? editor.id : "" },
function( result ) { self.update_file_id( result ); }
);
appendChildNodes( this.div, this.progress_iframe );
Pulldown.prototype.finish_init.call( this );
}
Upload_pulldown.prototype = new function () { this.prototype = Pulldown.prototype; };
Upload_pulldown.prototype.constructor = Upload_pulldown;
Upload_pulldown.prototype.init_frame = function () {
var self = this;
var doc = this.iframe.contentDocument || this.iframe.contentWindow.document;
Upload_pulldown.prototype.update_file_id = function ( result ) {
this.file_id = result.file_id;
withDocument( doc, function () {
connect( "upload_button", "onclick", function ( event ) {
withDocument( doc, function () {
self.upload_started( getElement( "file_id" ).value );
} );
} );
var upload_form = getElement( "upload_form" )
if ( upload_form )
upload_form.action = "/files/upload?file_id=" + this.file_id;
connect( doc.body, "onmouseover", function ( event ) {
self.ephemeral = false;
} );
} );
var file_id_node = getElement( "file_id" );
if ( file_id_node )
file_id_node.value = this.file_id;
this.upload_button.disabled = false;
}
Upload_pulldown.prototype.upload_started = function ( file_id ) {
this.file_id = file_id;
this.uploading = true;
var filename = base_upload_filename();
// make the upload iframe invisible but still present so that the upload continues
setElementDimensions( this.iframe, { "h": "0" } );
// if the current title is blank, replace the title with the upload's filename
var title = link_title( this.link );
if ( title == "" )
this.link.innerHTML = filename;
this.cancel_button = createDOM( "input", { "type": "submit", "id": "cancel_button", "class": "button", "value": "cancel" } );
removeElementClass( this.progress_iframe, "undisplayed" );
var progress_url = "/files/progress?file_id=" + file_id + "&filename=" + escape( filename );
var progress_area = createDOM( "table", {},
createDOM( "tr", {},
createDOM( "td", { "class": "field_label", "colspan": "2" }, "uploading " + filename + ": " )
),
createDOM( "tr", { "id": "progress_row" },
createDOM( "td", {},
createDOM( "div", { "id": "progress_border" },
createDOM( "img", { "src": "/static/images/tick.png", "id": "progress_bar" } )
)
),
createDOM( "td", { "class": "progress_right" },
createDOM( "span", { "id": "progress_percent" }, "0%" ),
this.cancel_button
)
)
);
this.progress_iframe.src = progress_url;
disconnectAll( this.upload_button );
addElementClass( this.upload_area, "undisplayed" );
appendChildNodes( this.div, progress_area );
this.upload_button = null;
var self = this;
connect( this.cancel_button, "onclick", function ( event ) {
self.cancel_due_to_click();
} );
// start polling for the upload progress
this.poller = setTimeout( function () { self.update_progress(); }, this.POLL_INTERVAL );
}
Upload_pulldown.prototype.update_progress = function () {
var self = this;
var BAR_WIDTH_EM = 20.0;
// TODO: send X- HTTP header nginx expects with file_id
this.invoker.invoke( "/files/progress", "GET",
{ "file_id": this.file_id },
function( result ) {
var fraction_done = 0.0;
if ( !self.uploading )
return;
if ( result.state == "error" ) {
if ( result.status == 413 )
self.cancel_due_to_quota();
else
self.cancel_due_to_error( "An error occurred when uploading the file." );
return;
}
if ( result.state == "uploading" && result.size > 0 )
fraction_done = Math.min( result.received / result.size, 1.0 );
else if ( result.state == "done" )
fraction_done = 1.0;
if ( fraction_done > 0.0 ) {
var percent = fraction_done * 100.0;
setElementDimensions( "progress_bar", { "w": fraction_done * BAR_WIDTH_EM }, "em" );
replaceChildNodes( "progress_percent", parseInt( percent ) + "%" );
}
// the brief delay gives a brief moment for the progress bar to appear at 100%
if ( result.state == "done" )
setTimeout( function () { self.upload_complete(); }, 1 );
else
this.poller = setTimeout( function () { self.update_progress(); }, self.POLL_INTERVAL );
}
);
};
Upload_pulldown.prototype.upload_complete = function () {
if ( /MSIE/.test( navigator.userAgent ) )
var quote_filename = true;
@ -3702,6 +3782,7 @@ Upload_pulldown.prototype.update_position = function ( always_left_align ) {
}
Upload_pulldown.prototype.cancel_due_to_click = function () {
// when the uploading iframe closes, that should effectively cancel the upload
this.uploading = false;
this.wiki.display_message( "The file upload has been cancelled." )
this.shutdown();
@ -3713,7 +3794,7 @@ Upload_pulldown.prototype.cancel_due_to_quota = function () {
this.wiki.display_error(
"That file is too large for your available storage space. Before uploading, please delete some notes or files, empty the trash, or",
[ createDOM( "a", { "href": "/upgrade" }, "upgrade" ), " your account." ]
[ createDOM( "a", { "href": "/pricing" }, "upgrade" ), " your account." ]
);
}
@ -3727,11 +3808,18 @@ Upload_pulldown.prototype.shutdown = function () {
if ( this.uploading )
return;
if ( this.poller )
clearTimeout( this.poller );
if ( this.upload_button )
disconnectAll( this.upload_button );
if ( this.cancel_button )
disconnectAll( this.cancel_button );
// in Internet Explorer, the upload won't actually cancel without an explicit Stop command
if ( !this.iframe.contentDocument && this.iframe.contentWindow ) {
if ( !this.iframe.contentDocument && this.iframe.contentWindow )
this.iframe.contentWindow.document.execCommand( 'Stop' );
this.progress_iframe.contentWindow.document.execCommand( 'Stop' );
}
Pulldown.prototype.shutdown.call( this );
if ( this.link )

View File

@ -2,86 +2,6 @@ import cgi
from config.Version import VERSION
def stream_progress( uploading_file, filename, fraction_reported ):
"""
Stream a progress meter as a file uploads.
"""
progress_bytes = 0
progress_width_em = 20
tick_increment = 0.01
progress_bar = u'<img src="/static/images/tick.png" style="width: %sem; height: 1em;" id="progress_bar" />' % \
( progress_width_em * tick_increment )
yield \
u"""
<html>
<head>
<link href="/static/css/upload.css?%s" type="text/css" rel="stylesheet" />
<script type="text/javascript" src="/static/js/MochiKit.js?%s"></script>
<meta content="text/html; charset=UTF-8" http_equiv="content-type" />
</head>
<body>
""" % ( VERSION, VERSION )
FILENAME_TRUNCATION_WIDTH = 40
base_filename = filename.split( u"/" )[ -1 ].split( u"\\" )[ -1 ]
if len( base_filename ) > FILENAME_TRUNCATION_WIDTH:
base_filename = base_filename[ : FILENAME_TRUNCATION_WIDTH ] + u"..."
yield \
u"""
<div class="field_label">uploading %s: </div>
<table><tr>
<td><div id="progress_border">
%s
</div></td>
<td></td>
<td><span id="status">0%%</span></td>
<td></td>
<td><input type="submit" id="cancel_button" class="button" value="cancel" onclick="withDocument( window.parent.document, function () { getFirstElementByTagAndClassName( "iframe", "upload_frame" ).pulldown.cancel_due_to_click(); } );" /></td>
</tr></table>
<script type="text/javascript">
function tick( fraction ) {
setElementDimensions(
"progress_bar",
{ "w": %s * fraction }, "em"
);
if ( fraction >= 1.0 )
replaceChildNodes( "status", "100%%" );
else
replaceChildNodes( "status", Math.floor( fraction * 100.0 ) + "%%" );
}
</script>
""" % ( cgi.escape( base_filename ), progress_bar, progress_width_em )
if uploading_file:
received_bytes = 0
while received_bytes < uploading_file.content_length:
received_bytes = uploading_file.wait_for_total_received_bytes()
fraction_done = float( received_bytes ) / float( uploading_file.content_length )
if fraction_done > 1.0: fraction_done = 1.0
if fraction_done == 1.0 or fraction_done > fraction_reported + tick_increment:
fraction_reported = fraction_done
yield '<script type="text/javascript">tick(%s);</script>' % fraction_reported
uploading_file.wait_for_complete()
if fraction_reported < 1.0:
yield "An error occurred when uploading the file.</body></html>"
return
yield \
u"""
<script type="text/javascript">
withDocument( window.parent.document, function () { var frame = getFirstElementByTagAndClassName( "iframe", "upload_frame" ); if ( frame && frame.pulldown ) frame.pulldown.upload_complete(); } );
</script>
</body>
</html>
"""
general_error_script = \
"""
withDocument( window.parent.document, function () { var frame = getFirstElementByTagAndClassName( "iframe", "upload_frame" ); if ( frame && frame.pulldown ) frame.pulldown.cancel_due_to_error( "%s" ); } );
@ -92,21 +12,3 @@ quota_error_script = \
"""
withDocument( window.parent.document, function () { var frame = getFirstElementByTagAndClassName( "iframe", "upload_frame" ); if ( frame && frame.pulldown ) frame.pulldown.cancel_due_to_quota(); } );
"""
def stream_quota_error():
yield \
u"""
<html>
<head>
<link href="/static/css/upload.css?%s" type="text/css" rel="stylesheet" />
<script type="text/javascript" src="/static/js/MochiKit.js?%s"></script>
<meta content="text/html; charset=UTF-8" http_equiv="content-type" />
</head>
<body>
<script type="text/javascript">
%s
</script>
</body>
</html>
""" % ( VERSION, VERSION, quota_error_script )

View File

@ -1,28 +0,0 @@
from Tags import Html, Head, Link, Meta, Body, P, Form, Span, Input
from config.Version import VERSION
class Upload_page( Html ):
def __init__( self, notebook_id, note_id, file_id, label_text, instructions_text ):
Html.__init__(
self,
Head(
Link( href = u"/static/css/upload.css?%s" % VERSION, type = u"text/css", rel = u"stylesheet" ),
Meta( content = u"text/html; charset=UTF-8", http_equiv = u"content-type" ),
),
Body(
Form(
Span( u"%s: " % label_text, class_ = u"field_label" ),
Input( type = u"hidden", id = u"notebook_id", name = u"notebook_id", value = notebook_id ),
Input( type = u"hidden", id = u"note_id", name = u"note_id", value = note_id or u"" ),
Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ),
Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ),
action = u"/files/upload?file_id=%s" % file_id,
method = u"post",
enctype = u"multipart/form-data",
),
P( instructions_text ),
Span( id = u"tick_preload" ),
Input( type = u"hidden", id = u"file_id", value = file_id ),
),
)