Browse Source

More work on file uploading. Unit tests need to be fixed.

Dan Helfman 10 years ago
parent
commit
731dc52623

+ 3
- 0
NEWS View File

@@ -1,3 +1,6 @@
1
+1.2.0: February ??, 2008
2
+ * Users can now upload files to attach to their notes.
3
+
1 4
 1.1.3: January 28, 2008
2 5
  * Now, if you delete a notebook and the only remaining notebook is read-only,
3 6
    then a new read-write notebook is created for you automatically.

+ 5
- 1
config/Common.py View File

@@ -63,7 +63,11 @@ settings = {
63 63
       """
64 64
       """,
65 65
   },
66
-  "/files/upload": {
66
+  "/files/download": {
67
+    "stream_response": True,
68
+    "encoding_filter.on": False,
69
+  },
70
+  "/files/progress": {
67 71
     "stream_response": True
68 72
   },
69 73
 }

+ 327
- 58
controller/Files.py View File

@@ -1,11 +1,17 @@
1 1
 import cgi
2
+import time
3
+import tempfile
2 4
 import cherrypy
5
+from threading import Lock, Event
3 6
 from Expose import expose
4
-from Validate import validate
7
+from Validate import validate, Valid_int, Validation_error
5 8
 from Database import Valid_id
6 9
 from Users import grab_user_id
7 10
 from Expire import strongly_expire
11
+from model.File import File
8 12
 from view.Upload_page import Upload_page
13
+from view.Blank_page import Blank_page
14
+from view.Json import Json
9 15
 
10 16
 
11 17
 class Access_error( Exception ):
@@ -36,6 +42,153 @@ class Upload_error( Exception ):
36 42
     )
37 43
 
38 44
 
45
+# map of upload id to Upload_file
46
+current_uploads = {}
47
+current_uploads_lock = Lock()
48
+
49
+
50
+class Upload_file( object ):
51
+  """
52
+  File-like object for storing file uploads.
53
+  """
54
+  def __init__( self, file_id, filename, content_length ):
55
+    self.__file = file( self.make_server_filename( file_id ), "w+" )
56
+    self.__file_id = file_id
57
+    self.__filename = filename
58
+    self.__content_length = content_length
59
+    self.__file_received_bytes = 0
60
+    self.__total_received_bytes = cherrypy.request.rfile.bytes_read
61
+    self.__total_received_bytes_updated = Event()
62
+    self.__complete = Event()
63
+  
64
+  def write( self, data ):
65
+    self.__file.write( data )
66
+    self.__file_received_bytes += len( data )
67
+    self.__total_received_bytes = cherrypy.request.rfile.bytes_read
68
+    self.__total_received_bytes_updated.set()
69
+
70
+  def tell( self ):
71
+    return self.__file.tell()
72
+
73
+  def seek( self, position ):
74
+    self.__file.seek( position )
75
+
76
+  def read( self, size = None ):
77
+    if size is None:
78
+      return self.__file.read()
79
+
80
+    return self.__file.read( size )
81
+
82
+  def wait_for_total_received_bytes( self ):
83
+    self.__total_received_bytes_updated.wait( timeout = cherrypy.server.socket_timeout )
84
+    self.__total_received_bytes_updated.clear()
85
+    return self.__total_received_bytes
86
+
87
+  def close( self ):
88
+    self.__file.close()
89
+    self.__complete.set()
90
+
91
+  def wait_for_complete( self ):
92
+    self.__complete.wait( timeout = cherrypy.server.socket_timeout )
93
+
94
+  @staticmethod
95
+  def make_server_filename( file_id ):
96
+    return u"files/%s" % file_id
97
+
98
+  filename = property( lambda self: self.__filename )
99
+
100
+  # expected byte count of the entire form upload, including the file and other form parameters
101
+  content_length = property( lambda self: self.__content_length )
102
+
103
+  # count of bytes received thus far for this file upload only
104
+  file_received_bytes = property( lambda self: self.__file_received_bytes )
105
+
106
+  # count of bytes received thus far for the form upload, including the file and other form
107
+  # parameters
108
+  total_received_bytes = property( lambda self: self.__total_received_bytes )
109
+
110
+
111
+class FieldStorage( cherrypy._cpcgifs.FieldStorage ):
112
+  """
113
+  Derived from cherrypy._cpcgifs.FieldStorage, which is in turn derived from cgi.FieldStorage, which
114
+  calls make_file() to create a temporary file where file uploads are stored. By wrapping this file
115
+  object, we can track its progress as its written. Inspired by:
116
+  http://www.cherrypy.org/attachment/ticket/546/uploadfilter.py
117
+
118
+  This method relies on a file_id parameter being present in the HTTP query string.
119
+
120
+  @type binary: NoneType
121
+  @param binary: ignored
122
+  @type user_id: unicode or NoneType
123
+  @param user_id: id of current logged-in user (if any)
124
+  @rtype: Upload_file
125
+  @return: wrapped temporary file used to store the upload
126
+  @raise Upload_error: the provided file_id value is invalid, or the filename or Content-Length is
127
+                       missing
128
+  """
129
+  def make_file( self, binary = None, user_id = None ):
130
+    global current_uploads, current_uploads_lock
131
+
132
+    cherrypy.server.max_request_body_size = 0 # remove CherryPy default file size limit of 100 MB
133
+    cherrypy.response.timeout = 3600 * 2 # increase upload timeout to 2 hours (default is 5 min)
134
+    cherrypy.server.socket_timeout = 60 # increase socket timeout to one minute (default is 10 sec)
135
+    DASHES_AND_NEWLINES = 6 # four dashes and two newlines
136
+
137
+    # release the cherrypy session lock so that the user can issue other commands while the file is
138
+    # uploading
139
+    cherrypy.session.release_lock()
140
+
141
+    # pluck the file id out of the query string. it would be preferable to grab it out of parsed
142
+    # form variables instead, but at this point in the processing, all the form variables might not
143
+    # be parsed
144
+    file_id = cgi.parse_qs( cherrypy.request.query_string ).get( u"file_id", [ None ] )[ 0 ]
145
+    try:
146
+      file_id = Valid_id()( file_id )
147
+    except ValueError:
148
+      raise Upload_error( "The file_id is invalid." )
149
+
150
+    if not self.filename:
151
+      raise Upload_error( "Please provide a filename." )
152
+
153
+    content_length =  cherrypy.request.headers.get( "content-length", 0 )
154
+    try:
155
+      content_length = Valid_int( min = 0 )( content_length ) - len( self.outerboundary ) - DASHES_AND_NEWLINES
156
+    except ValueError:
157
+      raise Upload_error( "The Content-Length header value is invalid." )
158
+
159
+    # file size is the entire content length of the POST, minus the size of the other form
160
+    # parameters and boundaries. note: this assumes that the uploaded file is sent as the last
161
+    # form parameter in the POST
162
+    # TODO: verify that the uploaded file is always sent as the last parameter
163
+    existing_file = current_uploads.get( file_id )
164
+    if existing_file:
165
+      existing_file.close()
166
+
167
+    upload_file = Upload_file( file_id, self.filename.strip(), content_length )
168
+
169
+    current_uploads_lock.acquire()
170
+    try:
171
+      current_uploads[ file_id ] = upload_file
172
+    finally:
173
+      current_uploads_lock.release()
174
+
175
+    return upload_file
176
+
177
+  def __write( self, line ):
178
+    """
179
+    This implementation of __write() is different than that of the base class, because it calls
180
+    make_file() whenever there is a filename instead of only for large enough files.
181
+    """
182
+    if self.__file is not None and self.filename:
183
+        self.file = self.make_file( '' )
184
+        self.file.write( self.__file.getvalue() )
185
+        self.__file = None
186
+
187
+    self.file.write( line )
188
+
189
+cherrypy._cpcgifs.FieldStorage = FieldStorage
190
+
191
+
39 192
 class Files( object ):
40 193
   """
41 194
   Controller for dealing with uploaded files, corresponding to the "/files" URL.
@@ -54,39 +207,96 @@ class Files( object ):
54 207
     self.__database = database
55 208
     self.__users = users
56 209
 
210
+  @expose()
211
+  @grab_user_id
212
+  @validate(
213
+    file_id = Valid_id(),
214
+    user_id = Valid_id( none_okay = True ),
215
+  )
216
+  def download( self, file_id, user_id = None ):
217
+    """
218
+    Return the contents of file that a user has previously uploaded.
219
+
220
+    @type file_id: unicode
221
+    @param file_id: id of the file to download
222
+    @type user_id: unicode or NoneType
223
+    @param user_id: id of current logged-in user (if any)
224
+    @rtype: unicode
225
+    @return: file data
226
+    @raise Access_error: the current user doesn't have access to the notebook that the file is in
227
+    """
228
+    db_file = self.__database.load( File, file_id )
229
+
230
+    if not db_file or not self.__users.check_access( user_id, db_file.notebook_id ):
231
+      raise Access_error()
232
+
233
+    db_file = self.__database.load( File, file_id )
234
+
235
+    cherrypy.response.headerMap[ u"Content-Disposition" ] = u"attachment; filename=%s" % db_file.filename
236
+    cherrypy.response.headerMap[ u"Content-Length" ] = db_file.size_bytes
237
+# TODO: send content type
238
+#    cherrypy.response.headerMap[ u"Content-Type" ] = u"image/png"
239
+
240
+    def stream():
241
+      CHUNK_SIZE = 8192
242
+      local_file = file( Upload_file.make_server_filename( file_id ) )
243
+
244
+      while True:
245
+        data = local_file.read( CHUNK_SIZE )
246
+        if len( data ) == 0: break
247
+        yield data        
248
+
249
+    return stream()
250
+
251
+
57 252
   @expose( view = Upload_page )
253
+  @strongly_expire
254
+  @grab_user_id
58 255
   @validate(
59 256
     notebook_id = Valid_id(),
60 257
     note_id = Valid_id(),
258
+    user_id = Valid_id( none_okay = True ),
61 259
   )
62
-  def upload_page( self, notebook_id, note_id ):
260
+  def upload_page( self, notebook_id, note_id, user_id ):
63 261
     """
64
-    Provide the information necessary to display the file upload page.
262
+    Provide the information necessary to display the file upload page, including the generation of a
263
+    unique file id.
65 264
 
66 265
     @type notebook_id: unicode
67 266
     @param notebook_id: id of the notebook that the upload will be to
68 267
     @type note_id: unicode
69 268
     @param note_id: id of the note that the upload will be to
269
+    @type user_id: unicode or NoneType
270
+    @param user_id: id of current logged-in user (if any)
70 271
     @rtype: unicode
71 272
     @return: rendered HTML page
273
+    @raise Access_error: the current user doesn't have access to the given notebook
72 274
     """
275
+    if not self.__users.check_access( user_id, notebook_id, read_write = True ):
276
+      raise Access_error()
277
+
278
+    file_id = self.__database.next_id( File )
279
+
73 280
     return dict(
74 281
       notebook_id = notebook_id,
75 282
       note_id = note_id,
283
+      file_id = file_id,
76 284
     )
77 285
 
78
-  @expose()
286
+  @expose( view = Blank_page )
79 287
   @strongly_expire
80 288
   @grab_user_id
81 289
   @validate(
82 290
     upload = (),
83 291
     notebook_id = Valid_id(),
84 292
     note_id = Valid_id(),
293
+    file_id = Valid_id(),
85 294
     user_id = Valid_id( none_okay = True ),
86 295
   )
87
-  def upload( self, upload, notebook_id, note_id, user_id ):
296
+  def upload( self, upload, notebook_id, note_id, file_id, user_id ):
88 297
     """
89
-    Upload a file from the client for attachment to a particular note.
298
+    Upload a file from the client for attachment to a particular note. The file_id must be provided
299
+    as part of the query string, even if the other values are submitted as form data.
90 300
 
91 301
     @type upload: cgi.FieldStorage
92 302
     @param upload: file handle to uploaded file
@@ -94,40 +304,90 @@ class Files( object ):
94 304
     @param notebook_id: id of the notebook that the upload is to
95 305
     @type note_id: unicode
96 306
     @param note_id: id of the note that the upload is to
97
-    @raise Access_error: the current user doesn't have access to the given notebook or note
98
-    @raise Upload_error: an error occurred when processing the uploaded file
307
+    @type file_id: unicode
308
+    @param file_id: id of the file being uploaded
99 309
     @type user_id: unicode or NoneType
100 310
     @param user_id: id of current logged-in user (if any)
101 311
     @rtype: unicode
102 312
     @return: rendered HTML page
313
+    @raise Access_error: the current user doesn't have access to the given notebook or note
314
+    @raise Upload_error: the Content-Length header value is invalid
103 315
     """
104
-    if not self.__users.check_access( user_id, notebook_id ):
316
+    global current_uploads, current_uploads_lock
317
+
318
+    if not self.__users.check_access( user_id, notebook_id, read_write = True ):
105 319
       raise Access_error()
106 320
 
107
-    cherrypy.server.max_request_body_size = 0 # remove file size limit of 100 MB
108
-    cherrypy.response.timeout = 3600    # increase upload timeout to one hour (default is 5 min)
109
-    cherrypy.server.socket_timeout = 60 # increase socket timeout to one minute (default is 10 sec)
110
-    CHUNK_SIZE = 8 * 1024 # 8 Kb
321
+    # write the file to the database
322
+    uploaded_file = current_uploads.get( file_id )
323
+    if not uploaded_file:
324
+      raise Upload_error()
325
+
326
+# TODO: grab content type and store it
327
+    #print upload.headers.get( "content-type", "MISSING" )
111 328
 
112
-    headers = {}
113
-    for key, val in cherrypy.request.headers.iteritems():
114
-      headers[ key.lower() ] = val
329
+# TODO: somehow detect when upload is canceled and abort
115 330
 
331
+    db_file = File.create( file_id, notebook_id, note_id, uploaded_file.filename, uploaded_file.file_received_bytes )
332
+    self.__database.save( db_file )
333
+    uploaded_file.close()
334
+
335
+    current_uploads_lock.acquire()
116 336
     try:
117
-      file_size = int( headers.get( "content-length", 0 ) )
118
-    except ValueError:
119
-      raise Upload_error()
120
-    if file_size <= 0:
121
-      raise Upload_error()
337
+      del( current_uploads[ file_id ] )
338
+    finally:
339
+      current_uploads_lock.release()
340
+
341
+    return dict()
122 342
 
123
-    filename = upload.filename.strip()
343
+  @expose()
344
+  @strongly_expire
345
+  @validate(
346
+    file_id = Valid_id(),
347
+    filename = unicode,
348
+  )
349
+  def progress( self, file_id, filename ):
350
+    """
351
+    Stream information on a file that is in the process of being uploaded. This method does not
352
+    perform any access checks, but the only information streamed is a progress bar and upload
353
+    percentage.
354
+
355
+    @type file_id: unicode
356
+    @param file_id: id of a currently uploading file
357
+    @type filename: unicode
358
+    @param filename: name of the file to report on
359
+    @rtype: unicode
360
+    @return: streaming HTML progress bar
361
+    """
362
+    # release the session lock before beginning to stream the upload report. otherwise, if the
363
+    # upload is cancelled before it's done, the lock won't be released
364
+    cherrypy.session.release_lock()
124 365
 
125
-    def process_upload():
366
+    # poll until the file is uploading (as determined by current_uploads) or completely uploaded (in
367
+    # the database with a filename)
368
+    while True:
369
+      uploading_file = current_uploads.get( file_id )
370
+      db_file = None
371
+
372
+      if uploading_file:
373
+        fraction_reported = 0.0
374
+        break
375
+
376
+      db_file = self.__database.load( File, file_id )
377
+      if not db_file:
378
+        raise Upload_error( u"The file id is unknown" )
379
+      if db_file.filename is None:
380
+        time.sleep( 0.1 )
381
+        continue
382
+      fraction_reported = 1.0
383
+      break
384
+
385
+    # TODO: maybe move this to the view/ directory
386
+    def report( uploading_file, fraction_reported ):
126 387
       """
127
-      Process the file upload while streaming a progress meter as it uploads.
388
+      Stream a progress meter as it uploads.
128 389
       """
129 390
       progress_bytes = 0
130
-      fraction_reported = 0.0
131 391
       progress_width_em = 20
132 392
       tick_increment = 0.01
133 393
       progress_bar = u'<img src="/static/images/tick.png" style="width: %sem; height: 1em;" id="progress_bar" />' % \
@@ -144,15 +404,6 @@ class Files( object ):
144 404
         <body>
145 405
         """
146 406
 
147
-      if not filename:
148
-        yield \
149
-          u"""
150
-          <div class="field_label">upload error: </div>
151
-          Please check that the filename is valid.
152
-          </body></html>
153
-          """
154
-        return
155
-
156 407
       base_filename = filename.split( u"/" )[ -1 ].split( u"\\" )[ -1 ]
157 408
       yield \
158 409
         u"""
@@ -180,46 +431,64 @@ class Files( object ):
180 431
         </script>
181 432
         """ % ( cgi.escape( base_filename ), progress_bar, progress_width_em )
182 433
 
183
-      while True:
184
-        chunk = upload.file.read( CHUNK_SIZE )
185
-        if not chunk: break
186
-        progress_bytes += len( chunk )
187
-        fraction_done = float( progress_bytes ) / float( file_size )
434
+      if uploading_file:
435
+        received_bytes = 0
436
+        while received_bytes < uploading_file.content_length:
437
+          received_bytes = uploading_file.wait_for_total_received_bytes()
438
+          fraction_done = float( received_bytes ) / float( uploading_file.content_length )
188 439
 
189
-        if fraction_done > fraction_reported + tick_increment:
190
-          yield '<script type="text/javascript">tick(%s);</script>' % fraction_reported
191
-          fraction_reported = fraction_done
440
+          if fraction_done == 1.0 or fraction_done > fraction_reported + tick_increment:
441
+            fraction_reported = fraction_done
442
+            yield '<script type="text/javascript">tick(%s);</script>' % fraction_reported
192 443
 
193
-        # TODO: write to the database
444
+        uploading_file.wait_for_complete()
194 445
 
195
-      if fraction_reported == 0:
446
+      if fraction_reported < 1.0:
196 447
         yield "An error occurred when uploading the file.</body></html>"
197 448
         return
198 449
 
199
-      # the file finished uploading, so fill out the progress meter to 100%
200
-      if fraction_reported < 1.0:
201
-        yield '<script type="text/javascript">tick(1.0);</script>'
202
-
203
-      # the setTimeout() below ensures that the 100% progress bar is displayed for at least a moment
204 450
       yield \
205 451
         u"""
206 452
         <script type="text/javascript">
207
-        setTimeout( 'withDocument( window.parent.document, function () { getElement( "upload_frame" ).pulldown.upload_complete(); } );', 10 );
453
+        withDocument( window.parent.document, function () { getElement( "upload_frame" ).pulldown.upload_complete(); } );
208 454
         </script>
209 455
         </body>
210 456
         </html>
211 457
         """
212 458
 
213
-      upload.file.close()
459
+    return report( uploading_file, fraction_reported )
214 460
 
215
-    # release the session lock before beginning the upload, because if the upload is cancelled
216
-    # before it's done, the lock won't be released
217
-    cherrypy.session.release_lock()
461
+  @expose( view = Json )
462
+  @strongly_expire
463
+  @grab_user_id
464
+  @validate(
465
+    file_id = Valid_id(),
466
+    user_id = Valid_id( none_okay = True ),
467
+  )
468
+  def stats( self, file_id, user_id = None ):
469
+    """
470
+    Return information on a file that has been completely uploaded and is stored in the database.
218 471
 
219
-    return process_upload()
472
+    @type file_id: unicode
473
+    @param file_id: id of the file to report on
474
+    @type user_id: unicode or NoneType
475
+    @param user_id: id of current logged-in user (if any)
476
+    @rtype: dict
477
+    @return: {
478
+      'filename': filename,
479
+      'size_bytes': filesize,
480
+    }
481
+    @raise Access_error: the current user doesn't have access to the notebook that the file is in
482
+    """
483
+    db_file = self.__database.load( File, file_id )
220 484
 
221
-  def stats( file_id ):
222
-    pass
485
+    if not db_file or not self.__users.check_access( user_id, db_file.notebook_id ):
486
+      raise Access_error()
487
+
488
+    return dict(
489
+      filename = db_file.filename,
490
+      size_bytes = db_file.size_bytes,
491
+    )
223 492
 
224 493
   def rename( file_id, filename ):
225
-    pass
494
+    pass # TODO

+ 2
- 1
controller/Users.py View File

@@ -125,7 +125,8 @@ def grab_user_id( function ):
125 125
     arg_names = list( function.func_code.co_varnames )
126 126
     if "user_id" in arg_names:
127 127
       arg_index = arg_names.index( "user_id" )
128
-      args[ arg_index ] = cherrypy.session.get( "user_id" )
128
+      args = list( args )
129
+      args[ arg_index - 1 ] = cherrypy.session.get( "user_id" )
129 130
     else:
130 131
       kwargs[ "user_id" ] = cherrypy.session.get( "user_id" )
131 132
 

+ 4
- 2
controller/Validate.py View File

@@ -146,10 +146,12 @@ class Valid_int( object ):
146 146
   def __call__( self, value ):
147 147
     value = int( value )
148 148
 
149
-    if self.min is not None and value < min:
149
+    if self.min is not None and value < self.min:
150 150
       self.message = "is too small"
151
-    if self.max is not None and value > max:
151
+      raise ValueError()
152
+    if self.max is not None and value > self.max:
152 153
       self.message = "is too large"
154
+      raise ValueError()
153 155
 
154 156
     return value
155 157
 

+ 24
- 6
controller/test/Test_controller.py View File

@@ -7,6 +7,19 @@ from StringIO import StringIO
7 7
 from copy import copy
8 8
 
9 9
 
10
+class Truncated_StringIO( StringIO ):
11
+  """
12
+  A wrapper for StringIO that forcibly closes the file when only some of it has been read. Used
13
+  for simulating an upload that is canceled part of the way through.
14
+  """
15
+  def readline( self, size = None ):
16
+    if self.tell() >= len( self.getvalue() ) * 0.25:
17
+      self.close()
18
+      return ""
19
+
20
+    return StringIO.readline( self, 256 )
21
+
22
+
10 23
 class Test_controller( object ):
11 24
   def __init__( self ):
12 25
     from model.User import User
@@ -427,7 +440,7 @@ class Test_controller( object ):
427 440
     finally:
428 441
       request.close()
429 442
 
430
-  def http_upload( self, http_path, form_args, filename, file_data, headers = None, session_id = None ):
443
+  def http_upload( self, http_path, form_args, filename, file_data, simulate_cancel = False, headers = None, session_id = None ):
431 444
     """
432 445
     Perform an HTTP POST with the given path on the test server, sending the provided form_args
433 446
     and file_data as a multipart form file upload. Return the result dict as returned by the
@@ -452,16 +465,21 @@ class Test_controller( object ):
452 465
       headers = []
453 466
 
454 467
     post_data = str( "".join( post_data ) )
455
-    headers.extend( [
456
-      ( "Content-Type", "multipart/form-data; boundary=%s" % boundary ),
457
-      ( "Content-Length", str( len( post_data ) ) ),
458
-    ] )
468
+    headers.append( ( "Content-Type", "multipart/form-data; boundary=%s" % boundary ) )
469
+
470
+    if "Content-Length" not in [ name for ( name, value ) in headers ]:
471
+      headers.append( ( "Content-Length", str( len( post_data ) ) ) )
459 472
 
460 473
     if session_id:
461 474
       headers.append( ( u"Cookie", "session_id=%s" % session_id ) ) # will break if unicode is used for the value
462 475
 
476
+    if simulate_cancel:
477
+      file_wrapper = Truncated_StringIO( post_data )
478
+    else:
479
+      file_wrapper = StringIO( post_data )
480
+
463 481
     request = cherrypy.server.request( ( u"127.0.0.1", 1234 ), u"127.0.0.5" )
464
-    response = request.run( "POST %s HTTP/1.0" % str( http_path ), headers = headers, rfile = StringIO( post_data ) )
482
+    response = request.run( "POST %s HTTP/1.0" % str( http_path ), headers = headers, rfile = file_wrapper )
465 483
     session_id = response.simple_cookie.get( u"session_id" )
466 484
     if session_id: session_id = session_id.value
467 485
 

+ 43
- 12
controller/test/Test_files.py View File

@@ -68,7 +68,7 @@ class Test_files( Test_controller ):
68 68
     assert result.get( u"notebook_id" ) == self.notebook.object_id
69 69
     assert result.get( u"note_id" ) == self.note.object_id
70 70
 
71
-  def test_upload_file( self ):
71
+  def test_upload( self ):
72 72
     self.login()
73 73
 
74 74
     result = self.http_upload(
@@ -101,12 +101,12 @@ class Test_files( Test_controller ):
101 101
         raise exc
102 102
 
103 103
     # assert that the progress bar is moving, and then completes
104
-    assert tick_count >= 3
104
+    assert tick_count >= 2
105 105
     assert tick_done
106 106
 
107 107
     # TODO: assert that the uploaded file actually got stored somewhere
108 108
 
109
-  def test_upload_file_without_login( self ):
109
+  def test_upload_without_login( self ):
110 110
     result = self.http_upload(
111 111
       "/files/upload",
112 112
       dict(
@@ -120,7 +120,7 @@ class Test_files( Test_controller ):
120 120
 
121 121
     assert u"access" in result.get( u"body" )[ 0 ]
122 122
 
123
-  def test_upload_file_without_access( self ):
123
+  def test_upload_without_access( self ):
124 124
     self.login2()
125 125
 
126 126
     result = self.http_upload(
@@ -136,7 +136,7 @@ class Test_files( Test_controller ):
136 136
 
137 137
     assert u"access" in result.get( u"body" )[ 0 ]
138 138
 
139
-  def assert_inline_error( self, result ):
139
+  def assert_streaming_error( self, result ):
140 140
     gen = result[ u"body" ]
141 141
     assert isinstance( gen, types.GeneratorType )
142 142
 
@@ -152,7 +152,7 @@ class Test_files( Test_controller ):
152 152
 
153 153
     assert found_error
154 154
 
155
-  def test_upload_file_unnamed( self ):
155
+  def test_upload_unnamed( self ):
156 156
     self.login()
157 157
 
158 158
     result = self.http_upload(
@@ -166,9 +166,9 @@ class Test_files( Test_controller ):
166 166
       session_id = self.session_id,
167 167
     )
168 168
 
169
-    self.assert_inline_error( result )
169
+    self.assert_streaming_error( result )
170 170
 
171
-  def test_upload_file_empty( self ):
171
+  def test_upload_empty( self ):
172 172
     self.login()
173 173
 
174 174
     result = self.http_upload(
@@ -182,12 +182,43 @@ class Test_files( Test_controller ):
182 182
       session_id = self.session_id,
183 183
     )
184 184
 
185
-    self.assert_inline_error( result )
185
+    self.assert_streaming_error( result )
186 186
 
187
-  def test_upload_file_cancel( self ):
188
-    raise NotImplementError()
187
+  def test_upload_invalid_content_length( self ):
188
+    self.login()
189
+
190
+    result = self.http_upload(
191
+      "/files/upload",
192
+      dict(
193
+        notebook_id = self.notebook.object_id,
194
+        note_id = self.note.object_id,
195
+      ),
196
+      filename = self.filename,
197
+      file_data = self.file_data,
198
+      headers = [ ( "Content-Length", "-10" ) ],
199
+      session_id = self.session_id,
200
+    )
201
+
202
+    assert "invalid" in result[ "body" ][ 0 ]
203
+
204
+  def test_upload_cancel( self ):
205
+    self.login()
206
+
207
+    result = self.http_upload(
208
+      "/files/upload",
209
+      dict(
210
+        notebook_id = self.notebook.object_id,
211
+        note_id = self.note.object_id,
212
+      ),
213
+      filename = self.filename,
214
+      file_data = self.file_data,
215
+      simulate_cancel = True,
216
+      session_id = self.session_id,
217
+    )
218
+
219
+    self.assert_streaming_error( result )
189 220
 
190
-  def test_upload_file_over_quota( self ):
221
+  def test_upload_over_quota( self ):
191 222
     raise NotImplementError()
192 223
 
193 224
   def login( self ):

+ 109
- 0
model/File.py View File

@@ -0,0 +1,109 @@
1
+from Persistent import Persistent, quote
2
+from psycopg2 import Binary
3
+from StringIO import StringIO
4
+
5
+
6
+class File( Persistent ):
7
+  """
8
+  Metadata about an uploaded file. The actual file data is stored on the filesystem instead of in
9
+  the database. (Binary conversion to/from PostgreSQL's bytea is too slow, and the version of
10
+  psycopg2 I'm using doesn't have large object support.)
11
+  """
12
+  def __init__( self, object_id, revision = None, notebook_id = None, note_id = None,
13
+                filename = None, size_bytes = None ):
14
+    """
15
+    Create a File with the given id.
16
+
17
+    @type object_id: unicode
18
+    @param object_id: id of the File
19
+    @type revision: datetime or NoneType
20
+    @param revision: revision timestamp of the object (optional, defaults to now)
21
+    @type notebook_id: unicode or NoneType
22
+    @param notebook_id: id of the notebook containing the file
23
+    @type note_id: unicode or NoneType
24
+    @param note_id: id of the note linking to the file
25
+    @type filename: unicode
26
+    @param filename: name of the file on the client
27
+    @type size_bytes: int
28
+    @param size_bytes: length of the file data in bytes
29
+    @rtype: File
30
+    @return: newly constructed File
31
+    """
32
+    Persistent.__init__( self, object_id, revision )
33
+    self.__notebook_id = notebook_id
34
+    self.__note_id = note_id
35
+    self.__filename = filename
36
+    self.__size_bytes = size_bytes
37
+
38
+  @staticmethod
39
+  def create( object_id, notebook_id = None, note_id = None, filename = None, size_bytes = None ):
40
+    """
41
+    Convenience constructor for creating a new File.
42
+
43
+    @type object_id: unicode
44
+    @param object_id: id of the File
45
+    @type notebook_id: unicode or NoneType
46
+    @param notebook_id: id of the notebook containing the file
47
+    @type note_id: unicode or NoneType
48
+    @param note_id: id of the note linking to the file
49
+    @type filename: unicode
50
+    @param filename: name of the file on the client
51
+    @type size_bytes: int
52
+    @param size_bytes: length of the file data in bytes
53
+    @rtype: File
54
+    @return: newly constructed File
55
+    """
56
+    return File( object_id, notebook_id = notebook_id, note_id = note_id, filename = filename,
57
+                 size_bytes = size_bytes )
58
+
59
+  @staticmethod
60
+  def sql_load( object_id, revision = None ):
61
+    # Files don't store old revisions
62
+    if revision:
63
+      raise NotImplementedError()
64
+
65
+    return \
66
+      """
67
+      select
68
+        file.id, file.revision, file.notebook_id, file.note_id, file.filename, size_bytes
69
+      from
70
+        file
71
+      where
72
+        file.id = %s;
73
+      """ % quote( object_id )
74
+
75
+  @staticmethod
76
+  def sql_id_exists( object_id, revision = None ):
77
+    if revision:
78
+      raise NotImplementedError()
79
+
80
+    return "select id from file where id = %s;" % quote( object_id )
81
+
82
+  def sql_exists( self ):
83
+    return File.sql_id_exists( self.object_id )
84
+
85
+  def sql_create( self ):
86
+    return "insert into file ( id, revision, notebook_id, note_id, filename, size_bytes ) values ( %s, %s, %s, %s, %s, %s );" % \
87
+    ( quote( self.object_id ), quote( self.revision ), quote( self.__notebook_id ), quote( self.__note_id ),
88
+      quote( self.__filename ), self.__size_bytes or 'null' )
89
+
90
+  def sql_update( self ):
91
+    return "update file set revision = %s, notebook_id = %s, note_id = %s, filename = %s, size_bytes = %s where id = %s;" % \
92
+    ( quote( self.revision ), quote( self.__notebook_id ), quote( self.__note_id ), quote( self.__filename ),
93
+      self.__size_bytes or 'null', quote( self.object_id ) )
94
+
95
+  def to_dict( self ):
96
+    d = Persistent.to_dict( self )
97
+    d.update( dict(
98
+      notebook_id = self.__notebook_id,
99
+      note_id = self.__note_id,
100
+      filename = self.__filename,
101
+      size_bytes = self.__size_bytes,
102
+    ) )
103
+
104
+    return d
105
+
106
+  notebook_id = property( lambda self: self.__notebook_id )
107
+  note_id = property( lambda self: self.__note_id )
108
+  filename = property( lambda self: self.__filename )
109
+  size_bytes = property( lambda self: self.__size_bytes )

+ 1
- 1
model/Invite.py View File

@@ -66,7 +66,7 @@ class Invite( Persistent ):
66 66
 
67 67
   @staticmethod
68 68
   def sql_load( object_id, revision = None ):
69
-    # password resets don't store old revisions
69
+    # invites don't store old revisions
70 70
     if revision:
71 71
       raise NotImplementedError()
72 72
 

+ 9
- 0
model/delta/1.2.0.sql View File

@@ -0,0 +1,9 @@
1
+create table file (
2
+  id text,
3
+  revision timestamp with time zone,
4
+  notebook_id text,
5
+  note_id text,
6
+  filename text,
7
+  size_bytes integer
8
+);
9
+alter table file add primary key ( id );

+ 2
- 0
model/drop.sql View File

@@ -6,3 +6,5 @@ DROP VIEW notebook_current;
6 6
 DROP TABLE notebook;
7 7
 DROP TABLE password_reset;
8 8
 DROP TABLE user_notebook;
9
+DROP TABLE invite;
10
+DROP TABLE file;

+ 24
- 0
model/schema.sql View File

@@ -31,6 +31,22 @@ CREATE FUNCTION drop_html_tags(text) RETURNS text
31 31
 
32 32
 ALTER FUNCTION public.drop_html_tags(text) OWNER TO luminotes;
33 33
 
34
+--
35
+-- Name: file; Type: TABLE; Schema: public; Owner: luminotes; Tablespace: 
36
+--
37
+
38
+CREATE TABLE file (
39
+    id text NOT NULL,
40
+    revision timestamp with time zone,
41
+    notebook_id text,
42
+    note_id text,
43
+    filename text,
44
+    size_bytes integer
45
+);
46
+
47
+
48
+ALTER TABLE public.file OWNER TO luminotes;
49
+
34 50
 --
35 51
 -- Name: invite; Type: TABLE; Schema: public; Owner: luminotes; Tablespace: 
36 52
 --
@@ -161,6 +177,14 @@ CREATE TABLE user_notebook (
161 177
 
162 178
 ALTER TABLE public.user_notebook OWNER TO luminotes;
163 179
 
180
+
181
+-- Name: file_pkey; Type: CONSTRAINT; Schema: public; Owner: luminotes; Tablespace: 
182
+--
183
+
184
+ALTER TABLE ONLY file
185
+    ADD CONSTRAINT file_pkey PRIMARY KEY (id);
186
+
187
+
164 188
 --
165 189
 -- Name: invite_pkey; Type: CONSTRAINT; Schema: public; Owner: luminotes; Tablespace: 
166 190
 --

+ 33
- 0
model/test/Test_file.py View File

@@ -0,0 +1,33 @@
1
+from pytz import utc
2
+from datetime import datetime, timedelta
3
+from model.File import File
4
+
5
+
6
+class Test_file( object ):
7
+  def setUp( self ):
8
+    self.object_id = u"17"
9
+    self.notebook_id = u"18"
10
+    self.note_id = u"19"
11
+    self.filename = u"foo.png"
12
+    self.size_bytes = 2888
13
+    self.delta = timedelta( seconds = 1 )
14
+
15
+    self.file = File.create( self.object_id, self.notebook_id, self.note_id, self.filename,
16
+                             self.size_bytes )
17
+
18
+  def test_create( self ):
19
+    assert self.file.object_id == self.object_id
20
+    assert self.file.notebook_id == self.notebook_id
21
+    assert self.file.note_id == self.note_id
22
+    assert self.file.filename == self.filename
23
+    assert self.file.size_bytes == self.size_bytes
24
+
25
+  def test_to_dict( self ):
26
+    d = self.file.to_dict()
27
+
28
+    assert d.get( "object_id" ) == self.object_id
29
+    assert datetime.now( tz = utc ) - d.get( "revision" ) < self.delta
30
+    assert d.get( "notebook_id" ) == self.notebook_id
31
+    assert d.get( "note_id" ) == self.note_id
32
+    assert d.get( "filename" ) == self.filename
33
+    assert d.get( "size_bytes" ) == self.size_bytes

+ 0
- 1
model/test/Test_invite.py View File

@@ -1,6 +1,5 @@
1 1
 from pytz import utc
2 2
 from datetime import datetime, timedelta
3
-from model.User import User
4 3
 from model.Invite import Invite
5 4
 
6 5
 

+ 0
- 1
model/test/Test_password_reset.py View File

@@ -1,4 +1,3 @@
1
-from model.User import User
2 1
 from model.Password_reset import Password_reset
3 2
 
4 3
 

+ 1
- 1
static/css/style.css View File

@@ -615,7 +615,7 @@ img {
615 615
   color: #ff6600;
616 616
 }
617 617
 
618
-#upload_frame {
618
+.upload_frame {
619 619
   padding: 0;
620 620
   margin: 0;
621 621
   width: 40em;

+ 6
- 0
static/js/Editor.js View File

@@ -378,6 +378,12 @@ Editor.prototype.mouse_clicked = function ( event ) {
378 378
     return;
379 379
   }
380 380
 
381
+  // special case for links to uploaded files
382
+  if ( !link.target && /\/files\//.test( link.href ) ) {
383
+    location.href = link.href;
384
+    return;
385
+  }
386
+
381 387
   event.stop();
382 388
 
383 389
   // load the note corresponding to the clicked link

+ 36
- 9
static/js/Wiki.js View File

@@ -113,9 +113,13 @@ Wiki.prototype.update_next_id = function ( result ) {
113 113
 
114 114
 var KILOBYTE = 1024;
115 115
 var MEGABYTE = 1024 * KILOBYTE;
116
-function bytes_to_megabytes( bytes, or_kilobytes ) {
117
-  if ( or_kilobytes && bytes < MEGABYTE )
118
-    return Math.round( bytes / KILOBYTE ) + " KB";
116
+function bytes_to_megabytes( bytes, choose_units ) {
117
+  if ( choose_units ) {
118
+    if ( bytes < KILOBYTE )
119
+      return bytes + " bytes";
120
+    if ( bytes < MEGABYTE )
121
+      return Math.round( bytes / KILOBYTE ) + " KB";
122
+  }
119 123
 
120 124
   return Math.round( bytes / MEGABYTE ) + " MB";
121 125
 }
@@ -731,7 +735,7 @@ Wiki.prototype.display_link_pulldown = function ( editor, link ) {
731 735
   if ( link_title( link ).length > 0 ) {
732 736
     if ( !pulldown ) {
733 737
       this.clear_pulldowns();
734
-      // display a different pulldown dependong on whether the link is a note link or a file link
738
+      // display a different pulldown depending on whether the link is a note link or a file link
735 739
       if ( link.target || !/\/files\//.test( link.href ) )
736 740
         new Link_pulldown( this, this.notebook_id, this.invoker, editor, link );
737 741
       else
@@ -2246,9 +2250,11 @@ function Upload_pulldown( wiki, notebook_id, invoker, editor ) {
2246 2250
     "frameBorder": "0",
2247 2251
     "scrolling": "no",
2248 2252
     "id": "upload_frame",
2249
-    "name": "upload_frame"
2253
+    "name": "upload_frame",
2254
+    "class": "upload_frame"
2250 2255
   } );
2251 2256
   this.iframe.pulldown = this;
2257
+  this.file_id = null;
2252 2258
 
2253 2259
   var self = this;
2254 2260
   connect( this.iframe, "onload", function ( event ) { self.init_frame(); } );
@@ -2266,26 +2272,47 @@ Upload_pulldown.prototype.init_frame = function () {
2266 2272
   withDocument( doc, function () {
2267 2273
     connect( "upload_button", "onclick", function ( event ) {
2268 2274
       withDocument( doc, function () {
2269
-        self.upload_started( getElement( "upload" ).value );
2275
+        self.upload_started( getElement( "file_id" ).value, getElement( "upload" ).value );
2270 2276
       } );
2271 2277
     } );
2272 2278
   } );
2273 2279
 }
2274 2280
 
2275
-Upload_pulldown.prototype.upload_started = function ( filename ) {
2281
+Upload_pulldown.prototype.upload_started = function ( file_id, filename ) {
2282
+  this.file_id = file_id;
2283
+
2284
+  // make the upload iframe invisible but still present so that the upload continues
2285
+  addElementClass( this.iframe, "invisible" );
2286
+  setElementDimensions( this.iframe, { "h": "0" } );
2287
+
2276 2288
   // get the basename of the file
2277 2289
   var pieces = filename.split( "/" );
2278 2290
   filename = pieces[ pieces.length - 1 ];
2279 2291
   pieces = filename.split( "\\" );
2280 2292
   filename = pieces[ pieces.length - 1 ];
2281 2293
 
2282
-  // the current title is blank, replace the title with the upload's filename
2294
+  // if the current title is blank, replace the title with the upload's filename
2283 2295
   if ( link_title( this.link ) == "" )
2284 2296
     replaceChildNodes( this.link, this.editor.document.createTextNode( filename ) );
2285
-  // TODO: set the link's href to the file
2297
+
2298
+  // FIXME: this call might occur before upload() is even called
2299
+  var progress_iframe = createDOM( "iframe", {
2300
+    "src": "/files/progress?file_id=" + file_id + "&filename=" + escape( filename ),
2301
+    "frameBorder": "0",
2302
+    "scrolling": "no",
2303
+    "id": "progress_frame",
2304
+    "name": "progress_frame",
2305
+    "class": "upload_frame"
2306
+  } );
2307
+
2308
+  appendChildNodes( this.div, progress_iframe );
2286 2309
 }
2287 2310
 
2288 2311
 Upload_pulldown.prototype.upload_complete = function () {
2312
+  // now that the upload is done, the file link should point to the uploaded file
2313
+  this.link.href = "/files/download?file_id=" + this.file_id
2314
+
2315
+// FIXME: the upload pulldown is sometimes being closed here before the upload is complete, thereby truncating the upload
2289 2316
   new File_link_pulldown( this.wiki, this.notebook_id, this.invoker, this.editor, this.link );
2290 2317
   this.shutdown();
2291 2318
 }

+ 5
- 4
view/Upload_page.py View File

@@ -2,7 +2,7 @@ from Tags import Html, Head, Link, Meta, Body, P, Form, Span, Input
2 2
 
3 3
 
4 4
 class Upload_page( Html ):
5
-  def __init__( self, notebook_id, note_id ):
5
+  def __init__( self, notebook_id, note_id, file_id ):
6 6
     Html.__init__(
7 7
       self,
8 8
       Head(
@@ -12,15 +12,16 @@ class Upload_page( Html ):
12 12
       Body(
13 13
         Form(
14 14
           Span( u"attach file: ", class_ = u"field_label" ),
15
-          Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ),
16
-          Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ),
17 15
           Input( type = u"hidden", id = u"notebook_id", name = u"notebook_id", value = notebook_id ),
18 16
           Input( type = u"hidden", id = u"note_id", name = u"note_id", value = note_id ),
19
-          action = u"/files/upload",
17
+          Input( type = u"file", id = u"upload", name = u"upload", class_ = "text_field", size = u"30" ),
18
+          Input( type = u"submit", id = u"upload_button", class_ = u"button", value = u"upload" ),
19
+          action = u"/files/upload?file_id=%s" % file_id,
20 20
           method = u"post",
21 21
           enctype = u"multipart/form-data",
22 22
         ),
23 23
         P( u"Please select a file to upload." ),
24 24
         Span( id = u"tick_preload" ),
25
+        Input( type = u"hidden", id = u"file_id", value = file_id ),
25 26
       ),
26 27
     )