Browse Source

New support for nginx's X-Accel-Redirect for downloading files. Also made Files.thumbnail() non-streaming.

Dan Helfman 9 years ago
parent
commit
3013beddbd
5 changed files with 118 additions and 27 deletions
  1. 1
    1
      config/Common.py
  2. 12
    13
      controller/Expose.py
  3. 20
    10
      controller/Files.py
  4. 1
    0
      controller/Root.py
  5. 84
    3
      controller/test/Test_files.py

+ 1
- 1
config/Common.py View File

@@ -23,6 +23,7 @@ settings = {
23 23
     "luminotes.https_proxy_ip": "127.0.0.2",
24 24
     "luminotes.db_host": "localhost", # hostname for PostgreSQL or None (no quotes) for SQLite
25 25
     "luminotes.db_ssl_mode": "allow", # "disallow", "allow", "prefer", or "require"
26
+    "luminotes.web_server": "", # "", "apache", or "nginx" to use specific server support (optional)
26 27
     "luminotes.support_email": "",
27 28
     "luminotes.payment_email": "",
28 29
     "luminotes.rate_plans": [
@@ -136,7 +137,6 @@ settings = {
136 137
     "encoding_filter.on": False,
137 138
   },
138 139
   "/files/thumbnail": {
139
-    "stream_response": True,
140 140
     "encoding_filter.on": False,
141 141
   },
142 142
   "/files/image": {

+ 12
- 13
controller/Expose.py View File

@@ -66,8 +66,8 @@ def expose( view = None, rss = None ):
66 66
           cherrypy.root.report_traceback()
67 67
           result = dict( error = u"An error occurred when processing your request. Please try again or contact support." )
68 68
 
69
-      # if the result is a generator, it's streaming data, so just let CherryPy handle it
70
-      if isinstance( result, types.GeneratorType ):
69
+      # if the result is a generator or a string, it's streaming data or just data, so just let CherryPy handle it
70
+      if isinstance( result, ( types.GeneratorType, basestring ) ):
71 71
         return result
72 72
 
73 73
       redirect = result.get( u"redirect" )
@@ -83,18 +83,17 @@ def expose( view = None, rss = None ):
83 83
 
84 84
       # try using the supplied view to render the result
85 85
       try:
86
-        if view_override is None:
87
-          if rss and use_rss:
88
-            cherrypy.response.headers[ u"Content-Type" ] = u"application/xml"
89
-            return render( rss, result, encoding or "utf8" )
90
-          elif view:
91
-            return render( view, result, encoding )
92
-          elif result.get( "view" ):
93
-            result_view = result.get( "view" )
94
-            del( result[ "view" ] )
95
-            return render( result_view, result, encoding )
96
-        else:
86
+        if view_override is not None:
97 87
           return render( view_override, result, encoding )
88
+        elif rss and use_rss:
89
+          cherrypy.response.headers[ u"Content-Type" ] = u"application/xml"
90
+          return render( rss, result, encoding or "utf8" )
91
+        elif view:
92
+          return render( view, result, encoding )
93
+        elif result.get( "view" ):
94
+          result_view = result.get( "view" )
95
+          del( result[ "view" ] )
96
+          return render( result_view, result, encoding )
98 97
       except:
99 98
         if redirect is None:
100 99
           if original_error:

+ 20
- 10
controller/Files.py View File

@@ -244,7 +244,7 @@ class Files( object ):
244 244
   """
245 245
   Controller for dealing with uploaded files, corresponding to the "/files" URL.
246 246
   """
247
-  def __init__( self, database, users, download_products ):
247
+  def __init__( self, database, users, download_products, web_server ):
248 248
     """
249 249
     Create a new Files object.
250 250
 
@@ -254,12 +254,15 @@ class Files( object ):
254 254
     @param users: controller for all users
255 255
     @type download_products: [ { "name": unicode, ... } ]
256 256
     @param download_products: list of configured downloadable products
257
+    @type web_server: unicode
258
+    @param web_server: front-end web server (determines specific support for various features)
257 259
     @rtype: Files
258 260
     @return: newly constructed Files
259 261
     """
260 262
     self.__database = database
261 263
     self.__users = users
262 264
     self.__download_products = download_products
265
+    self.__web_server = web_server
263 266
 
264 267
   @expose()
265 268
   @weakly_expire
@@ -312,9 +315,14 @@ class Files( object ):
312 315
     cherrypy.response.headerMap[ u"Content-Disposition" ] = 'attachment; filename="%s"' % filename
313 316
     cherrypy.response.headerMap[ u"Content-Length" ] = db_file.size_bytes
314 317
 
318
+    if self.__web_server == u"nginx":
319
+      cherrypy.response.headerMap[ u"X-Accel-Redirect" ] = "/download/%s" % file_id
320
+      return ""
321
+
315 322
     def stream():
316 323
       CHUNK_SIZE = 8192
317 324
       local_file = Upload_file.open_file( file_id )
325
+      local_file.seek(0)
318 326
 
319 327
       while True:
320 328
         data = local_file.read( CHUNK_SIZE )
@@ -364,9 +372,14 @@ class Files( object ):
364 372
     cherrypy.response.headerMap[ u"Content-Disposition" ] = 'attachment; filename="%s"' % public_filename
365 373
     cherrypy.response.headerMap[ u"Content-Length" ] = os.path.getsize( local_filename )
366 374
 
375
+    if self.__web_server == u"nginx":
376
+      cherrypy.response.headerMap[ u"X-Accel-Redirect" ] = "/download_product/%s" % product[ u"filename" ]
377
+      return ""
378
+
367 379
     def stream():
368 380
       CHUNK_SIZE = 8192
369 381
       local_file = file( local_filename, "rb" )
382
+      local_file.seek(0)
370 383
 
371 384
       while True:
372 385
         data = local_file.read( CHUNK_SIZE )
@@ -463,15 +476,7 @@ class Files( object ):
463 476
       image.save( image_buffer, "PNG" )
464 477
       image_buffer.seek( 0 )
465 478
 
466
-    def stream( image_buffer ):
467
-      CHUNK_SIZE = 8192
468
-
469
-      while True:
470
-        data = image_buffer.read( CHUNK_SIZE )
471
-        if len( data ) == 0: break
472
-        yield data        
473
-
474
-    return stream( image_buffer )
479
+    return image_buffer.getvalue()
475 480
 
476 481
   @expose()
477 482
   @weakly_expire
@@ -501,9 +506,14 @@ class Files( object ):
501 506
 
502 507
     cherrypy.response.headerMap[ u"Content-Type" ] = db_file.content_type
503 508
 
509
+    if self.__web_server == u"nginx":
510
+      cherrypy.response.headerMap[ u"X-Accel-Redirect" ] = "/download/%s" % file_id
511
+      return ""
512
+
504 513
     def stream():
505 514
       CHUNK_SIZE = 8192
506 515
       local_file = Upload_file.open_file( file_id )
516
+      local_file.seek(0)
507 517
 
508 518
       while True:
509 519
         data = local_file.read( CHUNK_SIZE )

+ 1
- 0
controller/Root.py View File

@@ -57,6 +57,7 @@ class Root( object ):
57 57
       database,
58 58
       self.__users,
59 59
       settings[ u"global" ].get( u"luminotes.download_products", [] ),
60
+      settings[ u"global" ].get( u"luminotes.web_server", "" ),
60 61
     )
61 62
     self.__notebooks = Notebooks( database, self.__users, self.__files, settings[ u"global" ].get( u"luminotes.https_url", u"" ) )
62 63
     self.__forums = Forums( database, self.__notebooks, self.__users )

+ 84
- 3
controller/test/Test_files.py View File

@@ -137,9 +137,14 @@ class Test_files( Test_controller ):
137 137
 
138 138
     os.remove( u"products/test.exe" )
139 139
 
140
-  def test_download( self, filename = None, quote_filename = None, file_data = None, preview = None ):
140
+  def test_download( self, filename = None, quote_filename = None, file_data = None, preview = None, expected_file_data = None ):
141 141
     self.login()
142 142
 
143
+    if expected_file_data is None:
144
+      expected_file_data = file_data
145
+      if file_data is None:
146
+        expected_file_data = self.file_data
147
+
143 148
     self.http_upload(
144 149
       "/files/upload?file_id=%s" % self.file_id,
145 150
       dict(
@@ -191,8 +196,17 @@ class Test_files( Test_controller ):
191 196
       if u"session_storage" not in str( exc ):
192 197
         raise exc
193 198
 
194
-    file_data = "".join( pieces )
195
-    assert file_data == ( file_data or self.file_data )
199
+    received_file_data = "".join( pieces )
200
+    assert received_file_data == expected_file_data
201
+
202
+    return result
203
+
204
+  def test_download_with_nginx( self ):
205
+    cherrypy.root.files._Files__web_server = u"nginx"
206
+    result = self.test_download( self.filename, expected_file_data = "" )
207
+
208
+    headers = result[ u"headers" ]
209
+    assert headers[ u"X-Accel-Redirect" ] == u"/download/%s" % self.file_id
196 210
 
197 211
   def test_download_with_unicode_filename( self ):
198 212
     self.test_download( self.unicode_filename )
@@ -372,6 +386,44 @@ class Test_files( Test_controller ):
372 386
     file_data = "".join( pieces )
373 387
     assert file_data == self.file_data
374 388
 
389
+  def test_download_product_with_nginx( self ):
390
+    cherrypy.root.files._Files__web_server = u"nginx"
391
+    access_id = u"wheeaccessid"
392
+    item_number = u"5000"
393
+    transaction_id = u"txn"
394
+
395
+    self.login()
396
+
397
+    download_access = Download_access.create( access_id, item_number, transaction_id )
398
+    self.database.save( download_access )
399
+
400
+    result = self.http_get(
401
+      "/files/download_product?access_id=%s" % access_id,
402
+      session_id = self.session_id,
403
+    )
404
+
405
+    headers = result[ u"headers" ]
406
+    assert headers
407
+    assert headers[ u"Content-Type" ] == u"application/octet-stream"
408
+
409
+    filename = u"test.exe".encode( "utf8" )
410
+    assert headers[ u"Content-Disposition" ] == 'attachment; filename="%s"' % filename
411
+    assert headers[ u"X-Accel-Redirect" ] == u"/download_product/test.exe"
412
+
413
+    gen = result[ u"body" ]
414
+    assert isinstance( gen, types.GeneratorType )
415
+    pieces = []
416
+
417
+    try:
418
+      for piece in gen:
419
+        pieces.append( piece )
420
+    except AttributeError, exc:
421
+      if u"session_storage" not in str( exc ):
422
+        raise exc
423
+
424
+    file_data = "".join( pieces )
425
+    assert file_data == u""
426
+
375 427
   def test_download_product_without_login( self ):
376 428
     access_id = u"wheeaccessid"
377 429
     item_number = u"5000"
@@ -924,6 +976,35 @@ class Test_files( Test_controller ):
924 976
 
925 977
     assert "".join( result[ u"body" ] ) == self.IMAGE_DATA
926 978
 
979
+  def test_image_with_nginx( self ):
980
+    cherrypy.root.files._Files__web_server = u"nginx"
981
+    self.login()
982
+
983
+    self.http_upload(
984
+      "/files/upload?file_id=%s" % self.file_id,
985
+      dict(
986
+        notebook_id = self.notebook.object_id,
987
+        note_id = self.note.object_id,
988
+      ),
989
+      filename = self.filename,
990
+      file_data = self.IMAGE_DATA,
991
+      content_type = self.content_type,
992
+      session_id = self.session_id,
993
+    )
994
+
995
+    result = self.http_get(
996
+      "/files/image?file_id=%s" % self.file_id,
997
+      session_id = self.session_id,
998
+    )
999
+
1000
+    headers = result[ u"headers" ]
1001
+    assert headers
1002
+    assert headers[ u"Content-Type" ] == self.content_type
1003
+    assert u"Content-Disposition" not in headers
1004
+    assert headers[ u"X-Accel-Redirect" ] == u"/download/%s" % self.file_id
1005
+
1006
+    assert "".join( result[ u"body" ] ) == u""
1007
+
927 1008
   def test_image_with_non_image( self ):
928 1009
     self.login()
929 1010
 

Loading…
Cancel
Save