witten
/
luminotes
Archived
1
0
Fork 0
Code Releases Activity

Now using memcached in various places to improve performance. If the Python 

cmemcache module is not importable, then memcached simply won't be used.
This commit is contained in:
Dan Helfman 2008-03-05 00:34:58 +00:00
parent 1bd3cacb42
commit 21ccc97826
9 changed files with 116 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
INSTALL
View File

@ -143,6 +143,38 @@ shows up in various error messages and other places for a support contact
address.
memcached
---------
For improved performance, it is recommended that you install and use memcached
for production servers.
First, install the prerequisites:
* python-dev 2.4
* libmemcache-dev 1.4
* memcached 1.4
* cmemcache 0.91
In Debian GNU/Linux, you can issue the following command to install these
packages:
apt-get install python2.4-dev libmemcache-dev memcached
The cmemcache package is not currently included with Debian Etch, so you'll
have to build and install it manually. Download and untar the package from:
http://gijsbert.org/cmemcache/
From the untarred cmemcache directory, issue the following command as root:
python2.4 setup.py install
This should build and install the cmemcache module. Once installed, Luminotes
will use the module automatically. When Luminotes starts up, you should see a
"using memcached" message.
Python unit tests
-----------------

3
NEWS
View File

@ -1,4 +1,5 @@
1.2.2: March ?, 2008
1.2.2: March 4, 2008
* Introduced database object caching to improve performance.
* Wrote a database reaper script to delete unused notes, notebooks, etc.
* Added some database indices to improve select performance.
* Now scrolling the page vertically to show opened errors and messages.

53
controller/Database.py
View File

@ -1,20 +1,24 @@
import re
import os
import sha
import psycopg2 as psycopg
from psycopg2.pool import PersistentConnectionPool
import random
from model.Persistent import Persistent
class Database( object ):
ID_BITS = 128 # number of bits within an id
ID_DIGITS = "0123456789abcdefghijklmnopqrstuvwxyz"
def __init__( self, connection = None ):
def __init__( self, connection = None, cache = None ):
"""
Create a new database and return it.
@type connection: existing connection object with cursor()/close()/commit() methods, or NoneType
@param connection: database connection to use (optional, defaults to making a connection pool)
@type cache: cmemcache.Client or something with a similar API, or NoneType
@param cache: existing memory cache to use (optional, defaults to making a cache)
@rtype: Database
@return: newly constructed Database
"""
@ -33,6 +37,15 @@ class Database( object ):
"dbname=luminotes user=luminotes password=%s" % os.getenv( "PGPASSWORD", "dev" ),
)
self.__cache = cache
if not cache:
try:
import cmemcache
self.__cache = cmemcache.Client( [ "127.0.0.1:11211" ], debug = 0 )
print "using memcached"
except ImportError:
pass
def __get_connection( self ):
if self.__connection:
return self.__connection
@ -59,6 +72,8 @@ class Database( object ):
if commit:
connection.commit()
if self.__cache:
self.__cache.set( obj.cache_key, obj )
def commit( self ):
self.__get_connection().commit()
@ -78,9 +93,18 @@ class Database( object ):
@rtype: Object_type or NoneType
@return: loaded object, or None if no match
"""
return self.select_one( Object_type, Object_type.sql_load( object_id, revision ) )
if revision is None and self.__cache: # don't bother caching old revisions
obj = self.__cache.get( Persistent.make_cache_key( Object_type, object_id ) )
if obj:
return obj
def select_one( self, Object_type, sql_command ):
obj = self.select_one( Object_type, Object_type.sql_load( object_id, revision ) )
if obj and revision is None and self.__cache:
self.__cache.set( obj.cache_key, obj )
return obj
def select_one( self, Object_type, sql_command, use_cache = False ):
"""
Execute the given sql_command and return its results in the form of an object of Object_type,
or None if there was no match.
@ -89,9 +113,17 @@ class Database( object ):
@param Object_type: class of the object to load
@type sql_command: unicode
@param sql_command: SQL command to execute
@type use_cache: bool
@param use_cache: whether to look for and store objects in the cache
@rtype: Object_type or NoneType
@return: loaded object, or None if no match
"""
if use_cache and self.__cache:
cache_key = sha.new( sql_command ).hexdigest()
obj = self.__cache.get( cache_key )
if obj:
return obj
connection = self.__get_connection()
cursor = connection.cursor()
@ -102,9 +134,14 @@ class Database( object ):
return None
if Object_type in ( tuple, list ):
return Object_type( row )
obj = Object_type( row )
else:
return Object_type( *row )
obj = Object_type( *row )
if obj and use_cache and self.__cache:
self.__cache.set( cache_key, obj )
return obj
def select_many( self, Object_type, sql_command ):
"""
@ -160,6 +197,12 @@ class Database( object ):
if commit:
connection.commit()
def uncache_command( self, sql_command ):
if not self.__cache: return
cache_key = sha.new( sql_command ).hexdigest()
self.__cache.delete( cache_key )
@staticmethod
def generate_id():
int_id = random.getrandbits( Database.ID_BITS )

6
controller/Notebooks.py
View File

@ -190,7 +190,7 @@ class Notebooks( object ):
note = None
startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() )
total_notes_count = self.__database.select_one( int, notebook.sql_count_notes() )
total_notes_count = self.__database.select_one( int, notebook.sql_count_notes(), use_cache = True )
if self.__users.check_access( user_id, notebook_id, owner = True ):
invites = self.__database.select_many( Invite, Invite.sql_load_notebook_invites( notebook_id ) )
@ -549,6 +549,7 @@ class Notebooks( object ):
if new_revision:
self.__database.save( note, commit = False )
user = self.__users.update_storage( user_id, commit = False )
self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
self.__database.commit()
else:
user = None
@ -605,6 +606,7 @@ class Notebooks( object ):
self.__database.save( note, commit = False )
user = self.__users.update_storage( user_id, commit = False )
self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
self.__database.commit()
return dict( storage_bytes = user.storage_bytes )
@ -660,6 +662,7 @@ class Notebooks( object ):
self.__database.save( note, commit = False )
user = self.__users.update_storage( user_id, commit = False )
self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
self.__database.commit()
return dict( storage_bytes = user.storage_bytes )
@ -710,6 +713,7 @@ class Notebooks( object ):
self.__database.save( note, commit = False )
user = self.__users.update_storage( user_id, commit = False )
self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
self.__database.commit()
return dict(

16
controller/Users.py
View File

@ -435,7 +435,7 @@ class Users( object ):
@raise Access_error: user_id or anonymous user unknown
"""
# if there's no logged-in user, default to the anonymous user
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
if user_id:
user = self.__database.load( User, user_id )
else:
@ -513,7 +513,7 @@ class Users( object ):
@rtype: bool
@return: True if the user has access
"""
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
if self.__database.select_one( bool, anonymous.sql_has_access( notebook_id, read_write, owner ) ):
return True
@ -600,7 +600,7 @@ class Users( object ):
@raise Password_reset_error: an error occured when redeeming the password reset, such as an expired link
@raise Validation_error: one of the arguments is invalid
"""
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
if anonymous:
main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) )
@ -624,7 +624,7 @@ class Users( object ):
result = self.current( anonymous.object_id )
result[ "notebook" ] = main_notebook
result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() )
result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() )
result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True )
result[ "note_read_write" ] = False
result[ "notes" ] = [ Note.create(
object_id = u"password_reset",
@ -921,7 +921,7 @@ class Users( object ):
if not notebook:
raise Invite_error( "That notebook you've been invited to is unknown." )
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
if anonymous:
main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) )
invite_notebook = self.__database.load( Notebook, invite.notebook_id )
@ -933,7 +933,7 @@ class Users( object ):
result = self.current( anonymous.object_id )
result[ "notebook" ] = main_notebook
result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() )
result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() )
result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True )
result[ "note_read_write" ] = False
result[ "notes" ] = [ Note.create(
object_id = u"redeem_invite",
@ -1086,7 +1086,7 @@ class Users( object ):
"""
Provide the information necessary to display the subscription thanks page.
"""
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
if anonymous:
main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) )
else:
@ -1120,7 +1120,7 @@ class Users( object ):
result[ "notebook" ] = main_notebook
result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() )
result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() )
result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True )
result[ "note_read_write" ] = False
result[ "notes" ] = [ Note.create(
object_id = u"thanks",

9
controller/test/Stub_cache.py Normal file
View File

@ -0,0 +1,9 @@
class Stub_cache( object ):
def __init__( self ):
self.__objects = {}
def get( self, key ):
return self.__objects.get( key )
def set( self, key, value ):
self.__objects[ key ] = value

5
controller/test/Stub_database.py
View File

@ -42,7 +42,7 @@ class Stub_database( object ):
return None
def select_one( self, Object_type, sql_command ):
def select_one( self, Object_type, sql_command, use_cache = False ):
if callable( sql_command ):
result = sql_command( self )
if isinstance( result, list ):
@ -67,6 +67,9 @@ class Stub_database( object ):
raise NotImplementedError( sql_command )
def uncache_command( self, sql_command ):
pass
def next_id( self, Object_type, commit = True ):
self.__next_id += 1
return unicode( self.__next_id )

4
controller/test/Test_database.py
View File

@ -2,6 +2,7 @@ from pytz import utc
from pysqlite2 import dbapi2 as sqlite
from datetime import datetime
from Stub_object import Stub_object
from Stub_cache import Stub_cache
from controller.Database import Database
@ -9,10 +10,11 @@ class Test_database( object ):
def setUp( self ):
# make an in-memory sqlite database to use in place of PostgreSQL during testing
self.connection = sqlite.connect( ":memory:", detect_types = sqlite.PARSE_DECLTYPES | sqlite.PARSE_COLNAMES )
self.cache = Stub_cache()
cursor = self.connection.cursor()
cursor.execute( Stub_object.sql_create_table() )
self.database = Database( self.connection )
self.database = Database( self.connection, self.cache )
def tearDown( self ):
self.database.close()

5
model/Persistent.py
View File

@ -70,8 +70,13 @@ class Persistent( object ):
def update_revision( self ):
self.__revision = datetime.now( tz = utc )
@staticmethod
def make_cache_key( Object_type, object_id ):
return "%s_%s" % ( object_id, Object_type.__name__ )
object_id = property( lambda self: self.__object_id )
revision = property( lambda self: self.__revision )
cache_key = property( lambda self: Persistent.make_cache_key( type( self ), self.object_id ) )
def quote( value ):