From 21ccc9782644c0b61df16f7f7416cf4369910fac Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 5 Mar 2008 00:34:58 +0000 Subject: [PATCH] Now using memcached in various places to improve performance. If the Python cmemcache module is not importable, then memcached simply won't be used. --- INSTALL | 32 +++++++++++++++++++ NEWS | 3 +- controller/Database.py | 53 +++++++++++++++++++++++++++++--- controller/Notebooks.py | 6 +++- controller/Users.py | 16 +++++----- controller/test/Stub_cache.py | 9 ++++++ controller/test/Stub_database.py | 5 ++- controller/test/Test_database.py | 4 ++- model/Persistent.py | 5 +++ 9 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 controller/test/Stub_cache.py diff --git a/INSTALL b/INSTALL index bade66d..2d8ddb3 100644 --- a/INSTALL +++ b/INSTALL @@ -143,6 +143,38 @@ shows up in various error messages and other places for a support contact address. +memcached +--------- + +For improved performance, it is recommended that you install and use memcached +for production servers. + +First, install the prerequisites: + + * python-dev 2.4 + * libmemcache-dev 1.4 + * memcached 1.4 + * cmemcache 0.91 + +In Debian GNU/Linux, you can issue the following command to install these +packages: + + apt-get install python2.4-dev libmemcache-dev memcached + +The cmemcache package is not currently included with Debian Etch, so you'll +have to build and install it manually. Download and untar the package from: + + http://gijsbert.org/cmemcache/ + +From the untarred cmemcache directory, issue the following command as root: + + python2.4 setup.py install + +This should build and install the cmemcache module. Once installed, Luminotes +will use the module automatically. When Luminotes starts up, you should see a +"using memcached" message. + + Python unit tests ----------------- diff --git a/NEWS b/NEWS index 682d436..3e31ae4 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ -1.2.2: March ?, 2008 +1.2.2: March 4, 2008 + * Introduced database object caching to improve performance. * Wrote a database reaper script to delete unused notes, notebooks, etc. * Added some database indices to improve select performance. * Now scrolling the page vertically to show opened errors and messages. diff --git a/controller/Database.py b/controller/Database.py index a3ea79d..cc90189 100644 --- a/controller/Database.py +++ b/controller/Database.py @@ -1,20 +1,24 @@ import re import os +import sha import psycopg2 as psycopg from psycopg2.pool import PersistentConnectionPool import random +from model.Persistent import Persistent class Database( object ): ID_BITS = 128 # number of bits within an id ID_DIGITS = "0123456789abcdefghijklmnopqrstuvwxyz" - def __init__( self, connection = None ): + def __init__( self, connection = None, cache = None ): """ Create a new database and return it. @type connection: existing connection object with cursor()/close()/commit() methods, or NoneType @param connection: database connection to use (optional, defaults to making a connection pool) + @type cache: cmemcache.Client or something with a similar API, or NoneType + @param cache: existing memory cache to use (optional, defaults to making a cache) @rtype: Database @return: newly constructed Database """ @@ -33,6 +37,15 @@ class Database( object ): "dbname=luminotes user=luminotes password=%s" % os.getenv( "PGPASSWORD", "dev" ), ) + self.__cache = cache + if not cache: + try: + import cmemcache + self.__cache = cmemcache.Client( [ "127.0.0.1:11211" ], debug = 0 ) + print "using memcached" + except ImportError: + pass + def __get_connection( self ): if self.__connection: return self.__connection @@ -59,6 +72,8 @@ class Database( object ): if commit: connection.commit() + if self.__cache: + self.__cache.set( obj.cache_key, obj ) def commit( self ): self.__get_connection().commit() @@ -78,9 +93,18 @@ class Database( object ): @rtype: Object_type or NoneType @return: loaded object, or None if no match """ - return self.select_one( Object_type, Object_type.sql_load( object_id, revision ) ) + if revision is None and self.__cache: # don't bother caching old revisions + obj = self.__cache.get( Persistent.make_cache_key( Object_type, object_id ) ) + if obj: + return obj - def select_one( self, Object_type, sql_command ): + obj = self.select_one( Object_type, Object_type.sql_load( object_id, revision ) ) + if obj and revision is None and self.__cache: + self.__cache.set( obj.cache_key, obj ) + + return obj + + def select_one( self, Object_type, sql_command, use_cache = False ): """ Execute the given sql_command and return its results in the form of an object of Object_type, or None if there was no match. @@ -89,9 +113,17 @@ class Database( object ): @param Object_type: class of the object to load @type sql_command: unicode @param sql_command: SQL command to execute + @type use_cache: bool + @param use_cache: whether to look for and store objects in the cache @rtype: Object_type or NoneType @return: loaded object, or None if no match """ + if use_cache and self.__cache: + cache_key = sha.new( sql_command ).hexdigest() + obj = self.__cache.get( cache_key ) + if obj: + return obj + connection = self.__get_connection() cursor = connection.cursor() @@ -102,9 +134,14 @@ class Database( object ): return None if Object_type in ( tuple, list ): - return Object_type( row ) + obj = Object_type( row ) else: - return Object_type( *row ) + obj = Object_type( *row ) + + if obj and use_cache and self.__cache: + self.__cache.set( cache_key, obj ) + + return obj def select_many( self, Object_type, sql_command ): """ @@ -160,6 +197,12 @@ class Database( object ): if commit: connection.commit() + def uncache_command( self, sql_command ): + if not self.__cache: return + + cache_key = sha.new( sql_command ).hexdigest() + self.__cache.delete( cache_key ) + @staticmethod def generate_id(): int_id = random.getrandbits( Database.ID_BITS ) diff --git a/controller/Notebooks.py b/controller/Notebooks.py index dd63621..f6bdcf1 100644 --- a/controller/Notebooks.py +++ b/controller/Notebooks.py @@ -190,7 +190,7 @@ class Notebooks( object ): note = None startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() ) - total_notes_count = self.__database.select_one( int, notebook.sql_count_notes() ) + total_notes_count = self.__database.select_one( int, notebook.sql_count_notes(), use_cache = True ) if self.__users.check_access( user_id, notebook_id, owner = True ): invites = self.__database.select_many( Invite, Invite.sql_load_notebook_invites( notebook_id ) ) @@ -549,6 +549,7 @@ class Notebooks( object ): if new_revision: self.__database.save( note, commit = False ) user = self.__users.update_storage( user_id, commit = False ) + self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid self.__database.commit() else: user = None @@ -605,6 +606,7 @@ class Notebooks( object ): self.__database.save( note, commit = False ) user = self.__users.update_storage( user_id, commit = False ) + self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid self.__database.commit() return dict( storage_bytes = user.storage_bytes ) @@ -660,6 +662,7 @@ class Notebooks( object ): self.__database.save( note, commit = False ) user = self.__users.update_storage( user_id, commit = False ) + self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid self.__database.commit() return dict( storage_bytes = user.storage_bytes ) @@ -710,6 +713,7 @@ class Notebooks( object ): self.__database.save( note, commit = False ) user = self.__users.update_storage( user_id, commit = False ) + self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid self.__database.commit() return dict( diff --git a/controller/Users.py b/controller/Users.py index f555578..a360dbc 100644 --- a/controller/Users.py +++ b/controller/Users.py @@ -435,7 +435,7 @@ class Users( object ): @raise Access_error: user_id or anonymous user unknown """ # if there's no logged-in user, default to the anonymous user - anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) ) + anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True ) if user_id: user = self.__database.load( User, user_id ) else: @@ -513,7 +513,7 @@ class Users( object ): @rtype: bool @return: True if the user has access """ - anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) ) + anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True ) if self.__database.select_one( bool, anonymous.sql_has_access( notebook_id, read_write, owner ) ): return True @@ -600,7 +600,7 @@ class Users( object ): @raise Password_reset_error: an error occured when redeeming the password reset, such as an expired link @raise Validation_error: one of the arguments is invalid """ - anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) ) + anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True ) if anonymous: main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) ) @@ -624,7 +624,7 @@ class Users( object ): result = self.current( anonymous.object_id ) result[ "notebook" ] = main_notebook result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() ) - result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() ) + result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True ) result[ "note_read_write" ] = False result[ "notes" ] = [ Note.create( object_id = u"password_reset", @@ -921,7 +921,7 @@ class Users( object ): if not notebook: raise Invite_error( "That notebook you've been invited to is unknown." ) - anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) ) + anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True ) if anonymous: main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) ) invite_notebook = self.__database.load( Notebook, invite.notebook_id ) @@ -933,7 +933,7 @@ class Users( object ): result = self.current( anonymous.object_id ) result[ "notebook" ] = main_notebook result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() ) - result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() ) + result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True ) result[ "note_read_write" ] = False result[ "notes" ] = [ Note.create( object_id = u"redeem_invite", @@ -1086,7 +1086,7 @@ class Users( object ): """ Provide the information necessary to display the subscription thanks page. """ - anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) ) + anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True ) if anonymous: main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) ) else: @@ -1120,7 +1120,7 @@ class Users( object ): result[ "notebook" ] = main_notebook result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() ) - result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() ) + result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True ) result[ "note_read_write" ] = False result[ "notes" ] = [ Note.create( object_id = u"thanks", diff --git a/controller/test/Stub_cache.py b/controller/test/Stub_cache.py new file mode 100644 index 0000000..9cb1a29 --- /dev/null +++ b/controller/test/Stub_cache.py @@ -0,0 +1,9 @@ +class Stub_cache( object ): + def __init__( self ): + self.__objects = {} + + def get( self, key ): + return self.__objects.get( key ) + + def set( self, key, value ): + self.__objects[ key ] = value diff --git a/controller/test/Stub_database.py b/controller/test/Stub_database.py index aadab6f..f534e0b 100644 --- a/controller/test/Stub_database.py +++ b/controller/test/Stub_database.py @@ -42,7 +42,7 @@ class Stub_database( object ): return None - def select_one( self, Object_type, sql_command ): + def select_one( self, Object_type, sql_command, use_cache = False ): if callable( sql_command ): result = sql_command( self ) if isinstance( result, list ): @@ -67,6 +67,9 @@ class Stub_database( object ): raise NotImplementedError( sql_command ) + def uncache_command( self, sql_command ): + pass + def next_id( self, Object_type, commit = True ): self.__next_id += 1 return unicode( self.__next_id ) diff --git a/controller/test/Test_database.py b/controller/test/Test_database.py index 8cf085e..d793905 100644 --- a/controller/test/Test_database.py +++ b/controller/test/Test_database.py @@ -2,6 +2,7 @@ from pytz import utc from pysqlite2 import dbapi2 as sqlite from datetime import datetime from Stub_object import Stub_object +from Stub_cache import Stub_cache from controller.Database import Database @@ -9,10 +10,11 @@ class Test_database( object ): def setUp( self ): # make an in-memory sqlite database to use in place of PostgreSQL during testing self.connection = sqlite.connect( ":memory:", detect_types = sqlite.PARSE_DECLTYPES | sqlite.PARSE_COLNAMES ) + self.cache = Stub_cache() cursor = self.connection.cursor() cursor.execute( Stub_object.sql_create_table() ) - self.database = Database( self.connection ) + self.database = Database( self.connection, self.cache ) def tearDown( self ): self.database.close() diff --git a/model/Persistent.py b/model/Persistent.py index 78f6503..ec08d4f 100644 --- a/model/Persistent.py +++ b/model/Persistent.py @@ -70,8 +70,13 @@ class Persistent( object ): def update_revision( self ): self.__revision = datetime.now( tz = utc ) + @staticmethod + def make_cache_key( Object_type, object_id ): + return "%s_%s" % ( object_id, Object_type.__name__ ) + object_id = property( lambda self: self.__object_id ) revision = property( lambda self: self.__revision ) + cache_key = property( lambda self: Persistent.make_cache_key( type( self ), self.object_id ) ) def quote( value ):