Now using memcached in various places to improve performance. If the Python

cmemcache module is not importable, then memcached simply won't be used.

INSTALL

@@ -143,6 +143,38 @@ shows up in various error messages and other places for a support contact
 address.
 
 
+memcached
+---------
+
+For improved performance, it is recommended that you install and use memcached
+for production servers.
+
+First, install the prerequisites:
+
+ * python-dev 2.4
+ * libmemcache-dev 1.4
+ * memcached 1.4
+ * cmemcache 0.91
+
+In Debian GNU/Linux, you can issue the following command to install these
+packages:
+
+  apt-get install python2.4-dev libmemcache-dev memcached
+
+The cmemcache package is not currently included with Debian Etch, so you'll
+have to build and install it manually. Download and untar the package from:
+
+  http://gijsbert.org/cmemcache/
+
+From the untarred cmemcache directory, issue the following command as root:
+
+  python2.4 setup.py install
+
+This should build and install the cmemcache module. Once installed, Luminotes
+will use the module automatically. When Luminotes starts up, you should see a
+"using memcached" message.
+
+
 Python unit tests
 -----------------
 

NEWS

@@ -1,4 +1,5 @@
-1.2.2: March ?, 2008
+1.2.2: March 4, 2008
+ * Introduced database object caching to improve performance.
  * Wrote a database reaper script to delete unused notes, notebooks, etc.
  * Added some database indices to improve select performance.
  * Now scrolling the page vertically to show opened errors and messages.

controller/Database.py

@@ -1,20 +1,24 @@
 import re
 import os
+import sha
 import psycopg2 as psycopg
 from psycopg2.pool import PersistentConnectionPool
 import random
+from model.Persistent import Persistent
 
 
 class Database( object ):
   ID_BITS = 128 # number of bits within an id
   ID_DIGITS = "0123456789abcdefghijklmnopqrstuvwxyz"
 
-  def __init__( self, connection = None ):
+  def __init__( self, connection = None, cache = None ):
     """
     Create a new database and return it.
 
     @type connection: existing connection object with cursor()/close()/commit() methods, or NoneType
     @param connection: database connection to use (optional, defaults to making a connection pool)
+    @type cache: cmemcache.Client or something with a similar API, or NoneType
+    @param cache: existing memory cache to use (optional, defaults to making a cache)
     @rtype: Database
     @return: newly constructed Database
     """
@@ -33,6 +37,15 @@ class Database( object ):
         "dbname=luminotes user=luminotes password=%s" % os.getenv( "PGPASSWORD", "dev" ),
       )
 
+    self.__cache = cache
+    if not cache:
+      try:
+        import cmemcache
+        self.__cache = cmemcache.Client( [ "127.0.0.1:11211" ], debug = 0 )
+        print "using memcached"
+      except ImportError:
+        pass
+
   def __get_connection( self ):
     if self.__connection:
       return self.__connection
@@ -59,6 +72,8 @@ class Database( object ):
 
     if commit:
       connection.commit()
+      if self.__cache:
+        self.__cache.set( obj.cache_key, obj )
 
   def commit( self ):
     self.__get_connection().commit()
@@ -78,9 +93,18 @@ class Database( object ):
     @rtype: Object_type or NoneType
     @return: loaded object, or None if no match
     """
-    return self.select_one( Object_type, Object_type.sql_load( object_id, revision ) )
+    if revision is None and self.__cache: # don't bother caching old revisions
+      obj = self.__cache.get( Persistent.make_cache_key( Object_type, object_id ) )
+      if obj:
+        return obj
 
-  def select_one( self, Object_type, sql_command ):
+    obj = self.select_one( Object_type, Object_type.sql_load( object_id, revision ) )
+    if obj and revision is None and self.__cache:
+      self.__cache.set( obj.cache_key, obj )
+
+    return obj
+
+  def select_one( self, Object_type, sql_command, use_cache = False ):
     """
     Execute the given sql_command and return its results in the form of an object of Object_type,
     or None if there was no match.
@@ -89,9 +113,17 @@ class Database( object ):
     @param Object_type: class of the object to load 
     @type sql_command: unicode
     @param sql_command: SQL command to execute
+    @type use_cache: bool
+    @param use_cache: whether to look for and store objects in the cache
     @rtype: Object_type or NoneType
     @return: loaded object, or None if no match
     """
+    if use_cache and self.__cache:
+      cache_key = sha.new( sql_command ).hexdigest()
+      obj = self.__cache.get( cache_key )
+      if obj:
+        return obj
+
     connection = self.__get_connection()
     cursor = connection.cursor()
 
@@ -102,9 +134,14 @@ class Database( object ):
       return None
 
     if Object_type in ( tuple, list ):
-      return Object_type( row )
+      obj = Object_type( row )
     else:
-      return Object_type( *row )
+      obj = Object_type( *row )
+
+    if obj and use_cache and self.__cache:
+      self.__cache.set( cache_key, obj )
+
+    return obj
 
   def select_many( self, Object_type, sql_command ):
     """
@@ -160,6 +197,12 @@ class Database( object ):
     if commit:
       connection.commit()
 
+  def uncache_command( self, sql_command ):
+    if not self.__cache: return
+
+    cache_key = sha.new( sql_command ).hexdigest()
+    self.__cache.delete( cache_key )
+
   @staticmethod
   def generate_id():
     int_id = random.getrandbits( Database.ID_BITS )

controller/Notebooks.py

@@ -190,7 +190,7 @@ class Notebooks( object ):
       note = None
 
     startup_notes = self.__database.select_many( Note, notebook.sql_load_startup_notes() )
-    total_notes_count = self.__database.select_one( int, notebook.sql_count_notes() )
+    total_notes_count = self.__database.select_one( int, notebook.sql_count_notes(), use_cache = True )
 
     if self.__users.check_access( user_id, notebook_id, owner = True ):
       invites = self.__database.select_many( Invite, Invite.sql_load_notebook_invites( notebook_id ) )
@@ -549,6 +549,7 @@ class Notebooks( object ):
     if new_revision:
       self.__database.save( note, commit = False )
       user = self.__users.update_storage( user_id, commit = False )
+      self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
       self.__database.commit()
     else:
       user = None
@@ -605,6 +606,7 @@ class Notebooks( object ):
 
       self.__database.save( note, commit = False )
       user = self.__users.update_storage( user_id, commit = False )
+      self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
       self.__database.commit()
 
       return dict( storage_bytes = user.storage_bytes )
@@ -660,6 +662,7 @@ class Notebooks( object ):
 
       self.__database.save( note, commit = False )
       user = self.__users.update_storage( user_id, commit = False )
+      self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
       self.__database.commit()
 
       return dict( storage_bytes = user.storage_bytes )
@@ -710,6 +713,7 @@ class Notebooks( object ):
       self.__database.save( note, commit = False )
 
     user = self.__users.update_storage( user_id, commit = False )
+    self.__database.uncache_command( notebook.sql_count_notes() ) # cached note count is now invalid
     self.__database.commit()
 
     return dict(

controller/Users.py

@@ -435,7 +435,7 @@ class Users( object ):
     @raise Access_error: user_id or anonymous user unknown
     """
     # if there's no logged-in user, default to the anonymous user
-    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
+    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
     if user_id:
       user = self.__database.load( User, user_id )
     else:
@@ -513,7 +513,7 @@ class Users( object ):
     @rtype: bool
     @return: True if the user has access
     """
-    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
+    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
 
     if self.__database.select_one( bool, anonymous.sql_has_access( notebook_id, read_write, owner ) ):
       return True
@@ -600,7 +600,7 @@ class Users( object ):
     @raise Password_reset_error: an error occured when redeeming the password reset, such as an expired link
     @raise Validation_error: one of the arguments is invalid
     """
-    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
+    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
     if anonymous:
       main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) )
 
@@ -624,7 +624,7 @@ class Users( object ):
     result = self.current( anonymous.object_id )
     result[ "notebook" ] = main_notebook
     result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() )
-    result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() )
+    result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True )
     result[ "note_read_write" ] = False
     result[ "notes" ] = [ Note.create(
       object_id = u"password_reset",
@@ -921,7 +921,7 @@ class Users( object ):
     if not notebook:
       raise Invite_error( "That notebook you've been invited to is unknown." )
 
-    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
+    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
     if anonymous:
       main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) )
       invite_notebook = self.__database.load( Notebook, invite.notebook_id )
@@ -933,7 +933,7 @@ class Users( object ):
     result = self.current( anonymous.object_id )
     result[ "notebook" ] = main_notebook
     result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() )
-    result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() )
+    result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True )
     result[ "note_read_write" ] = False
     result[ "notes" ] = [ Note.create(
       object_id = u"redeem_invite",
@@ -1086,7 +1086,7 @@ class Users( object ):
     """
     Provide the information necessary to display the subscription thanks page.
     """
-    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ) )
+    anonymous = self.__database.select_one( User, User.sql_load_by_username( u"anonymous" ), use_cache = True )
     if anonymous:
       main_notebook = self.__database.select_one( Notebook, anonymous.sql_load_notebooks( undeleted_only = True ) )
     else:
@@ -1120,7 +1120,7 @@ class Users( object ):
 
     result[ "notebook" ] = main_notebook
     result[ "startup_notes" ] = self.__database.select_many( Note, main_notebook.sql_load_startup_notes() )
-    result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes() )
+    result[ "total_notes_count" ] = self.__database.select_one( Note, main_notebook.sql_count_notes(), use_cache = True )
     result[ "note_read_write" ] = False
     result[ "notes" ] = [ Note.create(
       object_id = u"thanks",

controller/test/Stub_cache.py

@@ -0,0 +1,9 @@
+class Stub_cache( object ):
+  def __init__( self ):
+    self.__objects = {}
+
+  def get( self, key ):
+    return self.__objects.get( key )
+
+  def set( self, key, value ):
+    self.__objects[ key ] = value

controller/test/Stub_database.py

@@ -42,7 +42,7 @@ class Stub_database( object ):
 
     return None
 
-  def select_one( self, Object_type, sql_command ):
+  def select_one( self, Object_type, sql_command, use_cache = False ):
     if callable( sql_command ):
       result = sql_command( self )
       if isinstance( result, list ):
@@ -67,6 +67,9 @@ class Stub_database( object ):
 
     raise NotImplementedError( sql_command )
 
+  def uncache_command( self, sql_command ):
+    pass
+
   def next_id( self, Object_type, commit = True ):
     self.__next_id += 1
     return unicode( self.__next_id )

controller/test/Test_database.py

@@ -2,6 +2,7 @@ from pytz import utc
 from pysqlite2 import dbapi2 as sqlite
 from datetime import datetime
 from Stub_object import Stub_object
+from Stub_cache import Stub_cache
 from controller.Database import Database
 
 
@@ -9,10 +10,11 @@ class Test_database( object ):
   def setUp( self ):
     # make an in-memory sqlite database to use in place of PostgreSQL during testing
     self.connection = sqlite.connect( ":memory:", detect_types = sqlite.PARSE_DECLTYPES | sqlite.PARSE_COLNAMES )
+    self.cache = Stub_cache()
     cursor = self.connection.cursor()
     cursor.execute( Stub_object.sql_create_table() )
 
-    self.database = Database( self.connection )
+    self.database = Database( self.connection, self.cache )
 
   def tearDown( self ):
     self.database.close()

model/Persistent.py

@@ -70,8 +70,13 @@ class Persistent( object ):
   def update_revision( self ):
     self.__revision = datetime.now( tz = utc )
 
+  @staticmethod
+  def make_cache_key( Object_type, object_id ):
+    return "%s_%s" % ( object_id, Object_type.__name__ )
+
   object_id = property( lambda self: self.__object_id )
   revision = property( lambda self: self.__revision )
+  cache_key = property( lambda self: Persistent.make_cache_key( type( self ), self.object_id ) )
 
 
 def quote( value ):