From 43e8c7fb17d9f7f6452014dbf2fc472940fb92ca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 2 Oct 2007 22:43:18 +0000 Subject: [PATCH] Fixed some bugs in convertdb.py. Wrote a tool to verify that convertdb.py does what it's supposed to. Added a todo comment to Scheduler about threading and generators. Updated INSTALL documentation about eventual Postgres requirement. --- INSTALL | 61 +++++++++++++- controller/Scheduler.py | 5 ++ tools/convertdb.py | 4 +- tools/verifyconvertdb.py | 176 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 3 deletions(-) create mode 100755 tools/verifyconvertdb.py diff --git a/INSTALL b/INSTALL index 72b1bae..599a9d1 100644 --- a/INSTALL +++ b/INSTALL @@ -6,12 +6,17 @@ First, install the prerequisites: * Python 2.5 * CherryPy 2.2 + * PostgreSQL 8.1 + * psycopg 2.0 * simplejson 1.3 In Debian GNU/Linux, you can issue the following command to install these packages: - apt-get install python2.5 python-cherrypy python-simplejson + apt-get install python2.5 python-cherrypy postgresql-8.1 python-psycopg2 python-simplejson + +If you're using Debian Etch, see the note below about "psycopg in Debian +Etch". development mode @@ -22,6 +27,24 @@ changes, because it uses CherryPy's built-in web server with auto-reload enabled, so the server will automatically reload any modified source files as soon as they're modified. +Configure PostgreSQL's pg_hba.conf to require passwords for local connections: + + local all all md5 + +Restart postgresql so these changes take effect: + + /etc/init.d/postgresql restart + +As the PostgreSQL superuser (usually "postgres"), create a new database user +and set the password to "dev". + + createuser -S -d -R -P -E luminotes + +Initialize the database with the starting schema and basic data: + + psql -U luminotes postgres -f model/schema.sql + psql -U luminotes postgres -f model/data.sql + To start the server in development mode, run: python2.5 luminotes.py -d @@ -82,6 +105,24 @@ domain you're using. For instance: "luminotes.http_url": "http://luminotes.com", "luminotes.https_url": "https://luminotes.com", +Configure PostgreSQL's pg_hba.conf to require passwords for local connections: + + local all all md5 + +Restart postgresql so these changes take effect: + + /etc/init.d/postgresql restart + +As the PostgreSQL superuser (usually "postgres"), create a new database user +and set the password to "dev". + + createuser -S -d -R -P -E luminotes + +Initialize the database with the starting schema and basic data: + + psql -U luminotes postgres -f model/schema.sql + psql -U luminotes postgres -f model/data.sql + Then to actually start the production mode server, run: python2.5 luminotes.py @@ -119,3 +160,21 @@ The run_tests.py script runs the tests inside browser windows and presumes you have both Firefox and Internet Explorer 6 installed. Edit run_tests.py if you need to specify different paths to the browser binaries or want to test with additional browsers. + + +psycopg in Debian Etch +---------------------- + +As of this writing, Debian Etch does not contain a version of psycopg with +support for Python 2.5. However, the version of psycopg in Debian testing does +support Python 2.5. So you can grab the source for python-psycopg2 from Debian +testing, install the build dependencies (including python2.5-dev), and build +the package yourself on an Etch machine. + +Then, edit /usr/share/python/debian_defaults and move "python2.5" from +"unsupported-versions" to "supported-versions". Finally, install the +python-psycopg2 package you've just built, and it should fully support Python +2.5. + +See Debian bug #404355 for more information. Note that it was fixed in +unstable, but not in Etch. diff --git a/controller/Scheduler.py b/controller/Scheduler.py index 16df6a8..385565e 100644 --- a/controller/Scheduler.py +++ b/controller/Scheduler.py @@ -20,6 +20,11 @@ class Scheduler( object ): self.add( self.__idle_thread() ) self.__idle.acquire() # don't count the idle thread + # TODO: Running the scheduler from anything other than the main Python thread somehow prevents + # tracebacks from within a generator from indicating the offending line and line number. So it + # would be really useful for debugging purposes to start the scheduler from the main thread. + # The reason that it's not done here is because CherryPy's blocking server must be started + # from the main Python thread. self.__scheduler_thread = Thread( target = self.run ) self.__scheduler_thread.setDaemon( True ) self.__scheduler_thread.start() diff --git a/tools/convertdb.py b/tools/convertdb.py index 806cbd6..1709c50 100755 --- a/tools/convertdb.py +++ b/tools/convertdb.py @@ -53,7 +53,7 @@ class Converter( object ): "insert into notebook " + "( id, revision, name, trash_id ) " + "values ( %s, %s, %s, %s );" % - ( quote( value.object_id ), quote( value.revision ), quote( value.name ), quote( value.trash and value.trash.object_id or "null" ) ) + ( quote( value.object_id ), quote( value.revision ), quote( value.name ), value.trash and quote( value.trash.object_id ) or "null" ) ) for note in value.notes: @@ -69,7 +69,7 @@ class Converter( object ): "insert into note " + "( id, revision, title, contents, notebook_id, startup, deleted_from_id, rank ) " + "values ( %s, %s, %s, %s, %s, %s, %s, %s );" % - ( quote( value.object_id ), quote( value.revision ), quote( value.title ), quote( value.contents ), quote( None ), quote( "f" ), quote( value.deleted_from or None ), quote( None ) ) + ( quote( value.object_id ), quote( value.revision ), quote( value.title or None ), quote( value.contents or None ), quote( None ), quote( "f" ), quote( value.deleted_from or None ), quote( None ) ) ) elif class_name == "User": if value.username is None: continue # note: this will skip all demo users diff --git a/tools/verifyconvertdb.py b/tools/verifyconvertdb.py new file mode 100755 index 0000000..858c6e5 --- /dev/null +++ b/tools/verifyconvertdb.py @@ -0,0 +1,176 @@ +#!/usr/bin/python2.5 + +import os +import os.path +import psycopg2 as psycopg +from controller.Database import Database +from controller.Scheduler import Scheduler + + +def quote( value ): + if value is None: + return "null" + + value = unicode( value ) + return "'%s'" % value.replace( "'", "''" ).replace( "\\", "\\\\" ) + + +class Verifier( object ): + """ + Verifies a conversion of a Luminotes database from bsddb to PostgreSQL that was performed with + convertdb.py. + """ + def __init__( self, scheduler, database ): + self.scheduler = scheduler + self.database = database + + self.conn = psycopg.connect( "dbname=luminotes user=luminotes password=dev" ) + self.cursor = self.conn.cursor() + + thread = self.verify_database() + self.scheduler.add( thread ) + self.scheduler.wait_for( thread ) + + def verify_database( self ): + inserts = set() + + for key in self.database._Database__db.keys(): + if not self.database._Database__db.get( key ): + continue + + self.database.load( key, self.scheduler.thread ) + value = ( yield Scheduler.SLEEP ) + + class_name = value.__class__.__name__ + + if class_name == "Notebook": + self.verify_notebook( value ) + elif class_name == "Note": + self.cursor.execute( + "select * from note where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) ) + ) + + for row in self.cursor.fetchmany(): + assert row[ 0 ] == value.object_id + assert row[ 1 ].replace( tzinfo = None ) == value.revision + assert row[ 2 ] == ( value.title and value.title.encode( "utf8" ) or None ) + assert row[ 3 ] == ( value.contents and value.contents.encode( "utf8" ) or None ) + # not checking for existence of row 4 (notebook_id), because notes deleted from the trash don't have a notebook id + assert row[ 5 ] is not None + assert row[ 6 ] == ( value.deleted_from or None ) + if row[ 5 ] is True: # if this is a startup note, it should have a rank + assert row[ 7 ] is not None + elif class_name == "User": + # skip demo users + if value.username is None: continue + + self.cursor.execute( + "select * from luminotes_user where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) ) + ) + + for row in self.cursor.fetchmany(): + assert row[ 0 ] == value.object_id + assert row[ 1 ].replace( tzinfo = None ) == value.revision + assert row[ 2 ] == value.username + assert row[ 3 ] == value._User__salt + assert row[ 4 ] == value._User__password_hash + assert row[ 5 ] == value.email_address + assert row[ 6 ] == value.storage_bytes + assert row[ 7 ] == value.rate_plan + + for notebook in value.notebooks: + if notebook is None: continue + + read_write = ( notebook.__class__.__name__ == "Notebook" ) + + self.cursor.execute( + "select * from user_notebook where user_id = %s and notebook_id = %s;" % ( quote( value.object_id ), quote( notebook.object_id ) ) + ) + + for row in self.cursor.fetchmany(): + assert row[ 0 ] == value.object_id + assert row[ 1 ] == notebook.object_id + assert row[ 2 ] == read_write + + self.verify_notebook( notebook ) + + elif class_name == "Read_only_notebook": + self.verify_notebook( value._Read_only_notebook__wrapped ) + elif class_name == "Password_reset": + # skip password resets that are already redeemed + if value.redeemed: continue + + self.cursor.execute( + "select * from password_reset where id = %s;" % quote( value.object_id ) + ) + + for row in self.cursor.fetchmany(): + assert row[ 0 ] == value.email_address + assert row[ 1 ] == False + assert row[ 2 ] == value.object_id + elif class_name == "User_list": + pass + else: + raise Exception( "Unverified value of type %s" % class_name ) + + self.conn.commit() + yield None + + def verify_notebook( self, value ): + self.cursor.execute( + "select * from notebook where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) ) + ) + + for row in self.cursor.fetchmany(): + assert row[ 0 ] == value.object_id + assert row[ 1 ].replace( tzinfo = None ) == value.revision + assert row[ 2 ] == value.name + if value.trash: + assert row[ 3 ] == value.trash.object_id + else: + assert row[ 3 ] == None + + startup_note_ids = [ note.object_id for note in value.startup_notes ] + for note in value.notes: + self.cursor.execute( + "select * from note where id = %s and revision = %s;" % ( quote( note.object_id ), quote( value.revision ) ) + ) + + for row in self.cursor.fetchmany(): + assert row[ 0 ] == note.object_id + assert row[ 1 ].replace( tzinfo = None ) == note.revision + assert row[ 2 ] == note.title + assert row[ 3 ] == note.contents + assert row[ 4 ] == value.object_id + assert row[ 5 ] == ( note.object_id in startup_note_ids ) + assert row[ 6 ] == note.deleted_from + if row[ 5 ] is True: # if this is a startup note, it should have a rank + assert row[ 7 ] is not None + + for note in value.startup_notes: + self.cursor.execute( + "select * from note where id = %s and revision = %s order by rank;" % ( quote( note.object_id ), quote( value.revision ) ) + ) + + rank = 0 + for row in self.cursor.fetchmany(): + assert row[ 0 ] == note.object_id + assert row[ 1 ].replace( tzinfo = None ) == note.revision + assert row[ 2 ] == note.title + assert row[ 3 ] == note.contents + assert row[ 4 ] == value.object_id + assert row[ 5 ] == True + assert row[ 6 ] == note.deleted_from + assert row[ 7 ] == rank + rank += 1 + + +def main(): + scheduler = Scheduler() + database = Database( scheduler, "data.db" ) + initializer = Verifier( scheduler, database ) + scheduler.wait_until_idle() + + +if __name__ == "__main__": + main()