Fixed some bugs in convertdb.py.
Wrote a tool to verify that convertdb.py does what it's supposed to. Added a todo comment to Scheduler about threading and generators. Updated INSTALL documentation about eventual Postgres requirement.
This commit is contained in:
Dan Helfman
parent
d3e953f8da
commit
43e8c7fb17
61
INSTALL
61
INSTALL
|
|
@ -6,12 +6,17 @@ First, install the prerequisites:
|
|||
|
||||
* Python 2.5
|
||||
* CherryPy 2.2
|
||||
* PostgreSQL 8.1
|
||||
* psycopg 2.0
|
||||
* simplejson 1.3
|
||||
|
||||
In Debian GNU/Linux, you can issue the following command to install these
|
||||
packages:
|
||||
|
||||
apt-get install python2.5 python-cherrypy python-simplejson
|
||||
apt-get install python2.5 python-cherrypy postgresql-8.1 python-psycopg2 python-simplejson
|
||||
|
||||
If you're using Debian Etch, see the note below about "psycopg in Debian
|
||||
Etch".
|
||||
|
||||
|
||||
development mode
|
||||
|
|
@ -22,6 +27,24 @@ changes, because it uses CherryPy's built-in web server with auto-reload
|
|||
enabled, so the server will automatically reload any modified source files as
|
||||
soon as they're modified.
|
||||
|
||||
Configure PostgreSQL's pg_hba.conf to require passwords for local connections:
|
||||
|
||||
local all all md5
|
||||
|
||||
Restart postgresql so these changes take effect:
|
||||
|
||||
/etc/init.d/postgresql restart
|
||||
|
||||
As the PostgreSQL superuser (usually "postgres"), create a new database user
|
||||
and set the password to "dev".
|
||||
|
||||
createuser -S -d -R -P -E luminotes
|
||||
|
||||
Initialize the database with the starting schema and basic data:
|
||||
|
||||
psql -U luminotes postgres -f model/schema.sql
|
||||
psql -U luminotes postgres -f model/data.sql
|
||||
|
||||
To start the server in development mode, run:
|
||||
|
||||
python2.5 luminotes.py -d
|
||||
|
|
@ -82,6 +105,24 @@ domain you're using. For instance:
|
|||
"luminotes.http_url": "http://luminotes.com",
|
||||
"luminotes.https_url": "https://luminotes.com",
|
||||
|
||||
Configure PostgreSQL's pg_hba.conf to require passwords for local connections:
|
||||
|
||||
local all all md5
|
||||
|
||||
Restart postgresql so these changes take effect:
|
||||
|
||||
/etc/init.d/postgresql restart
|
||||
|
||||
As the PostgreSQL superuser (usually "postgres"), create a new database user
|
||||
and set the password to "dev".
|
||||
|
||||
createuser -S -d -R -P -E luminotes
|
||||
|
||||
Initialize the database with the starting schema and basic data:
|
||||
|
||||
psql -U luminotes postgres -f model/schema.sql
|
||||
psql -U luminotes postgres -f model/data.sql
|
||||
|
||||
Then to actually start the production mode server, run:
|
||||
|
||||
python2.5 luminotes.py
|
||||
|
|
@ -119,3 +160,21 @@ The run_tests.py script runs the tests inside browser windows and presumes you
|
|||
have both Firefox and Internet Explorer 6 installed. Edit run_tests.py if you
|
||||
need to specify different paths to the browser binaries or want to test with
|
||||
additional browsers.
|
||||
|
||||
|
||||
psycopg in Debian Etch
|
||||
----------------------
|
||||
|
||||
As of this writing, Debian Etch does not contain a version of psycopg with
|
||||
support for Python 2.5. However, the version of psycopg in Debian testing does
|
||||
support Python 2.5. So you can grab the source for python-psycopg2 from Debian
|
||||
testing, install the build dependencies (including python2.5-dev), and build
|
||||
the package yourself on an Etch machine.
|
||||
|
||||
Then, edit /usr/share/python/debian_defaults and move "python2.5" from
|
||||
"unsupported-versions" to "supported-versions". Finally, install the
|
||||
python-psycopg2 package you've just built, and it should fully support Python
|
||||
2.5.
|
||||
|
||||
See Debian bug #404355 for more information. Note that it was fixed in
|
||||
unstable, but not in Etch.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@ class Scheduler( object ):
|
|||
self.add( self.__idle_thread() )
|
||||
self.__idle.acquire() # don't count the idle thread
|
||||
|
||||
# TODO: Running the scheduler from anything other than the main Python thread somehow prevents
|
||||
# tracebacks from within a generator from indicating the offending line and line number. So it
|
||||
# would be really useful for debugging purposes to start the scheduler from the main thread.
|
||||
# The reason that it's not done here is because CherryPy's blocking server must be started
|
||||
# from the main Python thread.
|
||||
self.__scheduler_thread = Thread( target = self.run )
|
||||
self.__scheduler_thread.setDaemon( True )
|
||||
self.__scheduler_thread.start()
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class Converter( object ):
|
|||
"insert into notebook " +
|
||||
"( id, revision, name, trash_id ) " +
|
||||
"values ( %s, %s, %s, %s );" %
|
||||
( quote( value.object_id ), quote( value.revision ), quote( value.name ), quote( value.trash and value.trash.object_id or "null" ) )
|
||||
( quote( value.object_id ), quote( value.revision ), quote( value.name ), value.trash and quote( value.trash.object_id ) or "null" )
|
||||
)
|
||||
|
||||
for note in value.notes:
|
||||
|
|
@ -69,7 +69,7 @@ class Converter( object ):
|
|||
"insert into note " +
|
||||
"( id, revision, title, contents, notebook_id, startup, deleted_from_id, rank ) " +
|
||||
"values ( %s, %s, %s, %s, %s, %s, %s, %s );" %
|
||||
( quote( value.object_id ), quote( value.revision ), quote( value.title ), quote( value.contents ), quote( None ), quote( "f" ), quote( value.deleted_from or None ), quote( None ) )
|
||||
( quote( value.object_id ), quote( value.revision ), quote( value.title or None ), quote( value.contents or None ), quote( None ), quote( "f" ), quote( value.deleted_from or None ), quote( None ) )
|
||||
)
|
||||
elif class_name == "User":
|
||||
if value.username is None: continue # note: this will skip all demo users
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
#!/usr/bin/python2.5
|
||||
|
||||
import os
|
||||
import os.path
|
||||
import psycopg2 as psycopg
|
||||
from controller.Database import Database
|
||||
from controller.Scheduler import Scheduler
|
||||
|
||||
|
||||
def quote( value ):
|
||||
if value is None:
|
||||
return "null"
|
||||
|
||||
value = unicode( value )
|
||||
return "'%s'" % value.replace( "'", "''" ).replace( "\\", "\\\\" )
|
||||
|
||||
|
||||
class Verifier( object ):
|
||||
"""
|
||||
Verifies a conversion of a Luminotes database from bsddb to PostgreSQL that was performed with
|
||||
convertdb.py.
|
||||
"""
|
||||
def __init__( self, scheduler, database ):
|
||||
self.scheduler = scheduler
|
||||
self.database = database
|
||||
|
||||
self.conn = psycopg.connect( "dbname=luminotes user=luminotes password=dev" )
|
||||
self.cursor = self.conn.cursor()
|
||||
|
||||
thread = self.verify_database()
|
||||
self.scheduler.add( thread )
|
||||
self.scheduler.wait_for( thread )
|
||||
|
||||
def verify_database( self ):
|
||||
inserts = set()
|
||||
|
||||
for key in self.database._Database__db.keys():
|
||||
if not self.database._Database__db.get( key ):
|
||||
continue
|
||||
|
||||
self.database.load( key, self.scheduler.thread )
|
||||
value = ( yield Scheduler.SLEEP )
|
||||
|
||||
class_name = value.__class__.__name__
|
||||
|
||||
if class_name == "Notebook":
|
||||
self.verify_notebook( value )
|
||||
elif class_name == "Note":
|
||||
self.cursor.execute(
|
||||
"select * from note where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) )
|
||||
)
|
||||
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == value.object_id
|
||||
assert row[ 1 ].replace( tzinfo = None ) == value.revision
|
||||
assert row[ 2 ] == ( value.title and value.title.encode( "utf8" ) or None )
|
||||
assert row[ 3 ] == ( value.contents and value.contents.encode( "utf8" ) or None )
|
||||
# not checking for existence of row 4 (notebook_id), because notes deleted from the trash don't have a notebook id
|
||||
assert row[ 5 ] is not None
|
||||
assert row[ 6 ] == ( value.deleted_from or None )
|
||||
if row[ 5 ] is True: # if this is a startup note, it should have a rank
|
||||
assert row[ 7 ] is not None
|
||||
elif class_name == "User":
|
||||
# skip demo users
|
||||
if value.username is None: continue
|
||||
|
||||
self.cursor.execute(
|
||||
"select * from luminotes_user where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) )
|
||||
)
|
||||
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == value.object_id
|
||||
assert row[ 1 ].replace( tzinfo = None ) == value.revision
|
||||
assert row[ 2 ] == value.username
|
||||
assert row[ 3 ] == value._User__salt
|
||||
assert row[ 4 ] == value._User__password_hash
|
||||
assert row[ 5 ] == value.email_address
|
||||
assert row[ 6 ] == value.storage_bytes
|
||||
assert row[ 7 ] == value.rate_plan
|
||||
|
||||
for notebook in value.notebooks:
|
||||
if notebook is None: continue
|
||||
|
||||
read_write = ( notebook.__class__.__name__ == "Notebook" )
|
||||
|
||||
self.cursor.execute(
|
||||
"select * from user_notebook where user_id = %s and notebook_id = %s;" % ( quote( value.object_id ), quote( notebook.object_id ) )
|
||||
)
|
||||
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == value.object_id
|
||||
assert row[ 1 ] == notebook.object_id
|
||||
assert row[ 2 ] == read_write
|
||||
|
||||
self.verify_notebook( notebook )
|
||||
|
||||
elif class_name == "Read_only_notebook":
|
||||
self.verify_notebook( value._Read_only_notebook__wrapped )
|
||||
elif class_name == "Password_reset":
|
||||
# skip password resets that are already redeemed
|
||||
if value.redeemed: continue
|
||||
|
||||
self.cursor.execute(
|
||||
"select * from password_reset where id = %s;" % quote( value.object_id )
|
||||
)
|
||||
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == value.email_address
|
||||
assert row[ 1 ] == False
|
||||
assert row[ 2 ] == value.object_id
|
||||
elif class_name == "User_list":
|
||||
pass
|
||||
else:
|
||||
raise Exception( "Unverified value of type %s" % class_name )
|
||||
|
||||
self.conn.commit()
|
||||
yield None
|
||||
|
||||
def verify_notebook( self, value ):
|
||||
self.cursor.execute(
|
||||
"select * from notebook where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) )
|
||||
)
|
||||
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == value.object_id
|
||||
assert row[ 1 ].replace( tzinfo = None ) == value.revision
|
||||
assert row[ 2 ] == value.name
|
||||
if value.trash:
|
||||
assert row[ 3 ] == value.trash.object_id
|
||||
else:
|
||||
assert row[ 3 ] == None
|
||||
|
||||
startup_note_ids = [ note.object_id for note in value.startup_notes ]
|
||||
for note in value.notes:
|
||||
self.cursor.execute(
|
||||
"select * from note where id = %s and revision = %s;" % ( quote( note.object_id ), quote( value.revision ) )
|
||||
)
|
||||
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == note.object_id
|
||||
assert row[ 1 ].replace( tzinfo = None ) == note.revision
|
||||
assert row[ 2 ] == note.title
|
||||
assert row[ 3 ] == note.contents
|
||||
assert row[ 4 ] == value.object_id
|
||||
assert row[ 5 ] == ( note.object_id in startup_note_ids )
|
||||
assert row[ 6 ] == note.deleted_from
|
||||
if row[ 5 ] is True: # if this is a startup note, it should have a rank
|
||||
assert row[ 7 ] is not None
|
||||
|
||||
for note in value.startup_notes:
|
||||
self.cursor.execute(
|
||||
"select * from note where id = %s and revision = %s order by rank;" % ( quote( note.object_id ), quote( value.revision ) )
|
||||
)
|
||||
|
||||
rank = 0
|
||||
for row in self.cursor.fetchmany():
|
||||
assert row[ 0 ] == note.object_id
|
||||
assert row[ 1 ].replace( tzinfo = None ) == note.revision
|
||||
assert row[ 2 ] == note.title
|
||||
assert row[ 3 ] == note.contents
|
||||
assert row[ 4 ] == value.object_id
|
||||
assert row[ 5 ] == True
|
||||
assert row[ 6 ] == note.deleted_from
|
||||
assert row[ 7 ] == rank
|
||||
rank += 1
|
||||
|
||||
|
||||
def main():
|
||||
scheduler = Scheduler()
|
||||
database = Database( scheduler, "data.db" )
|
||||
initializer = Verifier( scheduler, database )
|
||||
scheduler.wait_until_idle()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue