Fixed some bugs in convertdb.py.

Wrote a tool to verify that convertdb.py does what it's supposed to.
Added a todo comment to Scheduler about threading and generators.
Updated INSTALL documentation about eventual Postgres requirement.

INSTALL

@@ -6,12 +6,17 @@ First, install the prerequisites:
 
  * Python 2.5
  * CherryPy 2.2
+ * PostgreSQL 8.1
+ * psycopg 2.0
  * simplejson 1.3
 
 In Debian GNU/Linux, you can issue the following command to install these
 packages:
 
-  apt-get install python2.5 python-cherrypy python-simplejson
+  apt-get install python2.5 python-cherrypy postgresql-8.1 python-psycopg2 python-simplejson
+
+If you're using Debian Etch, see the note below about "psycopg in Debian
+Etch".
 
 
 development mode
@@ -22,6 +27,24 @@ changes, because it uses CherryPy's built-in web server with auto-reload
 enabled, so the server will automatically reload any modified source files as
 soon as they're modified.
 
+Configure PostgreSQL's pg_hba.conf to require passwords for local connections:
+
+  local all all md5
+
+Restart postgresql so these changes take effect:
+
+  /etc/init.d/postgresql restart
+
+As the PostgreSQL superuser (usually "postgres"), create a new database user
+and set the password to "dev".
+
+  createuser -S -d -R -P -E luminotes
+
+Initialize the database with the starting schema and basic data:
+
+  psql -U luminotes postgres -f model/schema.sql
+  psql -U luminotes postgres -f model/data.sql
+
 To start the server in development mode, run:
 
   python2.5 luminotes.py -d
@@ -82,6 +105,24 @@ domain you're using. For instance:
   "luminotes.http_url": "http://luminotes.com",
   "luminotes.https_url": "https://luminotes.com",
 
+Configure PostgreSQL's pg_hba.conf to require passwords for local connections:
+
+  local all all md5
+
+Restart postgresql so these changes take effect:
+
+  /etc/init.d/postgresql restart
+
+As the PostgreSQL superuser (usually "postgres"), create a new database user
+and set the password to "dev".
+
+  createuser -S -d -R -P -E luminotes
+
+Initialize the database with the starting schema and basic data:
+
+  psql -U luminotes postgres -f model/schema.sql
+  psql -U luminotes postgres -f model/data.sql
+
 Then to actually start the production mode server, run:
 
   python2.5 luminotes.py
@@ -119,3 +160,21 @@ The run_tests.py script runs the tests inside browser windows and presumes you
 have both Firefox and Internet Explorer 6 installed. Edit run_tests.py if you
 need to specify different paths to the browser binaries or want to test with
 additional browsers.
+
+
+psycopg in Debian Etch
+----------------------
+
+As of this writing, Debian Etch does not contain a version of psycopg with
+support for Python 2.5. However, the version of psycopg in Debian testing does
+support Python 2.5. So you can grab the source for python-psycopg2 from Debian
+testing, install the build dependencies (including python2.5-dev), and build
+the package yourself on an Etch machine.
+
+Then, edit /usr/share/python/debian_defaults and move "python2.5" from
+"unsupported-versions" to "supported-versions". Finally, install the
+python-psycopg2 package you've just built, and it should fully support Python
+2.5.
+
+See Debian bug #404355 for more information. Note that it was fixed in
+unstable, but not in Etch.

controller/Scheduler.py

@@ -20,6 +20,11 @@ class Scheduler( object ):
     self.add( self.__idle_thread() )
     self.__idle.acquire() # don't count the idle thread
 
+    # TODO: Running the scheduler from anything other than the main Python thread somehow prevents
+    # tracebacks from within a generator from indicating the offending line and line number. So it
+    # would be really useful for debugging purposes to start the scheduler from the main thread.
+    # The reason that it's not done here is because CherryPy's blocking server must be started
+    # from the main Python thread.
     self.__scheduler_thread = Thread( target = self.run )
     self.__scheduler_thread.setDaemon( True )
     self.__scheduler_thread.start()

tools/convertdb.py

@@ -53,7 +53,7 @@ class Converter( object ):
           "insert into notebook " +
           "( id, revision, name, trash_id ) " +
           "values ( %s, %s, %s, %s );" %
-          ( quote( value.object_id ), quote( value.revision ), quote( value.name ), quote( value.trash and value.trash.object_id or "null" ) )
+          ( quote( value.object_id ), quote( value.revision ), quote( value.name ), value.trash and quote( value.trash.object_id ) or "null" )
         )
 
         for note in value.notes:
@@ -69,7 +69,7 @@ class Converter( object ):
           "insert into note " +
           "( id, revision, title, contents, notebook_id, startup, deleted_from_id, rank ) " +
           "values ( %s, %s, %s, %s, %s, %s, %s, %s );" %
-          ( quote( value.object_id ), quote( value.revision ), quote( value.title ), quote( value.contents ), quote( None ), quote( "f" ), quote( value.deleted_from or None ), quote( None ) )
+          ( quote( value.object_id ), quote( value.revision ), quote( value.title or None ), quote( value.contents or None ), quote( None ), quote( "f" ), quote( value.deleted_from or None ), quote( None ) )
         )
       elif class_name == "User":
         if value.username is None: continue # note: this will skip all demo users

tools/verifyconvertdb.py

@@ -0,0 +1,176 @@
+#!/usr/bin/python2.5
+
+import os
+import os.path
+import psycopg2 as psycopg
+from controller.Database import Database
+from controller.Scheduler import Scheduler
+
+
+def quote( value ):
+  if value is None:
+    return "null"
+
+  value = unicode( value )
+  return "'%s'" % value.replace( "'", "''" ).replace( "\\", "\\\\" )
+
+
+class Verifier( object ):
+  """
+  Verifies a conversion of a Luminotes database from bsddb to PostgreSQL that was performed with
+  convertdb.py.
+  """
+  def __init__( self, scheduler, database ):
+    self.scheduler = scheduler
+    self.database = database
+
+    self.conn = psycopg.connect( "dbname=luminotes user=luminotes password=dev" )
+    self.cursor = self.conn.cursor()
+
+    thread = self.verify_database()
+    self.scheduler.add( thread )
+    self.scheduler.wait_for( thread )
+
+  def verify_database( self ):
+    inserts = set()
+
+    for key in self.database._Database__db.keys():
+      if not self.database._Database__db.get( key ):
+        continue
+
+      self.database.load( key, self.scheduler.thread )
+      value = ( yield Scheduler.SLEEP )
+
+      class_name = value.__class__.__name__
+
+      if class_name == "Notebook":
+        self.verify_notebook( value )
+      elif class_name == "Note":
+        self.cursor.execute(
+          "select * from note where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) )
+        )
+
+        for row in self.cursor.fetchmany():
+          assert row[ 0 ] == value.object_id
+          assert row[ 1 ].replace( tzinfo = None ) == value.revision
+          assert row[ 2 ] == ( value.title and value.title.encode( "utf8" ) or None )
+          assert row[ 3 ] == ( value.contents and value.contents.encode( "utf8" ) or None )
+          # not checking for existence of row 4 (notebook_id), because notes deleted from the trash don't have a notebook id
+          assert row[ 5 ] is not None
+          assert row[ 6 ] == ( value.deleted_from or None )
+          if row[ 5 ] is True: # if this is a startup note, it should have a rank
+            assert row[ 7 ] is not None
+      elif class_name == "User":
+        # skip demo users
+        if value.username is None: continue
+
+        self.cursor.execute(
+          "select * from luminotes_user where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) )
+        )
+
+        for row in self.cursor.fetchmany():
+          assert row[ 0 ] == value.object_id
+          assert row[ 1 ].replace( tzinfo = None ) == value.revision
+          assert row[ 2 ] == value.username
+          assert row[ 3 ] == value._User__salt
+          assert row[ 4 ] == value._User__password_hash
+          assert row[ 5 ] == value.email_address
+          assert row[ 6 ] == value.storage_bytes
+          assert row[ 7 ] == value.rate_plan
+
+        for notebook in value.notebooks:
+          if notebook is None: continue
+
+          read_write = ( notebook.__class__.__name__ == "Notebook" )
+
+          self.cursor.execute(
+            "select * from user_notebook where user_id = %s and notebook_id = %s;" % ( quote( value.object_id ), quote( notebook.object_id ) )
+          )
+
+          for row in self.cursor.fetchmany():
+            assert row[ 0 ] == value.object_id
+            assert row[ 1 ] == notebook.object_id
+            assert row[ 2 ] == read_write
+
+          self.verify_notebook( notebook )
+
+      elif class_name == "Read_only_notebook":
+        self.verify_notebook( value._Read_only_notebook__wrapped )
+      elif class_name == "Password_reset":
+        # skip password resets that are already redeemed
+        if value.redeemed: continue
+
+        self.cursor.execute(
+          "select * from password_reset where id = %s;" % quote( value.object_id )
+        )
+
+        for row in self.cursor.fetchmany():
+          assert row[ 0 ] == value.email_address
+          assert row[ 1 ] == False
+          assert row[ 2 ] == value.object_id
+      elif class_name == "User_list":
+        pass
+      else:
+        raise Exception( "Unverified value of type %s" % class_name )
+
+    self.conn.commit()
+    yield None
+
+  def verify_notebook( self, value ):
+    self.cursor.execute(
+      "select * from notebook where id = %s and revision = %s;" % ( quote( value.object_id ), quote( value.revision ) )
+    )
+
+    for row in self.cursor.fetchmany():
+      assert row[ 0 ] == value.object_id
+      assert row[ 1 ].replace( tzinfo = None ) == value.revision
+      assert row[ 2 ] == value.name
+      if value.trash:
+        assert row[ 3 ] == value.trash.object_id
+      else:
+        assert row[ 3 ] == None
+
+    startup_note_ids = [ note.object_id for note in value.startup_notes ]
+    for note in value.notes:
+      self.cursor.execute(
+        "select * from note where id = %s and revision = %s;" % ( quote( note.object_id ), quote( value.revision ) )
+      )
+
+      for row in self.cursor.fetchmany():
+        assert row[ 0 ] == note.object_id
+        assert row[ 1 ].replace( tzinfo = None ) == note.revision
+        assert row[ 2 ] == note.title
+        assert row[ 3 ] == note.contents
+        assert row[ 4 ] == value.object_id
+        assert row[ 5 ] == ( note.object_id in startup_note_ids )
+        assert row[ 6 ] == note.deleted_from
+        if row[ 5 ] is True: # if this is a startup note, it should have a rank
+          assert row[ 7 ] is not None
+      
+    for note in value.startup_notes:
+      self.cursor.execute(
+        "select * from note where id = %s and revision = %s order by rank;" % ( quote( note.object_id ), quote( value.revision ) )
+      )
+
+      rank = 0
+      for row in self.cursor.fetchmany():
+        assert row[ 0 ] == note.object_id
+        assert row[ 1 ].replace( tzinfo = None ) == note.revision
+        assert row[ 2 ] == note.title
+        assert row[ 3 ] == note.contents
+        assert row[ 4 ] == value.object_id
+        assert row[ 5 ] == True
+        assert row[ 6 ] == note.deleted_from
+        assert row[ 7 ] == rank
+        rank += 1
+
+
+def main():
+  scheduler = Scheduler()
+  database = Database( scheduler, "data.db" )
+  initializer = Verifier( scheduler, database )
+  scheduler.wait_until_idle()
+
+
+if __name__ == "__main__":
+  main()