diff --git a/controller/Schema_upgrader.py b/controller/Schema_upgrader.py new file mode 100644 index 0000000..bd91e5d --- /dev/null +++ b/controller/Schema_upgrader.py @@ -0,0 +1,138 @@ +import os.path +from model.Persistent import Persistent + + +class Schema_upgrader: + def __init__( self, database, glob = None, read_file = None ): + """ + Create a new schema upgrader and return it. + + @type database: Database + @param database: the database to use for all schema upgrades + @type glob: function or NoneType + @param glob: a custom function to use for globbing files as per glob.glob() (optional) + @type read_file: function or NoneType + @param read_file: a custom function to use for reading schema files (optional) + @rtype: Schema_upgrader + @return: newly constructed Schema_upgrader + """ + from glob import glob as real_glob + + self.__database = database + self.__glob = glob or real_glob + self.__read_file = read_file or Schema_upgrader.__read_file + + @staticmethod + def __read_file( filename ): + """ + Read a file and return all of its contents. + + @type filename: unicode + @param filename: full path of the file to read + @rtype: unicode + @return: full contents of the file + """ + return file( filename ).read() + + def upgrade_schema( self, to_version ): + """ + Upgrade the database from its current version to a given version, applying all intervening + schema delta files necessary to get there, and apply them in version order. If the given + version is unknown, this method will upgrade to the latest known schema version that is + before the given version. + + @type to_version: unicode + @param to_version: the desired version to upgrade to, as a string + """ + to_version = self.version_string_to_tuple( to_version ) + + try: + from_version = self.__database.select_one( tuple, "select * from schema_version;" ); + except: + from_version = None + + if self.__database.backend == Persistent.SQLITE_BACKEND: + extension = u"sqlite" + else: + extension = u"sql" + + filenames = self.__glob( u"model/delta/*.%s" % extension ) + versions = [] + + # make a list of all available schema delta files + for filename in filenames: + base_filename = os.path.basename( filename ) + + try: + version = self.version_string_to_tuple( base_filename ) + except ValueError: + continue + + # skip those versions that won't help us upgrade + if from_version and version <= from_version: + continue + if version > to_version: + continue + + versions.append( ( version, filename ) ) + + # sort the schema delta files by version + versions.sort( lambda a, b: cmp( a[ 0 ], b[ 0 ] ) ) + + # apply the schema delta files in sorted order + for ( version, filename ) in versions: + self.apply_schema_delta( version, filename ) + + self.__database.commit() + + def apply_schema_delta( self, version, filename ): + """ + Upgrade the database from its current version to a given version, applying only the named + schema delta file to do so. + + @type version: tuple + @param version: ( major, minor, release ) with each version part as an integer + @type filename: unicode + @param filename: full path to the schema delta file to apply + """ + self.__database.execute_script( self.__read_file( filename ), commit = False ) + + try: + self.__database.execute( "update schema_version set major = %s, minor = %s, release = %s;" % version, commit = False ); + # if the table doesn't yet exist, create it + except: + self.__database.execute( "create table schema_version ( major numeric, minor numeric, release numeric );", commit = False ); + self.__database.execute( "insert into schema_version values ( %s, %s, %s );" % version, commit = False ); + + @staticmethod + def version_string_to_tuple( version ): + """ + Given a version string with an optional file extension tacked on, convert the version to a + tuple of integers and return it. + + @type version: unicode + @param version: a version string of the form "major.minor.release" + @rtype: tuple + @return: ( major, minor, release ) with each version part as an integer + @raises ValueError: invalid version or version parts cannot be converted to integers + """ + VERSION_PARTS_COUNT = 3 + + parts = version.split( "." ) + length = len( parts ) + + if length == VERSION_PARTS_COUNT + 1: + ( major, minor, release, extension ) = parts + elif length == VERSION_PARTS_COUNT: + ( major, minor, release ) = parts + else: + raise ValueError() + + try: + major = int( major ) + minor = int( minor ) + release = int( release ) + except ( TypeError, ValueError ): + raise ValueError() + + return ( major, minor, release ) diff --git a/controller/test/Test_schema_upgrader.py b/controller/test/Test_schema_upgrader.py new file mode 100644 index 0000000..e98ea24 --- /dev/null +++ b/controller/test/Test_schema_upgrader.py @@ -0,0 +1,159 @@ +import re +from nose.tools import raises +from pysqlite2 import dbapi2 as sqlite +from Stub_object import Stub_object +from Stub_cache import Stub_cache +from model.Persistent import Persistent +from controller.Database import Database, Connection_wrapper +from controller.Schema_upgrader import Schema_upgrader + + +class Test_schema_upgrader( object ): + def setUp( self ): + # make an in-memory sqlite database to use during testing + self.connection = Connection_wrapper( sqlite.connect( ":memory:", detect_types = sqlite.PARSE_DECLTYPES, check_same_thread = False ) ) + self.cache = Stub_cache() + cursor = self.connection.cursor() + cursor.execute( Stub_object.sql_create_table() ) + + self.fake_files = {} # map of fake filename (full path) to fake file contents + self.database = Database( self.connection, self.cache ) + self.upgrader = Schema_upgrader( self.database, glob = self.glob, read_file = self.read_file ) + + def tearDown( self ): + self.database.close() + + def glob( self, glob_pattern ): + """ + A fake glob function that doesn't use the filesystem. + """ + re_pattern = re.compile( glob_pattern.replace( "*", "[^/]*" ) ) + + return [ filename for filename in self.fake_files.keys() if re_pattern.search( filename ) ] + + def read_file( self, filename ): + """ + A fake read file function that doesn't use the filesystem. + """ + contents = self.fake_files.get( filename ) + + if not contents: + raise IOError() + + return contents + + def test_upgrade_schema( self ): + self.fake_files = { + u"model/delta/5.6.7.sqlite": u"create table new_table ( foo text ); insert into new_table values ( 'hi' );", + u"model/delta/5.6.8.sqlite": u"insert into new_table values ( 'bye' );", + u"model/delta/5.6.10.sqlite": u"alter table new_table add column bar text;", + u"model/delta/5.7.11.sqlite": u"insert into new_table values ( 'whee', 'stuff' );", + u"model/delta/5.7.18.sqlite": u"insert into new_table values ( 'should not be present', 'nope' );", + } + + self.upgrader.upgrade_schema( u"5.7.11" ); + + result = self.database.select_many( tuple, u"select * from new_table;" ); + assert result == [ ( u"hi", None ), ( u"bye", None ), ( "whee", "stuff" ) ]; + + result = self.database.select_many( tuple, u"select * from schema_version;" ); + assert result == [ ( 5, 7, 11 ) ]; + + def test_upgrade_schema_with_schema_version_table( self ): + self.database.execute( u"create table schema_version ( major numeric, minor numeric, release numeric );" ); + self.database.execute( u"insert into schema_version values ( 0, 0, 0 );" ) + self.test_upgrade_schema(); + + def test_upgrade_schema_with_schema_version_table_and_specific_starting_version( self ): + self.database.execute( u"create table schema_version ( major numeric, minor numeric, release numeric );" ); + self.database.execute( u"insert into schema_version values ( 5, 6, 6 );" ) + + self.fake_files[ u"model/delta/5.6.1.sqlite" ] = u"this is not valid sql and should not be executed anyway;"; + self.fake_files[ u"model/delta/5.6.6.sqlite" ] = u"also invalid;"; + + self.test_upgrade_schema(); + + def test_upgrade_schema_with_future_ending_version( self ): + self.fake_files = { + u"model/delta/5.6.7.sqlite": u"create table new_table ( foo text ); insert into new_table values ( 'hi' );", + u"model/delta/5.6.8.sqlite": u"insert into new_table values ( 'bye' );", + u"model/delta/5.6.10.sqlite": u"alter table new_table add column bar text;", + u"model/delta/5.7.11.sqlite": u"insert into new_table values ( 'whee', 'stuff' );", + u"model/delta/5.7.18.sqlite": u"insert into new_table values ( 'more', 'and more' );", + } + + self.upgrader.upgrade_schema( u"5.8.55" ); + + result = self.database.select_many( tuple, u"select * from new_table;" ); + assert result == [ ( u"hi", None ), ( u"bye", None ), ( "whee", "stuff" ), ( "more", "and more" ) ]; + + result = self.database.select_many( tuple, u"select * from schema_version;" ); + assert result == [ ( 5, 7, 18 ) ]; + + def test_upgrade_schema_twice( self ): + self.test_upgrade_schema(); + + # the second upgrade should have no effect, because at this point it's already upgraded + self.test_upgrade_schema(); + + def test_upgrade_schema_with_filename_with_invalid_version( self ): + # the filename, not composed of all-integer parts, should be skipped + self.fake_files[ u"model/delta/5.6.9b.sqlite" ] = u"this is not valid sql and should not be executed anyway;"; + + self.test_upgrade_schema(); + + def test_apply_schema_delta( self ): + self.fake_files = { + u"model/delta/5.6.5.sqlite": u"insert into new_table values ( 'should not show up' );", + u"model/delta/5.6.7.sqlite": u"create table new_table ( foo text ); insert into new_table values ( 'hi' );", + u"model/delta/5.7.18.sqlite": u"insert into new_table values ( 'should not be present' );", + } + + self.upgrader.apply_schema_delta( ( 5, 6, 7 ), u"model/delta/5.6.7.sqlite" ) + + result = self.database.select_many( unicode, u"select * from new_table;" ); + assert result == [ u"hi" ]; + + result = self.database.select_many( tuple, u"select * from schema_version;" ); + assert result == [ ( 5, 6, 7 ) ]; + + def test_apply_schema_delta_with_schema_version_table( self ): + self.database.execute( u"create table schema_version ( major numeric, minor numeric, release numeric );" ); + self.database.execute( u"insert into schema_version values ( 0, 0, 0 );" ) + self.test_apply_schema_delta(); + + @raises( IOError ) + def test_apply_schema_delta_with_unknown_file( self ): + self.upgrader.apply_schema_delta( ( 5, 6, 7 ), u"model/delta/5.6.7.sqlite" ) + + def test_version_string_to_tuple( self ): + version = self.upgrader.version_string_to_tuple( "2.5.13" ) + + assert len( version ) == 3 + assert version[ 0 ] == 2 + assert version[ 1 ] == 5 + assert version[ 2 ] == 13 + + def test_version_string_to_tuple_with_extension( self ): + version = self.upgrader.version_string_to_tuple( "2.5.13.sqlite" ) + + assert len( version ) == 3 + assert version[ 0 ] == 2 + assert version[ 1 ] == 5 + assert version[ 2 ] == 13 + + @raises( ValueError ) + def test_version_string_to_tuple_with_too_many_parts( self ): + version = self.upgrader.version_string_to_tuple( "3.14.159.26.5" ) + + @raises( ValueError ) + def test_version_string_to_tuple_with_too_few_parts( self ): + version = self.upgrader.version_string_to_tuple( "3.14" ) + + @raises( ValueError ) + def test_version_string_to_tuple_with_non_integer_part( self ): + version = self.upgrader.version_string_to_tuple( "2.5b.13" ) + + @raises( ValueError ) + def test_version_string_to_tuple_with_empty_part( self ): + version = self.upgrader.version_string_to_tuple( "2..13" )