DB migration tests

Refactored migration tests to use OpportunisticTestCase, removed
unused code and ``test_migrations.conf`` file.

The main feature of this approach is to create a new database with
random name for each migration test.  This will avoid migration tests of
race conditions and reduce tests intersection. After this change, database
``openstack_citest`` will be used only for initial connection to the database.

``test_migrations.conf`` file not required anymore, because we create test
database for migration test, so we no longer need to keep database credentials.

Implements blueprint: db-migration-tests
Related-bug: #1266595
Change-Id: I4febd485ff53936b636947c86773a23724e24c65

cinder/cmd/manage.py

@@ -62,6 +62,7 @@ import warnings
 warnings.simplefilter('once', DeprecationWarning)
 
 from oslo.config import cfg
+from oslo.db.sqlalchemy import migration
 from oslo import messaging
 
 from cinder import i18n
@@ -71,7 +72,8 @@ i18n.enable_lazy()
 from cinder.common import config  # noqa
 from cinder import context
 from cinder import db
-from cinder.db import migration
+from cinder.db import migration as db_migration
+from cinder.db.sqlalchemy import api as db_api
 from cinder.i18n import _
 from cinder.openstack.common import log as logging
 from cinder.openstack.common import uuidutils
@@ -218,11 +220,13 @@ class DbCommands(object):
           help='Database version')
     def sync(self, version=None):
         """Sync the database up to the most recent version."""
-        return migration.db_sync(version)
+        return db_migration.db_sync(version)
 
     def version(self):
         """Print the current database version."""
-        print(migration.db_version())
+        print(migration.db_version(db_api.get_engine(),
+                                   db_migration.MIGRATE_REPO_PATH,
+                                   db_migration.INIT_VERSION))
 
 
 class VersionCommands(object):

cinder/db/migration.py

@@ -16,23 +16,46 @@
 
 """Database setup and migration commands."""
 
-from cinder import utils
+import os
+import threading
+
+from oslo.config import cfg
+from oslo import db
+from stevedore import driver
+
+from cinder.db.sqlalchemy import api as db_api
+
+INIT_VERSION = 000
+
+_IMPL = None
+_LOCK = threading.Lock()
+
+db.options.set_defaults(cfg.CONF)
+
+MIGRATE_REPO_PATH = os.path.join(
+    os.path.abspath(os.path.dirname(__file__)),
+    'sqlalchemy',
+    'migrate_repo',
+)
 
 
-IMPL = utils.LazyPluggable('db_backend',
-                           sqlalchemy='cinder.db.sqlalchemy.migration')
+def get_backend():
+    global _IMPL
+    if _IMPL is None:
+        with _LOCK:
+            if _IMPL is None:
+                _IMPL = driver.DriverManager(
+                    "cinder.database.migration_backend",
+                    cfg.CONF.database.backend).driver
+    return _IMPL
 
 
-def db_sync(version=None):
+def db_sync(version=None, init_version=INIT_VERSION, engine=None):
     """Migrate the database to `version` or the most recent version."""
-    return IMPL.db_sync(version=version)
 
-
-def db_version():
-    """Display the current database version."""
-    return IMPL.db_version()
-
-
-def db_initial_version():
-    """The starting version for the database."""
-    return IMPL.db_initial_version()
+    if engine is None:
+        engine = db_api.get_engine()
+    return get_backend().db_sync(engine=engine,
+                                 abs_path=MIGRATE_REPO_PATH,
+                                 version=version,
+                                 init_version=init_version)

cinder/db/sqlalchemy/migrate_repo/versions/001_cinder_init.py

@@ -23,10 +23,7 @@ from cinder.openstack.common import log as logging
 LOG = logging.getLogger(__name__)
 
 
-def upgrade(migrate_engine):
-    meta = MetaData()
-    meta.bind = migrate_engine
-
+def define_tables(meta):
     migrations = Table(
         'migrations', meta,
         Column('created_at', DateTime),
@@ -217,21 +214,27 @@ def upgrade(migrate_engine):
                nullable=True),
         mysql_engine='InnoDB'
     )
+    return [sm_flavors,
+            sm_backend_config,
+            snapshots,
+            volume_types,
+            volumes,
+            iscsi_targets,
+            migrations,
+            quotas,
+            services,
+            sm_volume,
+            volume_metadata,
+            volume_type_extra_specs]
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
 
     # create all tables
     # Take care on create order for those with FK dependencies
-    tables = [sm_flavors,
-              sm_backend_config,
-              snapshots,
-              volume_types,
-              volumes,
-              iscsi_targets,
-              migrations,
-              quotas,
-              services,
-              sm_volume,
-              volume_metadata,
-              volume_type_extra_specs]
+    tables = define_tables(meta)
 
     for table in tables:
         try:
@@ -268,4 +271,10 @@ def upgrade(migrate_engine):
 
 
 def downgrade(migrate_engine):
-    LOG.exception(_('Downgrade from initial Cinder install is unsupported.'))
+    meta = MetaData()
+    meta.bind = migrate_engine
+    tables = define_tables(meta)
+    tables.reverse()
+    for table in tables:
+        LOG.info("dropping table %(table)s" % {'table': table})
+        table.drop()

cinder/db/sqlalchemy/migrate_repo/versions/011_add_bootable_column.py

@@ -40,5 +40,4 @@ def downgrade(migrate_engine):
 
     volumes = Table('volumes', meta, autoload=True)
     bootable = volumes.columns.bootable
-    #bootable = Column('bootable', Boolean)
     volumes.drop_column(bootable)

cinder/db/sqlalchemy/migration.py

@@ -1,86 +0,0 @@
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-
-import os
-
-from migrate import exceptions as versioning_exceptions
-from migrate.versioning import api as versioning_api
-from migrate.versioning.repository import Repository
-import sqlalchemy
-
-from cinder.db.sqlalchemy.api import get_engine
-from cinder import exception
-from cinder.i18n import _
-
-INIT_VERSION = 000
-_REPOSITORY = None
-
-
-def db_sync(version=None):
-    if version is not None:
-        try:
-            version = int(version)
-        except ValueError:
-            raise exception.Error(_("version should be an integer"))
-
-    current_version = db_version()
-    repository = _find_migrate_repo()
-    if version is None or version > current_version:
-        return versioning_api.upgrade(get_engine(), repository, version)
-    else:
-        return versioning_api.downgrade(get_engine(), repository,
-                                        version)
-
-
-def db_version():
-    repository = _find_migrate_repo()
-    try:
-        return versioning_api.db_version(get_engine(), repository)
-    except versioning_exceptions.DatabaseNotControlledError:
-        # If we aren't version controlled we may already have the database
-        # in the state from before we started version control, check for that
-        # and set up version_control appropriately
-        meta = sqlalchemy.MetaData()
-        engine = get_engine()
-        meta.reflect(bind=engine)
-        tables = meta.tables
-        if len(tables) == 0:
-            db_version_control(INIT_VERSION)
-            return versioning_api.db_version(get_engine(), repository)
-        else:
-            raise exception.Error(_("Upgrade DB using Essex release first."))
-
-
-def db_initial_version():
-    return INIT_VERSION
-
-
-def db_version_control(version=None):
-    repository = _find_migrate_repo()
-    versioning_api.version_control(get_engine(), repository, version)
-    return version
-
-
-def _find_migrate_repo():
-    """Get the path for the migrate repository."""
-    global _REPOSITORY
-    path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                        'migrate_repo')
-    assert os.path.exists(path)
-    if _REPOSITORY is None:
-        _REPOSITORY = Repository(path)
-    return _REPOSITORY

cinder/test.py

@@ -76,13 +76,6 @@ class Database(fixtures.Fixture):
         self.engine = db_api.get_engine()
         self.engine.dispose()
         conn = self.engine.connect()
-        if sql_connection == "sqlite://":
-            if db_migrate.db_version() > db_migrate.db_initial_version():
-                return
-        else:
-            testdb = os.path.join(CONF.state_path, sqlite_db)
-            if os.path.exists(testdb):
-                return
         db_migrate.db_sync()
 #        self.post_migrations()
         if sql_connection == "sqlite://":
@@ -91,6 +84,7 @@ class Database(fixtures.Fixture):
             self.engine.dispose()
         else:
             cleandb = os.path.join(CONF.state_path, sqlite_clean_db)
+            testdb = os.path.join(CONF.state_path, sqlite_db)
             shutil.copyfile(testdb, cleandb)
 
     def setUp(self):

cinder/tests/test_cmd.py

@@ -337,11 +337,11 @@ class TestCinderManageCmd(test.TestCase):
         db_cmds.sync(version=version)
         db_sync.assert_called_once_with(version)
 
-    @mock.patch('cinder.db.migration.db_version')
+    @mock.patch('oslo.db.sqlalchemy.migration.db_version')
     def test_db_commands_version(self, db_version):
         db_cmds = cinder_manage.DbCommands()
         db_cmds.version()
-        db_version.assert_called_once_with()
+        self.assertEqual(1, db_version.call_count)
 
     @mock.patch('cinder.version.version_string')
     def test_versions_commands_list(self, version_string):

cinder/tests/test_migrations.conf

@@ -1,9 +0,0 @@
-[DEFAULT]
-# Set up any number of migration data stores you want, one
-# The "name" used in the test is the config variable key.
-#sqlite=sqlite:///test_migrations.db
-sqlite=sqlite://
-#mysql=mysql://root:@localhost/test_migrations
-#postgresql=postgresql://user:pass@localhost/test_migrations
-[walk_style]
-snake_walk=yes

setup.cfg

@@ -57,6 +57,9 @@ oslo.messaging.notify.drivers =
     cinder.openstack.common.notifier.rpc_notifier = oslo.messaging.notify._impl_messaging:MessagingDriver
     cinder.openstack.common.notifier.test_notifier = oslo.messaging.notify._impl_test:TestDriver
 
+cinder.database.migration_backend =
+    sqlalchemy = oslo.db.sqlalchemy.migration
+
 [build_sphinx]
 all_files = 1
 build-dir = doc/build

test-requirements.txt

@@ -11,6 +11,7 @@ mock>=1.0
 mox>=0.5.3
 MySQL-python
 psycopg2
+oslotest>=1.2.0  # Apache-2.0
 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
 python-subunit>=0.0.18
 testtools>=0.9.36,!=1.2.0