From 3cd2ebd3759c76fdf5a292e612127094c7aa2b17 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Fri, 16 Mar 2018 13:11:59 +0100 Subject: [PATCH] Avoid second restart on offline upgrades On offline upgrades, due to the rolling upgrade mechanism we need to restart the cinder services twice to complete the upgrade, just like in the rolling upgrade case. The current offline upgrade process is: - Stop cinder services - Upgrade the cinder nodes - Sync your DB - Start the cinder services - Restart all the cinder services This second restart creates a bad user experience and it should not be necessary on an offline upgrade, so this patch adds a new optional parameter -called "--bump-versions"- to the cinder-manage db sync command that allows us to skip the restart of the services. Closes-Bug: #1756321 Change-Id: I1b58c637f6b2187a78c9c00a6c4933335439ad6f --- cinder/cmd/manage.py | 37 ++++++++++++++++++- cinder/tests/unit/test_cmd.py | 29 ++++++++++++++- doc/source/man/cinder-manage.rst | 11 +++++- .../sync-bump-versions-a1e6f6359173892e.yaml | 16 ++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/sync-bump-versions-a1e6f6359173892e.yaml diff --git a/cinder/cmd/manage.py b/cinder/cmd/manage.py index fab416a7e0e..72a22dabbcf 100644 --- a/cinder/cmd/manage.py +++ b/cinder/cmd/manage.py @@ -67,6 +67,7 @@ from oslo_log import log as logging from oslo_utils import timeutils # Need to register global_opts +from cinder.backup import rpcapi as backup_rpcapi from cinder.common import config # noqa from cinder.common import constants from cinder import context @@ -77,7 +78,9 @@ from cinder.db.sqlalchemy import models from cinder import exception from cinder.i18n import _ from cinder import objects +from cinder.objects import base as ovo_base from cinder import rpc +from cinder.scheduler import rpcapi as scheduler_rpcapi from cinder import version from cinder.volume import rpcapi as volume_rpcapi from cinder.volume import utils as vutils @@ -85,6 +88,14 @@ from cinder.volume import utils as vutils CONF = cfg.CONF +RPC_VERSIONS = { + 'cinder-scheduler': scheduler_rpcapi.SchedulerAPI.RPC_API_VERSION, + 'cinder-volume': volume_rpcapi.VolumeAPI.RPC_API_VERSION, + 'cinder-backup': backup_rpcapi.BackupAPI.RPC_API_VERSION, +} + +OVO_VERSION = ovo_base.OBJ_VERSIONS.get_current() + def _get_non_shared_target_hosts(ctxt): hosts = [] @@ -266,18 +277,40 @@ class DbCommands(object): @args('version', nargs='?', default=None, type=int, help='Database version') - def sync(self, version=None): + @args('--bump-versions', dest='bump_versions', default=False, + action='store_true', + help='Update RPC and Objects versions when doing offline upgrades, ' + 'with this we no longer need to restart the services twice ' + 'after the upgrade to prevent ServiceTooOld exceptions.') + def sync(self, version=None, bump_versions=False): """Sync the database up to the most recent version.""" if version is not None and version > db.MAX_INT: print(_('Version should be less than or equal to ' '%(max_version)d.') % {'max_version': db.MAX_INT}) sys.exit(1) try: - return db_migration.db_sync(version) + result = db_migration.db_sync(version) except db_exc.DBMigrationError as ex: print("Error during database migration: %s" % ex) sys.exit(1) + try: + if bump_versions: + ctxt = context.get_admin_context() + services = objects.ServiceList.get_all(ctxt) + for service in services: + rpc_version = RPC_VERSIONS[service.binary] + if (service.rpc_current_version != rpc_version or + service.object_current_version != OVO_VERSION): + service.rpc_current_version = rpc_version + service.object_current_version = OVO_VERSION + service.save() + except Exception as ex: + print(_('Error during service version bump: %s') % ex) + sys.exit(2) + + return result + def version(self): """Print the current database version.""" print(migration.db_version(db_api.get_engine(), diff --git a/cinder/tests/unit/test_cmd.py b/cinder/tests/unit/test_cmd.py index 2b612ff0d52..3f36f2b001d 100644 --- a/cinder/tests/unit/test_cmd.py +++ b/cinder/tests/unit/test_cmd.py @@ -356,12 +356,39 @@ class TestCinderManageCmd(test.TestCase): ex = self.assertRaises(SystemExit, db_cmds.purge, age_in_days) self.assertEqual(1, ex.code) + @mock.patch('cinder.objects.ServiceList.get_all') @mock.patch('cinder.db.migration.db_sync') - def test_db_commands_sync(self, db_sync): + def test_db_commands_sync(self, db_sync, service_get_mock): version = 11 db_cmds = cinder_manage.DbCommands() db_cmds.sync(version=version) db_sync.assert_called_once_with(version) + service_get_mock.assert_not_called() + + @mock.patch('cinder.objects.Service.save') + @mock.patch('cinder.objects.ServiceList.get_all') + @mock.patch('cinder.db.migration.db_sync') + def test_db_commands_sync_bump_versions(self, db_sync, service_get_mock, + service_save): + ctxt = context.get_admin_context() + services = [fake_service.fake_service_obj(ctxt, + binary='cinder-' + binary, + rpc_current_version='0.1', + object_current_version='0.2') + for binary in ('volume', 'scheduler', 'backup')] + service_get_mock.return_value = services + + version = 11 + db_cmds = cinder_manage.DbCommands() + db_cmds.sync(version=version, bump_versions=True) + db_sync.assert_called_once_with(version) + + self.assertEqual(3, service_save.call_count) + for service in services: + self.assertEqual(cinder_manage.RPC_VERSIONS[service.binary], + service.rpc_current_version) + self.assertEqual(cinder_manage.OVO_VERSION, + service.object_current_version) @mock.patch('oslo_db.sqlalchemy.migration.db_version') def test_db_commands_version(self, db_version): diff --git a/doc/source/man/cinder-manage.rst b/doc/source/man/cinder-manage.rst index d4f816ab46a..d8c6cec2e05 100644 --- a/doc/source/man/cinder-manage.rst +++ b/doc/source/man/cinder-manage.rst @@ -50,10 +50,19 @@ Cinder Db Print the current database version. -``cinder-manage db sync`` +``cinder-manage db sync [--bump-versions] [version]`` Sync the database up to the most recent version. This is the standard way to create the db as well. + This command interprets the following options when it is invoked: + + version Database version + + --bump-versions Update RPC and Objects versions when doing offline + upgrades, with this we no longer need to restart the + services twice after the upgrade to prevent ServiceTooOld + exceptions. + ``cinder-manage db purge []`` Purge database entries that are marked as deleted, that are older than the number of days specified. diff --git a/releasenotes/notes/sync-bump-versions-a1e6f6359173892e.yaml b/releasenotes/notes/sync-bump-versions-a1e6f6359173892e.yaml new file mode 100644 index 00000000000..7a7aa80b43c --- /dev/null +++ b/releasenotes/notes/sync-bump-versions-a1e6f6359173892e.yaml @@ -0,0 +1,16 @@ +--- +features: + - Cinder-manage DB sync command can now bump the RPC and Objects versions of + the services to avoid a second restart when doing offline upgrades. +upgrade: + - On offline upgrades, due to the rolling upgrade mechanism we need to + restart the cinder services twice to complete the installation just like in + the rolling upgrades case. First you stop the cinder services, then you + upgrade them, you sync your DB, then you start all the cinder services, and + then you restart them all. To avoid this last restart we can now instruct + the DB sync to bump the services after the migration is completed, the + command to do this is `cinder-manage db sync --bump-versions` +fixes: + - After an offline upgrade we had to restart all Cinder services twice, now + with the `cinder-manage db sync --bump-versions` command we can avoid the + second restart.