From 942317aecd58264770778e29c580c96e06af7b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Thu, 16 Jun 2016 10:19:45 +0200 Subject: [PATCH] Add generic code for online data migrations To limit downtime during upgrade we've decided to ban data migrations done in migrations scripts and do them online during the runtime of the cloud. This commit implements generic infrastructure for that. This was part of a commit adding a mapping table between volume types and consistency groups, but I've separated it because it's needed independently. Also the aforementioned commit will probably be obsolete when we finish generic volume groups efforts. UpgradeImpact Related: blueprint online-schema-upgrades Change-Id: I655d47c39e44251cbbad6e4b3fbc8e8d73995d30 --- cinder/cmd/manage.py | 51 +++++++++++++++++++ cinder/tests/unit/test_cmd.py | 28 ++++++++++ ...ne-schema-migrations-d1c0d40f26d0f033.yaml | 13 +++++ 3 files changed, 92 insertions(+) create mode 100644 releasenotes/notes/cinder-manage-db-online-schema-migrations-d1c0d40f26d0f033.yaml diff --git a/cinder/cmd/manage.py b/cinder/cmd/manage.py index 1b58e00d6a3..1168de46367 100644 --- a/cinder/cmd/manage.py +++ b/cinder/cmd/manage.py @@ -208,6 +208,8 @@ class HostCommands(object): class DbCommands(object): """Class for managing the database.""" + online_migrations = () + def __init__(self): pass @@ -243,6 +245,55 @@ class DbCommands(object): "logs for more details.")) sys.exit(1) + def _run_migration(self, ctxt, max_count, ignore_state): + ran = 0 + for migration_meth in self.online_migrations: + count = max_count - ran + try: + found, done = migration_meth(ctxt, count, ignore_state) + except Exception: + print(_("Error attempting to run %(method)s") % + {'method': migration_meth.__name__}) + found = done = 0 + + if found: + print(_('%(total)i rows matched query %(meth)s, %(done)i ' + 'migrated') % {'total': found, + 'meth': migration_meth.__name__, + 'done': done}) + if max_count is not None: + ran += done + if ran >= max_count: + break + return ran + + @args('--max_count', metavar='', dest='max_count', type=int, + help='Maximum number of objects to consider.') + @args('--ignore_state', action='store_true', dest='ignore_state', + help='Force records to migrate even if another operation is ' + 'performed on them. This may be dangerous, please refer to ' + 'release notes for more information.') + def online_data_migrations(self, max_count=None, ignore_state=False): + """Perform online data migrations for the release in batches.""" + ctxt = context.get_admin_context() + if max_count is not None: + unlimited = False + if max_count < 1: + print(_('Must supply a positive value for max_number.')) + sys.exit(127) + else: + unlimited = True + max_count = 50 + print(_('Running batches of %i until complete.') % max_count) + + ran = None + while ran is None or ran != 0: + ran = self._run_migration(ctxt, max_count, ignore_state) + if not unlimited: + break + + sys.exit(1 if ran else 0) + class VersionCommands(object): """Class for exposing the codebase version.""" diff --git a/cinder/tests/unit/test_cmd.py b/cinder/tests/unit/test_cmd.py index 6de696e7508..e07e817e8d5 100644 --- a/cinder/tests/unit/test_cmd.py +++ b/cinder/tests/unit/test_cmd.py @@ -234,6 +234,34 @@ class TestCinderManageCmd(test.TestCase): with mock.patch('sys.stdout', new=six.StringIO()): self.assertRaises(exception.InvalidInput, db_cmds.sync, 1) + @mock.patch('cinder.cmd.manage.DbCommands.online_migrations', + (mock.Mock(side_effect=((2, 2), (0, 0)), __name__='foo'),)) + def test_db_commands_online_data_migrations(self): + db_cmds = cinder_manage.DbCommands() + exit = self.assertRaises(SystemExit, db_cmds.online_data_migrations) + self.assertEqual(0, exit.code) + cinder_manage.DbCommands.online_migrations[0].assert_has_calls( + (mock.call(mock.ANY, 50, False),) * 2) + + @mock.patch('cinder.cmd.manage.DbCommands.online_migrations', + (mock.Mock(side_effect=((2, 2), (0, 0)), __name__='foo'),)) + def test_db_commands_online_data_migrations_ignore_state_and_max(self): + db_cmds = cinder_manage.DbCommands() + exit = self.assertRaises(SystemExit, db_cmds.online_data_migrations, + 2, True) + self.assertEqual(1, exit.code) + cinder_manage.DbCommands.online_migrations[0].assert_called_once_with( + mock.ANY, 2, True) + + @mock.patch('cinder.cmd.manage.DbCommands.online_migrations', + (mock.Mock(side_effect=((2, 2), (0, 0)), __name__='foo'),)) + def test_db_commands_online_data_migrations_max_negative(self): + db_cmds = cinder_manage.DbCommands() + exit = self.assertRaises(SystemExit, db_cmds.online_data_migrations, + -1) + self.assertEqual(127, exit.code) + cinder_manage.DbCommands.online_migrations[0].assert_not_called() + @mock.patch('cinder.version.version_string') def test_versions_commands_list(self, version_string): version_cmds = cinder_manage.VersionCommands() diff --git a/releasenotes/notes/cinder-manage-db-online-schema-migrations-d1c0d40f26d0f033.yaml b/releasenotes/notes/cinder-manage-db-online-schema-migrations-d1c0d40f26d0f033.yaml new file mode 100644 index 00000000000..4bed0827862 --- /dev/null +++ b/releasenotes/notes/cinder-manage-db-online-schema-migrations-d1c0d40f26d0f033.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - To get rid of long running DB data migrations that must be run offline, + Cinder will now be able to execute them online, on a live cloud. Before + upgrading from Newton to Ocata operator needs to perform all the Newton + data migrations. To achieve that he needs to perform `cinder-manage db + online-data-migrations` until there are no records to be updated. To limit + DB performance impact migrations can be performed in chunks limited by + `--max_number` option. If your intent is to upgrade Cinder in a non-live + manner, you can use `--ignore-state` option safely. Please note that + finishing all the Newton data migrations will be enforced by the first + schema migration in Ocata, so you won't be able to upgrade to Ocata without + that.