openstack/cinder
Code Issues Proposed changes

Purge deleted rows

Browse Source 
Adds the ability to clean up rows that are already marked as
deleted of a certain specified age. Age is calculated as timedelta
from now() in days, which are given at command line

DocImpact
Change-Id: Ia50ab0dc4aa1547a5a6a2430f7941aab194e4baf
Implements: blueprint database-purge
This commit is contained in:
Abel Lopez 2015-01-12 18:50:00 -08:00
parent fadc833474
commit 7587e194a0
4 changed files with 178 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cinder/cmd/manage.py
View File

@ -228,6 +228,17 @@ class DbCommands(object):
db_migration.MIGRATE_REPO_PATH,
db_migration.INIT_VERSION))
@args('age_in_days', type=int,
help='Purge deleted rows older than age in days')
def purge(self, age_in_days):
"""Purge deleted rows older than a given age from cinder tables."""
age_in_days = int(age_in_days)
if age_in_days <= 0:
print(_("Must supply a positive, non-zero value for age"))
exit(1)
ctxt = context.get_admin_context()
db.purge_deleted_rows(ctxt, age_in_days)
class VersionCommands(object):
"""Class for exposing the codebase version."""

9
cinder/db/api.py
View File

@ -935,3 +935,12 @@ def cgsnapshot_update(context, cgsnapshot_id, values):
def cgsnapshot_destroy(context, cgsnapshot_id):
"""Destroy the cgsnapshot or raise if it does not exist."""
return IMPL.cgsnapshot_destroy(context, cgsnapshot_id)
def purge_deleted_rows(context, age_in_days):
"""Purge deleted rows older than given age from cinder tables
Raises InvalidParameterValue if age_in_days is incorrect.
:returns: number of deleted rows
"""
return IMPL.purge_deleted_rows(context, age_in_days=age_in_days)

54
cinder/db/sqlalchemy/api.py
View File

@ -19,6 +19,8 @@
"""Implementation of SQLAlchemy backend."""
from datetime import datetime
from datetime import timedelta
import functools
import sys
import threading
@ -34,9 +36,11 @@ from oslo.utils import timeutils
import osprofiler.sqlalchemy
import six
import sqlalchemy
from sqlalchemy import MetaData
from sqlalchemy import or_
from sqlalchemy.orm import joinedload, joinedload_all
from sqlalchemy.orm import RelationshipProperty
from sqlalchemy.schema import Table
from sqlalchemy.sql.expression import literal_column
from sqlalchemy.sql.expression import true
from sqlalchemy.sql import func
@ -44,7 +48,7 @@ from sqlalchemy.sql import func
from cinder.common import sqlalchemyutils
from cinder.db.sqlalchemy import models
from cinder import exception
from cinder.i18n import _, _LW
from cinder.i18n import _, _LW, _LE, _LI
from cinder.openstack.common import log as logging
from cinder.openstack.common import uuidutils
@ -3285,3 +3289,51 @@ def cgsnapshot_destroy(context, cgsnapshot_id):
'deleted': True,
'deleted_at': timeutils.utcnow(),
'updated_at': literal_column('updated_at')})
@require_admin_context
def purge_deleted_rows(context, age_in_days):
"""Purge deleted rows older than age from cinder tables."""
try:
age_in_days = int(age_in_days)
except ValueError:
msg = _LE('Invalid valude for age, %(age)s')
LOG.exception(msg, {'age': age_in_days})
raise exception.InvalidParameterValue(msg % {'age': age_in_days})
if age_in_days <= 0:
msg = _LE('Must supply a positive value for age')
LOG.exception(msg)
raise exception.InvalidParameterValue(msg)
engine = get_engine()
session = get_session()
metadata = MetaData()
metadata.bind = engine
tables = []
for model_class in models.__dict__.itervalues():
if hasattr(model_class, "__tablename__"):
tables.append(model_class.__tablename__)
# Reorder the list so the volumes table is last to avoid FK constraints
tables.remove("volumes")
tables.append("volumes")
for table in tables:
t = Table(table, metadata, autoload=True)
LOG.info(_LI('Purging deleted rows older than age=%(age)d days '
'from table=%(table)s'), {'age': age_in_days,
'table': table})
deleted_age = datetime.now() - timedelta(days=age_in_days)
try:
with session.begin():
result = session.execute(
t.delete()
.where(t.c.deleted_at < deleted_age))
except db_exc.DBReferenceError:
LOG.exception(_LE('DBError detected when purging from '
'table=%(table)s'), {'table': table})
raise
rows_purged = result.rowcount
LOG.info(_LI("Deleted %(row)d rows from table=%(table)s"),
{'row': rows_purged, 'table': table})

105
cinder/tests/db/test_purge.py Normal file
View File

@ -0,0 +1,105 @@
# Copyright (C) 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for db purge."""
from datetime import datetime
from datetime import timedelta
import uuid
from cinder import context
from cinder import db
from cinder.db.sqlalchemy import api as db_api
from cinder import exception
from cinder.openstack.common import log as logging
from cinder import test
from oslo_db.sqlalchemy import utils as sqlalchemyutils
LOG = logging.getLogger(__name__)
class PurgeDeletedTest(test.TestCase):
def setUp(self):
super(PurgeDeletedTest, self).setUp()
self.context = context.get_admin_context()
self.engine = db_api.get_engine()
self.session = db_api.get_session()
self.conn = self.engine.connect()
self.volumes = sqlalchemyutils.get_table(
self.engine, "volumes")
# The volume_metadata table has a FK of volume_id
self.vm = sqlalchemyutils.get_table(
self.engine, "volume_metadata")
self.uuidstrs = []
for unused in range(6):
self.uuidstrs.append(uuid.uuid4().hex)
# Add 6 rows to table
for uuidstr in self.uuidstrs:
ins_stmt = self.volumes.insert().values(id=uuidstr)
self.conn.execute(ins_stmt)
ins_stmt = self.vm.insert().values(volume_id=uuidstr)
self.conn.execute(ins_stmt)
# Set 4 of them deleted, 2 are 60 days ago, 2 are 20 days ago
old = datetime.now() - timedelta(days=20)
older = datetime.now() - timedelta(days=60)
make_old = self.volumes.update().\
where(self.volumes.c.id.in_(self.uuidstrs[1:3]))\
.values(deleted_at=old)
make_older = self.volumes.update().\
where(self.volumes.c.id.in_(self.uuidstrs[4:6]))\
.values(deleted_at=older)
make_meta_old = self.vm.update().\
where(self.vm.c.volume_id.in_(self.uuidstrs[1:3]))\
.values(deleted_at=old)
make_meta_older = self.vm.update().\
where(self.vm.c.volume_id.in_(self.uuidstrs[4:6]))\
.values(deleted_at=older)
self.conn.execute(make_old)
self.conn.execute(make_older)
self.conn.execute(make_meta_old)
self.conn.execute(make_meta_older)
def test_purge_deleted_rows_old(self):
# Purge at 30 days old, should only delete 2 rows
db.purge_deleted_rows(self.context, age_in_days=30)
rows = self.session.query(self.volumes).count()
meta_rows = self.session.query(self.vm).count()
# Verify that we only deleted 2
self.assertEqual(4, rows)
self.assertEqual(4, meta_rows)
def test_purge_deleted_rows_older(self):
# Purge at 10 days old now, should delete 2 more rows
db.purge_deleted_rows(self.context, age_in_days=10)
rows = self.session.query(self.volumes).count()
meta_rows = self.session.query(self.vm).count()
# Verify that we only have 2 rows now
self.assertEqual(2, rows)
self.assertEqual(2, meta_rows)
def test_purge_deleted_rows_bad_args(self):
# Test with no age argument
self.assertRaises(TypeError, db.purge_deleted_rows, self.context)
# Test purge with non-integer
self.assertRaises(exception.InvalidParameterValue,
db.purge_deleted_rows, self.context,
age_in_days='ten')
# Test with negative value
self.assertRaises(exception.InvalidParameterValue,
db.purge_deleted_rows, self.context,
age_in_days=-1)