From 2155d681571337e3aa6f8ff02268dc8555213d05 Mon Sep 17 00:00:00 2001
From: Nilesh Thathagar <nilesh.thathagar@dell.com>
Date: Fri, 17 Jan 2025 06:25:50 +0000
Subject: [PATCH] Dell PowerMax: Enhanced the retry mechanism to verify the
 active snapshots.

Improved the retry mechanism of ``_cleanup_device_retry``
to account for active snapshots that take longer than
expected to be removed, thereby preventing deletion errors.

Updated the retry count based on trial and error.
The maximum observed retries reached the 6th attempt,
so we have set it to 6+1=7 retries.

Closes-Bug: #2092259
Change-Id: I9a09eb8e5b6efee59688684211d893e1af055241
---
 .../dell_emc/powermax/test_powermax_common.py | 31 +++++++++++++++++++
 .../drivers/dell_emc/powermax/common.py       |  2 +-
 ...-fix-active-snapshot-ccc3f9b6251d2634.yaml | 12 +++++++
 3 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 releasenotes/notes/bug-2092259-dell-powermax-volume-delete-failed-fix-active-snapshot-ccc3f9b6251d2634.yaml

diff --git a/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py b/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py
index f716d5f76a4..f114c956898 100644
--- a/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py
+++ b/cinder/tests/unit/volume/drivers/dell_emc/powermax/test_powermax_common.py
@@ -4900,3 +4900,34 @@ class PowerMaxCommonTest(test.TestCase):
         self.mock_object(self.common, 'configuration', configuration)
         kwargs_returned = self.common.get_attributes_from_cinder_config()
         self.assertEqual(kwargs_expected, kwargs_returned)
+
+    @mock.patch.object(common.PowerMaxCommon, '_cleanup_device_snapvx')
+    @mock.patch.object(rest.PowerMaxRest, 'get_volume_snapshot_list',
+                       side_effect=([{'snapshotName': 'temp-clone-snapshot'}],
+                                    []))
+    @mock.patch.object(rest.PowerMaxRest, 'find_snap_vx_sessions',
+                       side_effect=[(None, None)])
+    def test_cleanup_device_retry_1(self, mock_snapvx,
+                                    mock_ss_list, mock_clean):
+        self.common._cleanup_device_retry(
+            self.data.array, self.data.device_id, self.data.extra_specs)
+        self.assertEqual(2, mock_ss_list.call_count)
+        self.assertEqual(1, mock_snapvx.call_count)
+        self.assertEqual(2, mock_clean.call_count)
+
+    @mock.patch.object(common.PowerMaxCommon, '_cleanup_device_snapvx')
+    @mock.patch.object(rest.PowerMaxRest, 'get_volume_snapshot_list',
+                       return_value=[{'snapshotName': 'temp-clone-snapshot'}])
+    @mock.patch.object(rest.PowerMaxRest, 'find_snap_vx_sessions',
+                       side_effect=[(None, None)])
+    def test_cleanup_device_retry_2(self, mock_snapvx,
+                                    mock_ss_list, mock_clean):
+        self.assertRaises(
+            exception.VolumeBackendAPIException,
+            self.common._cleanup_device_retry,
+            self.data.array,
+            self.data.device_id,
+            self.data.extra_specs)
+        self.assertEqual(7, mock_ss_list.call_count)
+        self.assertEqual(0, mock_snapvx.call_count)
+        self.assertEqual(7, mock_clean.call_count)
diff --git a/cinder/volume/drivers/dell_emc/powermax/common.py b/cinder/volume/drivers/dell_emc/powermax/common.py
index ef4a5cfdea9..490cc17e298 100644
--- a/cinder/volume/drivers/dell_emc/powermax/common.py
+++ b/cinder/volume/drivers/dell_emc/powermax/common.py
@@ -2106,7 +2106,7 @@ class PowerMaxCommon(object):
             array, device_id, volume_name, extra_specs)
         return volume_name
 
-    @retry(retry_exc_tuple, interval=1, retries=3)
+    @retry(retry_exc_tuple, interval=2, retries=7)
     def _cleanup_device_retry(self, array, device_id, extra_specs):
         """Cleanup snapvx on the device
 
diff --git a/releasenotes/notes/bug-2092259-dell-powermax-volume-delete-failed-fix-active-snapshot-ccc3f9b6251d2634.yaml b/releasenotes/notes/bug-2092259-dell-powermax-volume-delete-failed-fix-active-snapshot-ccc3f9b6251d2634.yaml
new file mode 100644
index 00000000000..88c829b1e0c
--- /dev/null
+++ b/releasenotes/notes/bug-2092259-dell-powermax-volume-delete-failed-fix-active-snapshot-ccc3f9b6251d2634.yaml
@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    Dell PowerMax Driver `bug #2092259
+    <https://bugs.launchpad.net/cinder/+bug/2092259>`_: Before
+    a volume can be deleted, the driver issues a command to clean up
+    active snapshots in the backend and then polls the backend
+    to make sure the cleanup has occurred.
+    This fix enhances the polling mechanism to give the backend
+    more time to do the cleanup, thereby increasing the probability
+    that the driver will be able to make a successful volume deletion
+    request.