Merge "Dell PowerMax: multi detach req caused race conditions"

cinder/volume/drivers/dell_emc/powermax/fc.py

@@ -18,6 +18,7 @@ import ast
 from oslo_log import log as logging
 
 from cinder.common import constants
+from cinder import coordination
 from cinder import exception
 from cinder import interface
 from cinder.volume import driver
@@ -143,6 +144,8 @@ class PowerMaxFCDriver(san.SanDriver, driver.FibreChannelDriver):
     # ThirdPartySystems wiki
     CI_WIKI_NAME = "DellEMC_PowerMAX_CI"
 
+    driver_prefix = 'powermax'
+
     def __init__(self, *args, **kwargs):
 
         super(PowerMaxFCDriver, self).__init__(*args, **kwargs)
@@ -255,6 +258,7 @@ class PowerMaxFCDriver(san.SanDriver, driver.FibreChannelDriver):
         """
         pass
 
+    @coordination.synchronized('{self.driver_prefix}-{volume.id}')
     def initialize_connection(self, volume, connector):
         """Initializes the connection and returns connection info.
 
@@ -327,6 +331,7 @@ class PowerMaxFCDriver(san.SanDriver, driver.FibreChannelDriver):
 
         return data
 
+    @coordination.synchronized('{self.driver_prefix}-{volume.id}')
     def terminate_connection(self, volume, connector, **kwargs):
         """Disallow connection from connector.
 
@@ -790,3 +795,7 @@ class PowerMaxFCDriver(san.SanDriver, driver.FibreChannelDriver):
         :param snapshot: the cinder snapshot object
         """
         self.common.revert_to_snapshot(volume, snapshot)
+
+    @classmethod
+    def clean_volume_file_locks(cls, volume_id):
+        coordination.synchronized_remove(f'{cls.driver_prefix}-{volume_id}')

cinder/volume/drivers/dell_emc/powermax/iscsi.py

@@ -22,6 +22,7 @@ from oslo_log import log as logging
 from oslo_utils import strutils
 
 from cinder.common import constants
+from cinder import coordination
 from cinder import exception
 from cinder.i18n import _
 from cinder import interface
@@ -148,6 +149,8 @@ class PowerMaxISCSIDriver(san.SanISCSIDriver):
     # ThirdPartySystems wiki
     CI_WIKI_NAME = "DellEMC_PowerMAX_CI"
 
+    driver_prefix = 'powermax'
+
     def __init__(self, *args, **kwargs):
 
         super(PowerMaxISCSIDriver, self).__init__(*args, **kwargs)
@@ -260,6 +263,7 @@ class PowerMaxISCSIDriver(san.SanISCSIDriver):
         :param volume_id: the volume id
         """
 
+    @coordination.synchronized('{self.driver_prefix}-{volume.id}')
     def initialize_connection(self, volume, connector):
         """Initializes the connection and returns connection info.
 
@@ -440,6 +444,7 @@ class PowerMaxISCSIDriver(san.SanISCSIDriver):
 
         return properties
 
+    @coordination.synchronized('{self.driver_prefix}-{volume.id}')
     def terminate_connection(self, volume, connector, **kwargs):
         """Disallow connection from connector.
 
@@ -699,3 +704,7 @@ class PowerMaxISCSIDriver(san.SanISCSIDriver):
         :param snapshot: the cinder snapshot object
         """
         self.common.revert_to_snapshot(volume, snapshot)
+
+    @classmethod
+    def clean_volume_file_locks(cls, volume_id):
+        coordination.synchronized_remove(f'{cls.driver_prefix}-{volume_id}')

cinder/volume/drivers/dell_emc/powermax/rest.py

@@ -30,6 +30,7 @@ from cinder import exception
 from cinder.i18n import _
 from cinder.utils import retry
 from cinder.volume.drivers.dell_emc.powermax import utils
+from cinder.volume import volume_utils
 
 LOG = logging.getLogger(__name__)
 SLOPROVISIONING = 'sloprovisioning'
@@ -219,6 +220,7 @@ class PowerMaxRest(object):
             self.u4p_failover_lock = False
             raise exception.VolumeBackendAPIException(message=msg)
 
+    @volume_utils.trace()
     def request(self, target_uri, method, params=None, request_object=None,
                 u4p_check=False, retry=False):
         """Sends a request (GET, POST, PUT, DELETE) to the target api.

releasenotes/notes/bug-2089656-dell-powermax-fix-multi-detach-req-eb0f189841689ce8.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Dell PowerMax Driver `Bug #2089656
+    <https://bugs.launchpad.net/cinder/+bug/2089656>`_: Fixed
+    the issue that multiple detach requests caused race conditions
+    in Dell PowerMax driver. Also, improved trace logs for
+    PowerMax RESTAPI requests.