py3: Replace unicode with six.text_type

The unicode type was renamed to str in Python 3. Use six.text_type to
make the modified code compatible with Python 2 and Python 3.

The initial patch was generated by the unicode operation of the sixer
tool on: bin/* swift/ test/.

Change-Id: I9e13748ccde36ee8110756202d55d3ae945d4860

swift/account/reaper.py

@@ -24,6 +24,7 @@ from hashlib import md5
 import itertools
 
 from eventlet import GreenPool, sleep, Timeout
+import six
 
 import swift.common.db
 from swift.account.backend import AccountBroker, DATADIR
@@ -387,7 +388,7 @@ class AccountReaper(Daemon):
                     self.logger.error('ERROR: invalid storage policy index: %r'
                                       % policy_index)
                 for obj in objects:
-                    if isinstance(obj['name'], unicode):
+                    if isinstance(obj['name'], six.text_type):
                         obj['name'] = obj['name'].encode('utf8')
                     pool.spawn(self.reap_object, account, container, part,
                                nodes, obj['name'], policy_index)

swift/common/bufferedhttp.py

@@ -36,6 +36,7 @@ import socket
 import eventlet
 from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \
     HTTPResponse, HTTPSConnection, _UNKNOWN
+import six
 
 httplib = eventlet.import_patched('httplib')
 httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
@@ -198,12 +199,12 @@ def http_connect(ipaddr, port, device, partition, method, path,
     :param ssl: set True if SSL should be used (default: False)
     :returns: HTTPConnection object
     """
-    if isinstance(path, unicode):
+    if isinstance(path, six.text_type):
         try:
             path = path.encode("utf-8")
         except UnicodeError as e:
             logging.exception(_('Error encoding to UTF-8: %s'), str(e))
-    if isinstance(device, unicode):
+    if isinstance(device, six.text_type):
         try:
             device = device.encode("utf-8")
         except UnicodeError as e:

swift/common/constraints.py

@@ -19,6 +19,7 @@ import urllib
 import time
 from urllib import unquote
 
+import six
 from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
 
 from swift.common import utils, exceptions
@@ -333,7 +334,7 @@ def check_utf8(string):
     if not string:
         return False
     try:
-        if isinstance(string, unicode):
+        if isinstance(string, six.text_type):
             string.encode('utf-8')
         else:
             decoded = string.decode('UTF-8')
@@ -422,7 +423,7 @@ def check_name_format(req, name, target_type):
         raise HTTPPreconditionFailed(
             request=req,
             body='%s name cannot be empty' % target_type)
-    if isinstance(name, unicode):
+    if isinstance(name, six.text_type):
         name = name.encode('utf-8')
     if '/' in name:
         raise HTTPPreconditionFailed(

swift/common/db.py

@@ -23,6 +23,7 @@ from uuid import uuid4
 import sys
 import time
 import errno
+import six
 import six.moves.cPickle as pickle
 from swift import gettext_ as _
 from tempfile import mkstemp
@@ -48,11 +49,12 @@ PENDING_CAP = 131072
 
 
 def utf8encode(*args):
-    return [(s.encode('utf8') if isinstance(s, unicode) else s) for s in args]
+    return [(s.encode('utf8') if isinstance(s, six.text_type) else s)
+            for s in args]
 
 
 def utf8encodekeys(metadata):
-    uni_keys = [k for k in metadata if isinstance(k, unicode)]
+    uni_keys = [k for k in metadata if isinstance(k, six.text_type)]
     for k in uni_keys:
         sv = metadata[k]
         del metadata[k]

swift/common/middleware/dlo.py

@@ -15,6 +15,7 @@
 
 import os
 
+import six
 from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
 
 from hashlib import md5
@@ -81,7 +82,7 @@ class GetContext(WSGIContext):
                     break
 
                 seg_name = segment['name']
-                if isinstance(seg_name, unicode):
+                if isinstance(seg_name, six.text_type):
                     seg_name = seg_name.encode("utf-8")
 
                 # (obj path, etag, size, first byte, last byte)

swift/common/middleware/slo.py

@@ -355,7 +355,7 @@ class SloGetContext(WSGIContext):
                 # Restore the first/last state
                 self.first_byte, self.last_byte = orig_start, orig_end
             else:
-                if isinstance(seg_dict['name'], unicode):
+                if isinstance(seg_dict['name'], six.text_type):
                     seg_dict['name'] = seg_dict['name'].encode("utf-8")
                 yield (seg_dict,
                        max(0, self.first_byte) + range_start,
@@ -655,7 +655,7 @@ class StaticLargeObject(object):
         last_obj_path = None
         for index, seg_dict in enumerate(parsed_data):
             obj_name = seg_dict['path']
-            if isinstance(obj_name, unicode):
+            if isinstance(obj_name, six.text_type):
                 obj_name = obj_name.encode('utf-8')
             obj_path = '/'.join(['', vrs, account, obj_name.lstrip('/')])
             if req.path == quote(obj_path):

swift/common/middleware/versioned_writes.py

@@ -113,6 +113,7 @@ Disable versioning from a container (x is any value except empty)::
 -H "X-Remove-Versions-Location: x" http://<storage_url>/container
 """
 
+import six
 import time
 from urllib import quote, unquote
 from swift.common.utils import get_logger, Timestamp, json, \
@@ -412,7 +413,7 @@ class VersionedWritesMiddleware(object):
         # for backwards compatibility feature is enabled.
         object_versions = container_info.get(
             'sysmeta', {}).get('versions-location')
-        if object_versions and isinstance(object_versions, unicode):
+        if object_versions and isinstance(object_versions, six.text_type):
             object_versions = object_versions.encode('utf-8')
         elif not object_versions:
             object_versions = container_info.get('versions')

swift/common/request_helpers.py

@@ -25,6 +25,9 @@ import itertools
 import sys
 import time
 from urllib import unquote
+
+import six
+
 from swift import gettext_ as _
 from swift.common.storage_policy import POLICIES
 from swift.common.constraints import FORMAT2CONTENT_TYPE
@@ -51,7 +54,7 @@ def get_param(req, name, default=None):
     :raises: HTTPBadRequest if param not valid UTF-8 byte sequence
     """
     value = req.params.get(name, default)
-    if value and not isinstance(value, unicode):
+    if value and not isinstance(value, six.text_type):
         try:
             value.decode('utf8')    # Ensure UTF8ness
         except UnicodeDecodeError:

swift/common/swob.py

@@ -45,6 +45,7 @@ import random
 import functools
 import inspect
 
+import six
 from six import BytesIO
 from six import StringIO
 from six.moves import urllib
@@ -248,7 +249,7 @@ class HeaderEnvironProxy(MutableMapping):
     def __setitem__(self, key, value):
         if value is None:
             self.environ.pop(self._normalize(key), None)
-        elif isinstance(value, unicode):
+        elif isinstance(value, six.text_type):
             self.environ[self._normalize(key)] = value.encode('utf-8')
         else:
             self.environ[self._normalize(key)] = str(value)
@@ -293,7 +294,7 @@ class HeaderKeyDict(dict):
     def __setitem__(self, key, value):
         if value is None:
             self.pop(key.title(), None)
-        elif isinstance(value, unicode):
+        elif isinstance(value, six.text_type):
             return dict.__setitem__(self, key.title(), value.encode('utf-8'))
         else:
             return dict.__setitem__(self, key.title(), str(value))
@@ -332,7 +333,7 @@ def _resp_status_property():
             self.status_int = value
             self.explanation = self.title = RESPONSE_REASONS[value][0]
         else:
-            if isinstance(value, unicode):
+            if isinstance(value, six.text_type):
                 value = value.encode('utf-8')
             self.status_int = int(value.split(' ', 1)[0])
             self.explanation = self.title = value.split(' ', 1)[1]
@@ -357,7 +358,7 @@ def _resp_body_property():
         return self._body
 
     def setter(self, value):
-        if isinstance(value, unicode):
+        if isinstance(value, six.text_type):
             value = value.encode('utf-8')
         if isinstance(value, str):
             self.content_length = len(value)
@@ -753,7 +754,7 @@ def _req_environ_property(environ_field):
         return self.environ.get(environ_field, None)
 
     def setter(self, value):
-        if isinstance(value, unicode):
+        if isinstance(value, six.text_type):
             self.environ[environ_field] = value.encode('utf-8')
         else:
             self.environ[environ_field] = value
@@ -847,7 +848,7 @@ class Request(object):
         """
         headers = headers or {}
         environ = environ or {}
-        if isinstance(path, unicode):
+        if isinstance(path, six.text_type):
             path = path.encode('utf-8')
         parsed_path = urllib.parse.urlparse(path)
         server_name = 'localhost'

swift/common/utils.py

@@ -2760,7 +2760,7 @@ def get_valid_utf8_str(str_or_unicode):
 
     :param str_or_unicode: a string or an unicode which can be invalid utf-8
     """
-    if isinstance(str_or_unicode, unicode):
+    if isinstance(str_or_unicode, six.text_type):
         (str_or_unicode, _len) = utf8_encoder(str_or_unicode, 'replace')
     (valid_utf8_str, _len) = utf8_decoder(str_or_unicode, 'replace')
     return valid_utf8_str.encode('utf-8')

swift/container/backend.py

@@ -19,8 +19,9 @@ Pluggable Back-ends for Container Server
 import os
 from uuid import uuid4
 import time
-import six.moves.cPickle as pickle
 
+import six
+import six.moves.cPickle as pickle
 from six.moves import range
 import sqlite3
 
@@ -686,7 +687,7 @@ class ContainerBroker(DatabaseBroker):
         :param source: if defined, update incoming_sync with the source
         """
         for item in item_list:
-            if isinstance(item['name'], unicode):
+            if isinstance(item['name'], six.text_type):
                 item['name'] = item['name'].encode('utf-8')
 
         def _really_merge_items(conn):

swift/proxy/controllers/base.py

@@ -36,6 +36,7 @@ from urllib import quote
 
 from eventlet import sleep
 from eventlet.timeout import Timeout
+import six
 
 from swift.common.wsgi import make_pre_authed_env
 from swift.common.utils import Timestamp, config_true_value, \
@@ -469,7 +470,7 @@ def _get_info_cache(app, env, account, container=None):
         info = memcache.get(cache_key)
         if info:
             for key in info:
-                if isinstance(info[key], unicode):
+                if isinstance(info[key], six.text_type):
                     info[key] = info[key].encode("utf-8")
             env[env_key] = info
         return info

swift/proxy/server.py

@@ -23,6 +23,7 @@ import functools
 import sys
 
 from eventlet import Timeout
+import six
 
 from swift import __canonical_version__ as swift_version
 from swift.common import constraints
@@ -343,7 +344,7 @@ class Application(object):
             try:
                 controller, path_parts = self.get_controller(req)
                 p = req.path_info
-                if isinstance(p, unicode):
+                if isinstance(p, six.text_type):
                     p = p.encode('utf-8')
             except APIVersionError:
                 self.logger.increment('errors')

test/functional/tests.py

@@ -506,7 +506,7 @@ class TestContainer(Base):
 
     def testSlashInName(self):
         if Utils.create_name == Utils.create_utf8_name:
-            cont_name = list(unicode(Utils.create_name(), 'utf-8'))
+            cont_name = list(six.text_type(Utils.create_name(), 'utf-8'))
         else:
             cont_name = list(Utils.create_name())
 

test/unit/account/test_reaper.py

@@ -23,6 +23,7 @@ import unittest
 from logging import DEBUG
 from mock import patch, call, DEFAULT
 from contextlib import nested
+import six
 
 from swift.account import reaper
 from swift.account.backend import DATADIR
@@ -173,7 +174,7 @@ class TestReaper(unittest.TestCase):
             raise self.myexp
         objects = [{'name': 'o1'},
                    {'name': 'o2'},
-                   {'name': unicode('o3')},
+                   {'name': six.text_type('o3')},
                    {'name': ''}]
         return None, objects
 

test/unit/common/middleware/test_bulk.py

@@ -21,6 +21,7 @@ import tarfile
 import urllib
 import zlib
 import mock
+import six
 from six import BytesIO
 from shutil import rmtree
 from tempfile import mkdtemp
@@ -102,7 +103,7 @@ def build_dir_tree(start_path, tree_obj):
             dir_path = os.path.join(start_path, dir_name)
             os.mkdir(dir_path)
             build_dir_tree(dir_path, obj)
-    if isinstance(tree_obj, unicode):
+    if isinstance(tree_obj, six.text_type):
         tree_obj = tree_obj.encode('utf8')
     if isinstance(tree_obj, str):
         obj_path = os.path.join(start_path, tree_obj)
@@ -121,7 +122,7 @@ def build_tar_tree(tar, start_path, tree_obj, base_path=''):
             tar_info.type = tarfile.DIRTYPE
             tar.addfile(tar_info)
             build_tar_tree(tar, dir_path, obj, base_path=base_path)
-    if isinstance(tree_obj, unicode):
+    if isinstance(tree_obj, six.text_type):
         tree_obj = tree_obj.encode('utf8')
     if isinstance(tree_obj, str):
         obj_path = os.path.join(start_path, tree_obj)

test/unit/container/test_server.py

@@ -29,6 +29,7 @@ import random
 
 from eventlet import spawn, Timeout, listen
 import simplejson
+import six
 from six import BytesIO
 from six import StringIO
 
@@ -2090,11 +2091,11 @@ class TestContainerController(unittest.TestCase):
         container = dom.getElementsByTagName('container')[0]
         self.assertTrue(len(container.getElementsByTagName('subdir')) == 1)
         subdir = container.getElementsByTagName('subdir')[0]
-        self.assertEqual(unicode(subdir.attributes['name'].value),
+        self.assertEqual(six.text_type(subdir.attributes['name'].value),
                          u'<\'sub\' "dir">/')
         self.assertTrue(len(subdir.getElementsByTagName('name')) == 1)
         name = subdir.getElementsByTagName('name')[0]
-        self.assertEqual(unicode(name.childNodes[0].data),
+        self.assertEqual(six.text_type(name.childNodes[0].data),
                          u'<\'sub\' "dir">/')
 
     def test_GET_path(self):

test/unit/obj/test_expirer.py

@@ -22,6 +22,7 @@ from tempfile import mkdtemp
 from shutil import rmtree
 
 import mock
+import six
 
 from swift.common import internal_client, utils
 from swift.obj import expirer
@@ -153,10 +154,11 @@ class TestObjectExpirer(TestCase):
                     sum([len(self.containers[x]) for x in self.containers])
 
             def iter_containers(self, *a, **kw):
-                return [{'name': unicode(x)} for x in self.containers.keys()]
+                return [{'name': six.text_type(x)}
+                        for x in self.containers.keys()]
 
             def iter_objects(self, account, container):
-                return [{'name': unicode(x)}
+                return [{'name': six.text_type(x)}
                         for x in self.containers[container]]
 
             def delete_container(*a, **kw):
@@ -525,7 +527,7 @@ class TestObjectExpirer(TestCase):
         got_unicode = [False]
 
         def delete_actual_object_test_for_unicode(actual_obj, timestamp):
-            if isinstance(actual_obj, unicode):
+            if isinstance(actual_obj, six.text_type):
                 got_unicode[0] = True
 
         fake_swift = InternalClient(