diff --git a/tools/opendev-migrate b/tools/opendev-migrate new file mode 100644 index 0000000000..de1472af57 --- /dev/null +++ b/tools/opendev-migrate @@ -0,0 +1,128 @@ +#!/usr/bin/python3 + +# Copyright (c) 2019 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. + +import csv +import io +import json +import requests +import yaml + +# this will hold our mapping of old names(paces) to new +moves = {} + +# here's a list of all (non-meta)projects in gerrit +repos = [r for r in json.loads(requests.get( + 'http://review.openstack.org/projects/').text[5:]).keys() if '/' in r] + +# a map of the first pair of columns from the namespace request ethercalc +overrides = dict([r[:2] for r in csv.reader(io.StringIO( + requests.get('https://ethercalc.openstack.org/opendev-transition.csv').text + )) if '/' in r[1]]) + +# all projects which are officially governed by openstack or osf +openstack = [] +o_gov = 'https://opendev.org/openstack/governance/raw/branch/master/reference/' +data = yaml.safe_load(requests.get(o_gov + 'projects.yaml').text) +for team in data.values(): + for deli in team['deliverables'].values(): + for repo in deli['repos']: + openstack.append(repo) +for f in ('foundation-board-repos.yaml', 'sigs-repos.yaml', + 'technical-committee-repos.yaml', 'user-committee-repos.yaml'): + data = yaml.safe_load(requests.get(o_gov + f).text) + for team in data.values(): + for repo in team: + openstack.append(repo['repo']) + +# projects which were at one time officially governed by openstack +openstack_legacy = [] +data = yaml.safe_load(requests.get(o_gov + 'legacy.yaml').text) +for team in data.values(): + for deli in team['deliverables'].values(): + for repo in deli['repos']: + openstack_legacy.append(repo) + +# use the jeepyb config to identify whitelabeled oip git projects +airship = [] +starlingx = [] +zuul = [] +data = yaml.safe_load(requests.get( + 'https://opendev.org/openstack-infra/project-config/raw/branch/master/' + 'gerrit/projects.yaml').text) +for project in data: + if 'cgit-alias' in project: + if project['cgit-alias']['site'] == 'git.airshipit.org': + airship.append(project['project']) + elif project['cgit-alias']['site'] == 'git.starlingx.io': + starlingx.append(project['project']) + elif project['cgit-alias']['site'] == 'git.zuul-ci.org': + zuul.append(project['project']) + +for repo in repos: + # apply the requested namespace overrides first + if repo in overrides: + moves[repo] = overrides[repo] + + # airship repos identified drop the airship- prefix and move to airship + elif repo in airship: + moves[repo] = 'airship/' + repo.split('/')[1].replace('airship-', '') + + # starlingx repos drop the stx- prefix and move to starlingx + elif repo in starlingx: + moves[repo] = 'starlingx/' + repo.split('/')[1].replace('stx-', '') + + # all current openstack repos move to openstack + elif repo in openstack: + moves[repo] = 'openstack/' + repo.split('/')[1] + + # zuul repos move to zuul + elif repo in zuul: + moves[repo] = 'zuul/' + repo.split('/')[1] + + # former openstack repositories which aren't accounted for go in openstack + elif repo in openstack_legacy: + moves[repo] = 'openstack/' + repo.split('/')[1] + + # unofficial repositories move from openstack to x + elif repo.startswith('openstack/'): + moves[repo] = 'x/' + repo.split('/')[1] + + # everything else is unchanged + else: + moves[repo] = repo + +# we'll use this data structure for the rename_repos playbook input +output = {'repos': []} + +for mapping in moves.items(): + if mapping[0] != mapping[1]: + # convenient stdout feedback is for sharing with people + print('%s -> %s' % mapping) + + # update the rename_repos data structure + output['repos'].append({'old': mapping[0], 'new': mapping[1]}) + +# https://docs.openstack.org/infra/system-config/gerrit.html#renaming-a-project +with open('repos.yaml', 'w') as outfile: + yaml.dump(output, outfile) + +# We should add this to the rename playbook, but time is short +with open('zuul-rename.sh', 'w') as outfile: + keyroot = '/var/lib/zuul/keys' + for d in output['repos']: + outfile.write('mv %s/ssh/project/gerrit/%s %s/ssh/project/gerrit/%s\n' % + (keyroot, d['old'], keyroot, d['new'])) + outfile.write('mv %s/secrets/project/gerrit/%s %s/secrets/project/gerrit/%s\n' % + (keyroot, d['old'], keyroot, d['new'])) diff --git a/tools/opendev-patching b/tools/opendev-patching new file mode 100644 index 0000000000..50efd15724 --- /dev/null +++ b/tools/opendev-patching @@ -0,0 +1,219 @@ +#!/usr/bin/python3 + +# Copyright (c) 2019 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. + +import os +import re +import shutil +import subprocess +import sys +import tempfile + +import yaml + + +def run(commandlist): + """Wrapper to run a shell command and return a list of stdout lines.""" + (o, x) = subprocess.Popen( + commandlist, env=gitenv, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL).communicate() + return o.decode('utf-8').strip().split('\n') + + +class EncryptedPKCS1_OAEP(yaml.YAMLObject): + """Causes pyyaml to skip custom YAML tags Zuul groks.""" + yaml_tag = u'!encrypted/pkcs1-oaep' + yaml_loader = yaml.SafeLoader + + def __init__(self, x): + pass + + @classmethod + def from_yaml(cls, loader, node): + return cls(node.value) + + +# the gerrit git directory +top = sys.argv[1] + +# the repo renames file and a corresponding regex for finding them +renames = {} +for repo in yaml.safe_load(open(sys.argv[2]))['repos']: + renames[repo['old']] = repo['new'] +renames_regex = re.compile( + '([^a-z0-9_-]|^)(%s)([^a-z0-9_-]|$)' % '|'.join(renames.keys())) + +# our custom git author/committer used by the run function +gitenv = dict(os.environ) +gitenv.update({ + 'GIT_AUTHOR_NAME': 'OpenDev Sysadmins', + 'GIT_AUTHOR_EMAIL': 'openstack-infra@lists.openstack.org', + 'GIT_COMMITTER_NAME': 'OpenDev Sysadmins', + 'GIT_COMMITTER_EMAIL': 'openstack-infra@lists.openstack.org', + }) + +# commit message string for generated commits +commit_message = """\ +OpenDev Migration Patch + +This commit was bulk generated and pushed by the OpenDev sysadmins +as a part of the Git hosting and code review systems migration +detailed in these mailing list posts: + +http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html +http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html + +Attempts have been made to correct repository namespaces and +hostnames based on simple pattern matching, but it's possible some +were updated incorrectly or missed entirely. Please reach out to us +via the contact information listed at https://opendev.org/ with any +questions you may have. +""" + +# find all second-level directories on which we will operate +repos = run(['find', top, '-maxdepth', '2', '-mindepth', '2', '-name', '*.git', '-type', 'd']) + +# iterate over each repo +for bare in repos: + # clone the repo into a temporary working tree + with tempfile.TemporaryDirectory() as repodir: + run(['git', 'clone', bare, repodir]) + origdir = os.getcwd() + os.chdir(repodir) + + # build a list of branches for this repo + branches = [] + branchdump = run(['git', 'branch', '-a']) + + # iterate over each branch + for line in branchdump: + branch = re.match('^remotes/origin/([^ ]+)$', line.strip()) + if branch: + branches.append(branch.group(1)) + for branch in branches: + run(['git', 'checkout', '-B', branch, 'origin/' + branch]) + + # build up a list of files to edit + editfiles = set() + + # find zuul configs and add ansible playbooks they reference + zuulfiles = run([ + 'find', '.zuul.d/', 'zuul.d/', '.zuul.yaml', 'zuul.yaml', + '-name', '*.yaml', '-type', 'f']) + for zuulfile in zuulfiles: + if zuulfile: + conf = yaml.safe_load(open(zuulfile)) + if not conf: + # some repos have empty zuul configs + continue + for node in conf: + if 'job' in node: + for subnode in ('post-run', 'pre-run', 'run'): + if subnode in node['job']: + if type(node['job'][subnode]) is list: + editfiles.update(node['job'][subnode]) + else: + editfiles.add(node['job'][subnode]) + + # if there are roles dirs relative to the playbooks, add them too + for playbook in list(editfiles): + rolesdir = os.path.join(os.path.dirname(playbook), 'roles') + if os.path.isdir(rolesdir): + editfiles.update(run([ + 'find', rolesdir, '-type', 'f', '(', '-name', '*.j2', + '-o', '-name', '*.yaml', '-o', '-name', '*.yml', ')'])) + + # zuul looks at the top level roles dir too + editfiles.update(run([ + 'find', 'roles', '-type', 'f', '(', '-name', '*.j2', '-o', + '-name', '*.yaml', '-o', '-name', '*.yml', ')'])) + + # and add the zuul configs themselves + editfiles.update(zuulfiles) + + # and add .gitreview of course + editfiles.add('.gitreview') + + # and zuul/main.yaml so we catch the tenant config + editfiles.add('zuul/main.yaml') + + # and gerrit/projects.yaml for manage-projects + editfiles.add('gerrit/projects.yaml') + + # and gerritbot/channels.yaml for gerritbot + editfiles.add('gerritbot/channels.yaml') + + # drop any empty filename we ended up with + editfiles.discard('') + + # read through each file and replace specific patterns + for fname in editfiles: + if not os.path.exists(fname): + continue + with open(fname) as rfd, tempfile.NamedTemporaryFile() as wfd: + # track modifications for efficiency + modified = False + for line in rfd: + # apply renames from the mapping + found = renames_regex.search(line) + while found: + line = line.replace( + found.group(2), renames[found.group(2)]) + modified = True + found = renames_regex.search(line) + + # same for git.openstack.org -> opendev.org + found = re.search("git\.openstack\.org", line) + while found: + line = line.replace( + "git.openstack.org", "opendev.org") + modified = True + found = renames_regex.search(line) + + # and review.openstack.org -> review.opendev.org + found = re.search("review\.openstack\.org", line) + while found: + line = line.replace( + "review.openstack.org", "review.opendev.org") + modified = True + found = renames_regex.search(line) + + wfd.write(line.encode('utf-8')) + + # copy any modified file back into the worktree + if modified: + wfd.flush() + shutil.copyfile(wfd.name, fname) + modified = False + + # special logic to rename Gerrit ACL files + if bare.endswith('/project-config.git'): + for acl in run(['git', 'ls-files', 'gerrit/acls/']): + found = renames_regex.search(acl) + if found: + newpath = acl.replace( + found.group(2), renames[found.group(2)]) + os.makedirs(os.path.dirname(newpath), exist_ok=True) + run(['git', 'mv', acl, newpath]) + + # commit and push our changes, if there are any + if run(['git', 'diff']): + with tempfile.NamedTemporaryFile() as message: + message.write(commit_message.encode('utf-8')) + message.flush() + run(['git', 'commit', '-a', '-F', message.name]) + run(['git', 'push', 'origin', 'HEAD']) + + # switch back before the context manager deletes our cwd + os.chdir(origdir)