From caebf387b4df64775539d6722a4678cea00518fc Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 10 Jul 2019 09:03:31 -0400 Subject: [PATCH] Translate gitea project creation to python Sadly, as readable as the use of the uri module to do the interactions with gitea is, more reent ansible changed how subprocesses are forked and this makes iterating over all the projects in projects.yaml take an incredibly long amount of time. Instead of doing it in yaml, make a python module that takes the list one time and does looping and requests calls. This should make it be possible to run the actual gitea creation playbook in integration tests. Change-Id: Ifff3291c1092e6df09ae339c9e7dddb5ee692685 --- playbooks/group_vars/gitea.yaml | 1 + .../roles/gitea-git-repos/library/__init__.py | 0 .../library/gitea_create_repos.py | 177 ++++++++++++++++++ .../roles/gitea-git-repos/tasks/main.yaml | 50 +---- .../gitea-git-repos/tasks/setup-org.yaml | 56 ------ .../gitea-git-repos/tasks/setup-repo.yaml | 98 ---------- playbooks/roles/gitea/tasks/main.yaml | 5 + playbooks/sync-gitea-projects.yaml | 2 - 8 files changed, 190 insertions(+), 199 deletions(-) create mode 100644 playbooks/roles/gitea-git-repos/library/__init__.py create mode 100755 playbooks/roles/gitea-git-repos/library/gitea_create_repos.py delete mode 100644 playbooks/roles/gitea-git-repos/tasks/setup-org.yaml delete mode 100644 playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml diff --git a/playbooks/group_vars/gitea.yaml b/playbooks/group_vars/gitea.yaml index 1934a54338..19eedab34a 100644 --- a/playbooks/group_vars/gitea.yaml +++ b/playbooks/group_vars/gitea.yaml @@ -1,3 +1,4 @@ +ansible_python_interpreter: python3 gitea_root_email: infra-root@openstack.org gitea_gerrit_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVuhTMAz1H2Jr9AC3py9A0vlNna6Sdt4yrvZOayxukPqQ7GPZd+Mo7MVyypxLD479N2mA09JAdsbq1eTiPP8ksEkB+dNxZzw8mY1653R/IXSW6J9xPcoDa88HF2s/xHN24IWzgiDjNNe79AQ+sKleByEQZ++xXny3MRpy258hKUvAtjjOLOnM1PBs8JNOzBL+UPgWRgSX6GG0qywJZqjD1Qx5kvH9RTRLi+tcMhEi4laN7BYvn4csY0sYzTzPG4ZTu3ootIJoRlQGtQ0LmoFO1vSwyEJUags6/ZZGjgy3jl3kwcU/b8ZnFlF4MDw1OB1QqMb4r6bMHbXNIupp4zJbz gerrit-replication-2014-04-25 iptables_extra_public_tcp_ports: diff --git a/playbooks/roles/gitea-git-repos/library/__init__.py b/playbooks/roles/gitea-git-repos/library/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py new file mode 100755 index 0000000000..c846ab9887 --- /dev/null +++ b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time +import requests +import urllib.parse + +from ansible.module_utils.basic import AnsibleModule + +SB_REPO = 'https://storyboard.openstack.org/#!/project/{org}/{repo}' +SB_FORMAT = 'https://storyboard.openstack.org/#!/story/{{index}}' +LP_REPO = 'https://bugs.launchpad.net/{repo}' +LP_FORMAT = 'https://bugs.launchpad.net/{repo}/+bug/{{index}}' + + + +class Gitea(object): + + def __init__(self, url, password, always_update, projects): + self.url = url + self.password = password + self.always_update = always_update + self.projects = projects + self.orgs = { f['project'].split('/')[0] for f in self.projects } + + def request(self, method, endpoint, *args, **kwargs): + resp = requests.request( + method, + urllib.parse.urljoin(self.url, endpoint), + auth=('root', self.password), + verify=False, + *args, **kwargs) + resp.raise_for_status() + return resp + + def get(self, endpoint, *args, **kwargs): + return self.request('GET', endpoint, *args, **kwargs) + + def post(self, endpoint, *args, **kwargs): + return self.request('POST', endpoint, *args, **kwargs) + + def put(self, endpoint, *args, **kwargs): + return self.request('PUT', endpoint, *args, **kwargs) + + def get_gitea_orgs(self): + orgs = self.get("/api/v1/user/orgs").json() + return [f['username'] for f in orgs] + + def make_gitea_org(self, org): + self.post( + '/api/v1/admin/users/root/orgs', + json=dict(username=org)) + + def ensure_gitea_teams(self, org): + team_list = self.get('/api/v1/orgs/{org}/teams'.format(org=org)).json() + owner_id = [f['id'] for f in team_list if f['name'] == 'Owners'][0] + + org_owners = self.get( + '/api/v1/teams/{owner_id}/members'.format(owner_id=owner_id)) + if 'gerrit' not in [f['username'] for f in org_owners.json()]: + self.put('/api/v1/teams/{owner_id}/members/gerrit'.format( + owner_id=owner_id)) + + def get_org_repo_list(self, org): + return self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json() + + def get_csrf_token(self): + resp = self.get('/') + return urllib.parse.unquote(resp.cookies.get('_csrf')) + + def make_gitea_project(self, project, csrf_token): + org, repo = project['project'].split('/', 1) + resp = self.post( + '/api/v1/org/{org}/repos'.format(org=org), + json=dict( + auto_init=True, + description=project.get('description', '')[:255], + name=repo, + private=False, + readme='Default')) + if project.get('use-storyboard'): + external_tracker_url = SB_REPO.format(org=org, repo=repo) + tracker_url_format = SB_FORMAT + elif project.get('groups'): + external_tracker_url = LP_REPO.format(repo=project['groups'][0]) + tracker_url_format = LP_FORMAT.format(repo=project['groups'][0]) + else: + external_tracker_url = LP_REPO.format(repo=repo) + tracker_url_format = LP_FORMAT.format(repo=repo) + + self.post( + '/{org}/{repo}/settings'.format(org=org, repo=repo), + data=dict( + _csrf=csrf_token, + action='advanced', + # enable_pulls is not provided, which disables it + # enable_wiki is not provided, which disables it + enable_external_wiki=False, + external_wiki_url='', + # enable_issues is on so that issue links work + enable_issues='on', + enable_external_tracker=True, + external_tracker_url=external_tracker_url, + tracker_url_format=tracker_url_format, + tracker_issue_style='numeric', + )) + + for count in range(0, 5): + try: + return self.post( + '/{org}/{repo}/settings/branches'.format( + org=org, repo=repo), + data=dict( + _csrf=csrf_token, + action='default_branch', + branch='master', + )) + except requests.exceptions.HTTPError as e: + time.sleep(3) + raise Exception("Could not update branch settings") + + def run(self): + gitea_orgs = self.get_gitea_orgs() + gitea_repos = [] + for org in self.orgs: + if org not in gitea_orgs: + self.make_gitea_org(org) + self.ensure_gitea_teams(org) + gitea_repos.extend(self.get_org_repo_list(org)) + csrf_token = self.get_csrf_token() + + for project in self.projects: + if project['project'] not in gitea_repos or self.always_update: + self.make_gitea_project(project, csrf_token) + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + url=dict(required=True), + password=dict(required=True), + projects=dict(required=True, type='list'), + always_update=dict(type='bool', default=True), + ) + ) + + p = module.params + + gitea = Gitea( + url=p.get('url'), + password=p.get('password'), + always_update=p.get('always_update'), + projects=p.get('projects'), + ) + try: + gitea.run() + except Exception as e: + module.fail_json(msg=str(e), changed=True) + + module.exit_json(changed=True) + + +if __name__ == '__main__': + ansible_main() diff --git a/playbooks/roles/gitea-git-repos/tasks/main.yaml b/playbooks/roles/gitea-git-repos/tasks/main.yaml index d356e9c8a8..e4f3f5c5d9 100644 --- a/playbooks/roles/gitea-git-repos/tasks/main.yaml +++ b/playbooks/roles/gitea-git-repos/tasks/main.yaml @@ -1,44 +1,8 @@ -- name: Get Gerrit project list - set_fact: - gerrit_projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}" -- name: Parse Gerrit org list - set_fact: - gerrit_orgs: "{{ gerrit_projects | map(attribute='project') | map('regex_search', '^(.*?)/') | list | unique | select | map('regex_replace', '/', '') | list }}" -- name: debug - debug: - msg: "{{ gerrit_orgs }}" -- name: Get Gitea org list - # We assume that all the orgs we are interested in are owned by root - uri: - url: "{{ gitea_url }}/api/v1/user/orgs" - user: root +- name: Create Gitea Repos and Org + gitea_create_repos: + url: "{{ gitea_url }}" password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_list -- name: Parse Gitea org list - set_fact: - gitea_orgs: "{{ gitea_org_list.json | map(attribute='username') | list }}" -- name: Create orgs - loop: "{{ gerrit_orgs }}" - loop_control: - loop_var: org - include_tasks: 'setup-org.yaml' -- name: Get a CSRF token - uri: - url: "{{ gitea_url }}/" - validate_certs: false - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - register: gitea_token -- name: Parse CSRF taken - set_fact: - gitea_token: "{{ gitea_token.cookies._csrf|regex_replace('%3D','=') }}" -- name: Create repos - loop: "{{ gerrit_projects }}" - loop_control: - loop_var: project - include_tasks: 'setup-repo.yaml' - when: gitea_always_update or project.project not in gitea_repos + always_update: "{{ gitea_always_update }}" + # Lookup runs locally on the calling machine, so doesn't need + # /opt/project-config remotely + projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}" diff --git a/playbooks/roles/gitea-git-repos/tasks/setup-org.yaml b/playbooks/roles/gitea-git-repos/tasks/setup-org.yaml deleted file mode 100644 index a4938842e1..0000000000 --- a/playbooks/roles/gitea-git-repos/tasks/setup-org.yaml +++ /dev/null @@ -1,56 +0,0 @@ -- name: Process org - debug: - msg: "Processing org {{ org }}" -- name: Create org - when: org not in gitea_orgs - uri: - url: "{{ gitea_url }}/api/v1/admin/users/root/orgs" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 201 - method: POST - body_format: json - body: - username: "{{ org }}" -- name: Get org team list - uri: - url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/teams" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_team_list -- name: Get org owners - uri: - url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_members -- name: Add Gerrit user to org - when: "'gerrit' not in gitea_org_members.json | map(attribute='username')" - uri: - url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members/gerrit" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 204 - method: PUT -- name: Get org repo list - uri: - url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/repos" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 200 - register: gitea_org_repo_list -- name: Parse org repo list - set_fact: - gitea_repos: "{{ gitea_org_repo_list.json | map(attribute='full_name') | list + gitea_repos | default([]) }}" diff --git a/playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml b/playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml deleted file mode 100644 index b13ca74e80..0000000000 --- a/playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml +++ /dev/null @@ -1,98 +0,0 @@ -- name: debug - debug: - msg: "{{ project }}" -- name: Parse project name - set_fact: - org: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\1') }}" - repo: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\2') }}" -- name: Create repo - when: project.project not in gitea_repos - uri: - url: "{{ gitea_url }}/api/v1/org/{{ org }}/repos" - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - validate_certs: false - status_code: 201 - method: POST - body_format: json - body: - auto_init: true - description: "{{ (project.description | default(''))[:255] }}" - name: "{{ repo }}" - private: false - readme: Default - register: create_repo_result - -- name: Set storyboard tracker url - when: "'use-storyboard' in project and project['use-storyboard']" - set_fact: - external_tracker_url: "https://storyboard.openstack.org/#!/project/{{ org }}/{{ repo }}" -- name: Set storyboard tracker url format - when: "'use-storyboard' in project and project['use-storyboard']" - set_fact: - tracker_url_format: "https://storyboard.openstack.org/#!/story/{index}" -- name: Set launchpad tracker url - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])" - set_fact: - external_tracker_url: "https://bugs.launchpad.net/{{ repo }}" -- name: Set launchpad tracker url format - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])" - set_fact: - tracker_url_format: "https://bugs.launchpad.net/{{ repo }}/+bug/{index}" -- name: Set launchpad tracker url if group set - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])" - set_fact: - external_tracker_url: "https://bugs.launchpad.net/{{ project.groups[0] }}" -- name: Set launchpad tracker url format if group set - when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])" - set_fact: - tracker_url_format: "https://bugs.launchpad.net/{{ project.groups[0] }}/+bug/{index}" - -- name: Adjust repo settings - when: gitea_always_update or project.project not in gitea_repos - register: result - retries: 3 - until: result is succeeded - delay: 5 - uri: - url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings" - validate_certs: false - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - status_code: 302 - method: POST - body_format: form-urlencoded - body: - _csrf: "{{ gitea_token }}" - action: advanced - # enable_pulls is not provided, which disables it - # enable_wiki is not provided, which disables it - enable_external_wiki: false - external_wiki_url: - # enable_issues is on so that issue links work - enable_issues: on - enable_external_tracker: true - external_tracker_url: "{{ external_tracker_url }}" - tracker_url_format: "{{ tracker_url_format }}" - tracker_issue_style: numeric -- name: Set default branch - when: gitea_always_update or project.project not in gitea_repos - register: result - retries: 3 - until: result is succeeded - delay: 5 - uri: - url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings/branches" - validate_certs: false - user: root - password: "{{ gitea_root_password }}" - force_basic_auth: true - status_code: 302 - method: POST - body_format: form-urlencoded - body: - _csrf: "{{ gitea_token }}" - action: default_branch - branch: master diff --git a/playbooks/roles/gitea/tasks/main.yaml b/playbooks/roles/gitea/tasks/main.yaml index 8a1aba61b6..7e51a49a9a 100644 --- a/playbooks/roles/gitea/tasks/main.yaml +++ b/playbooks/roles/gitea/tasks/main.yaml @@ -32,6 +32,11 @@ template: src: app.ini.j2 dest: /var/gitea/conf/app.ini +- name: Install requests + package: + name: + - python3-requests + state: present - name: Install docker-compose package: name: diff --git a/playbooks/sync-gitea-projects.yaml b/playbooks/sync-gitea-projects.yaml index bd25a86fb0..b12af45b08 100644 --- a/playbooks/sync-gitea-projects.yaml +++ b/playbooks/sync-gitea-projects.yaml @@ -8,7 +8,6 @@ repo: https://git.openstack.org/openstack-infra/project-config dest: /opt/project-config force: yes - register: gitinfo - hosts: "gitea:!disabled" name: "Create repos on gitea servers" @@ -16,5 +15,4 @@ max_fail_percentage: 1 roles: - role: gitea-git-repos - project_config_ref: "{{ hostvars.localhost.gitinfo.after }}" gitea_always_update: true