From 362bf845863a7e6e8b0cb50135de175dbf5bb9c5 Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Wed, 6 Jul 2016 13:20:14 -0500 Subject: [PATCH] osa-differ: Refactor and filtered reports This patch adds filtered report support to the osa-differ script and refactors some code that was repeated more than once. A user can provide ``--projects-only`` on the command line to see only the commits that were made to OpenStack projects that OSA builds. The user can also use ``--roles-only`` to limit the report only to the changes in OSA roles. All commits are rendered through the same jinja template, which should make future cosmetic updates much easier. Each section for a project or role now includes links back to the repository itself and links to each commit. The script now also shows the commits on OSA itself. That seems fairly important and it should have been there earlier. Change-Id: I139f0f1bab3d4afbc5004dc52756800e426aa605 --- osa-differ/osa-differ.py | 178 ++++++++++++++---------- osa-differ/templates/header.j2 | 18 +++ osa-differ/templates/project_details.j2 | 17 --- osa-differ/templates/repo_details.j2 | 18 +++ osa-differ/templates/role_details.j2 | 17 --- 5 files changed, 142 insertions(+), 106 deletions(-) create mode 100644 osa-differ/templates/header.j2 delete mode 100644 osa-differ/templates/project_details.j2 create mode 100644 osa-differ/templates/repo_details.j2 delete mode 100644 osa-differ/templates/role_details.j2 diff --git a/osa-differ/osa-differ.py b/osa-differ/osa-differ.py index 34f7fe43..301cce46 100755 --- a/osa-differ/osa-differ.py +++ b/osa-differ/osa-differ.py @@ -42,14 +42,21 @@ def get_arguments(): nargs=1, help="Git SHA of the newer commit", ) + display_opts = parser.add_mutually_exclusive_group() + display_opts.add_argument( + "--projects-only", + action="store_true", + help="Only display commits for OpenStack projects" + ) + display_opts.add_argument( + "--roles-only", + action="store_true", + help="Only display commits for OpenStack-Ansible roles" + ) + return vars(parser.parse_args()) -def short_commit(commit_sha): - """Return a short commit hash string.""" - return commit_sha[0:8] - - def get_commit_data(commit_sha): """Get information about a commit from Github.""" commit_data = gh.repos.commits.get(user='openstack', @@ -70,22 +77,71 @@ def get_project_names(project_dict): return [x[:-9] for x in project_dict if x.endswith('git_repo')] -def get_projects(yaml_urls): +def get_projects(base_url, commit): """Get all projects from multiple YAML files.""" + # Assemble the full URLs to our YAML files that contain our OpenStack + # projects' details. + repo_files = [ + 'playbooks/defaults/repo_packages/openstack_services.yml', + 'playbooks/defaults/repo_packages/openstack_other.yml' + ] + yaml_urls = [base_url.format(commit, x) for x in repo_files] + + # Loop through both YAML files and merge the data into one dictionary. yaml_parsed = [] for yaml_url in yaml_urls: r = requests.get(yaml_url) yaml_parsed.append(yaml.load(r.text)) merged_dicts = {k: v for d in yaml_parsed for k, v in d.items()} + return merged_dicts +def render_commit_template(user, repo, old_commit, new_commit, extra_vars={}, + template_file='repo_details.j2'): + """Render a template to generate RST content for commits.""" + global gh + global jinja_env + + # Compare the two commits in the project's repository to see what + # the differences are between them. + comparison = gh.repos.commits.compare( + user=user, + repo=repo, + base=old_commit, + head=new_commit + ) + + # Render the jinja2 template + rendered_template = jinja_env.get_template(template_file).render( + repo=repo, + commits=comparison.commits, + latest_sha=short_commit(new_commit), + older_sha=short_commit(old_commit), + extra_vars=extra_vars + ) + + return rendered_template + + +def short_commit(commit_sha): + """Return a short commit hash string.""" + return commit_sha[0:8] + if __name__ == "__main__": # Set up some initial variables gh = pygithub3.Github() args = get_arguments() - report = '' + + # Load our Jinja templates + TEMPLATE_DIR = "{0}/templates".format( + os.path.dirname(os.path.abspath(__file__)) + ) + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(TEMPLATE_DIR), + trim_blocks=True + ) # Store our arguments into variables to make things easier. old_commit = args['old_commit'][0] @@ -105,54 +161,43 @@ if __name__ == "__main__": print("The new commit SHA was not found: {0}".format(new_commit)) sys.exit(1) - report_header = """ -OpenStack-Ansible Release Diff Generator ----------------------------------------- - -Showing changes between ``{0} {1}`` and ``{2} {3}`` in OpenStack-Ansible. - ----- -""".format( - short_commit(old_commit), - old_commit_message, - short_commit(new_commit), - new_commit_message + # Generate header and initial report for OpenStack-Ansible itself + report = render_commit_template( + user='openstack', + repo='openstack-ansible', + old_commit=old_commit, + new_commit=new_commit, + template_file='header.j2', + extra_vars={ + 'roles_only': args['roles_only'], + 'projects_only': args['projects_only'] + } ) - report += report_header + + # Add a horizontal line to report after the OpenStack-Ansible commits. + report += "----\n" # Set up the base url that allows us to retrieve data from # OpenStack-Ansible at a particular commit. base_url = 'https://raw.githubusercontent.com/openstack/' \ 'openstack-ansible/{0}/{1}' - # Get all of the OpenStack projects that OpenStack-Ansible builds - repo_files = [ - 'playbooks/defaults/repo_packages/openstack_services.yml', - 'playbooks/defaults/repo_packages/openstack_other.yml' - ] - old_commit_yaml_urls = [base_url.format(old_commit, x) for x in repo_files] - old_commit_projects = get_projects(old_commit_yaml_urls) - new_commit_yaml_urls = [base_url.format(new_commit, x) for x in repo_files] - new_commit_projects = get_projects(new_commit_yaml_urls) + if args['roles_only']: + # Short circuit here and don't get any projects since the user only + # wants to see role commits. + old_commit_projects = [] + new_commit_projects = [] + else: + report += "\nOpenStack Projects\n------------------\n" + + # Get all of the OpenStack projects that OpenStack-Ansible builds + old_commit_projects = get_projects(base_url, old_commit) + new_commit_projects = get_projects(base_url, new_commit) # Get the bare project names from the YAML data we retrieved old_commit_project_names = get_project_names(old_commit_projects) new_commit_project_names = get_project_names(new_commit_projects) - # Load our Jinja templates - TEMPLATE_DIR = "{0}/templates".format( - os.path.dirname(os.path.abspath(__file__)) - ) - jinja_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(TEMPLATE_DIR), - trim_blocks=True - ) - - report += """ -OpenStack Projects -~~~~~~~~~~~~~~~~~~ -""" - # Loop through each OpenStack project found in the latest commit for project in sorted(new_commit_project_names): @@ -172,34 +217,29 @@ OpenStack Projects except: continue - # Compare the two commits in the project's repository to see what - # the differences are between them. - comparison = gh.repos.commits.compare( + # Render a template showing the commits in this project's repository. + report += render_commit_template( user=user, repo=project_repo_name, - base=older_sha, - head=latest_sha - ) - - report += jinja_env.get_template('project_details.j2').render( - project=project, - commits=comparison.commits, - latest_sha=short_commit(latest_sha), - older_sha=short_commit(older_sha) + old_commit=older_sha, + new_commit=latest_sha ) # Set up the URLs for the old and new ansible-role-requirements.yml old_role_url = base_url.format(old_commit, 'ansible-role-requirements.yml') new_role_url = base_url.format(new_commit, 'ansible-role-requirements.yml') - # Retrieve the roles YAML - old_role_yaml = yaml.load(requests.get(old_role_url).text) - new_role_yaml = yaml.load(requests.get(new_role_url).text) + if args['projects_only']: + # Short circuit here and don't get any roles since the user only wants + # to see OpenStack project commits. + old_role_yaml = {} + new_role_yaml = {} + else: + report += "\nOpenStack-Ansible Roles\n-----------------------\n" - report += """ -OpenStack-Ansible Roles -~~~~~~~~~~~~~~~~~~~~~~~ -""" + # Retrieve the roles YAML + old_role_yaml = yaml.load(requests.get(old_role_url).text) + new_role_yaml = yaml.load(requests.get(new_role_url).text) # Loop through each OpenStack-Ansible role found in the latest commit for role in new_role_yaml: @@ -212,18 +252,12 @@ OpenStack-Ansible Roles older_sha = next(x['version'] for x in old_role_yaml if x['name'] == role['name']) - comparison = gh.repos.commits.compare( + # Render a template showing the commits in this role's repository. + report += render_commit_template( user=user, repo=role_repo_name, - base=older_sha, - head=latest_sha - ) - - report += jinja_env.get_template('role_details.j2').render( - role=role_repo_name, - commits=comparison.commits, - latest_sha=short_commit(latest_sha), - older_sha=short_commit(older_sha) + old_commit=older_sha, + new_commit=latest_sha ) print(report) diff --git a/osa-differ/templates/header.j2 b/osa-differ/templates/header.j2 new file mode 100644 index 00000000..d7d060b9 --- /dev/null +++ b/osa-differ/templates/header.j2 @@ -0,0 +1,18 @@ +OpenStack-Ansible Diff Generator +================================ + +This report shows changes from ``{{ older_sha }}`` to ``{{ latest_sha }}`` in +OpenStack-Ansible. + +{% if extra_vars['roles_only'] %} +This report also includes the changes that were made in OpenStack-Ansible roles +between these two OpenStack commits. +{% elif extra_vars['projects_only'] %} +This report also includes the changes that were made in OpenStack projects +between thse two OpenStack-Ansible commits. +{% else %} +This report also includes the changes that were made in OpenStack projects and +OpenStack-Ansible roles between these two OpenStack-Ansible commits. +{% endif %} + +{% include 'repo_details.j2' %} diff --git a/osa-differ/templates/project_details.j2 b/osa-differ/templates/project_details.j2 deleted file mode 100644 index d4b4786b..00000000 --- a/osa-differ/templates/project_details.j2 +++ /dev/null @@ -1,17 +0,0 @@ - -**{{ project }}** -{% if commits | length < 1 %} - No commits were found in the project between the OSA commits provided. -{% elif commits | length == 1 %} - 1 commit was found in the project (``{{ older_sha }}`` › ``{{ latest_sha }}``): - -:: -{% elif commits | length > 1 %} - {{ commits | length }} commits were found in the project (``{{ older_sha }}`` › ``{{ latest_sha }}``): - -:: -{% endif %} - -{% for commit in commits if not commit.commit['message'].split('\n')[0][0:7] == 'Merge "' %} -{{ commit.sha[0:8] | indent(4, true)}} {{ commit.commit['message'].split('\n')[0] }} -{% endfor %} diff --git a/osa-differ/templates/repo_details.j2 b/osa-differ/templates/repo_details.j2 new file mode 100644 index 00000000..d1a93447 --- /dev/null +++ b/osa-differ/templates/repo_details.j2 @@ -0,0 +1,18 @@ + +{{ repo }} +{{ '~' * repo | length}} + +{% if commits | length < 1 %} +No commits were found in `{{ repo }} `_ +between the OSA commits provided. +{% elif commits | length == 1 %} +1 commit was found in `{{ repo }} `_ +from ``{{ older_sha }}`` to ``{{ latest_sha }}``: +{% elif commits | length > 1 %} +{{ commits | length }} commits were found in `{{ repo }} `_ +from ``{{ older_sha }}`` to ``{{ latest_sha }}``: +{% endif %} + +{% for commit in commits if not commit.commit['message'].split('\n')[0][0:7] == 'Merge "' %} +* {{ commit.commit['message'].split('\n')[0] }} [`{{ commit.sha[0:8] }} `_] +{% endfor %} diff --git a/osa-differ/templates/role_details.j2 b/osa-differ/templates/role_details.j2 deleted file mode 100644 index 9963d0c6..00000000 --- a/osa-differ/templates/role_details.j2 +++ /dev/null @@ -1,17 +0,0 @@ - -**{{ role }}** -{% if commits | length < 1 %} - No commits were found in the role between the OSA commits provided. -{% elif commits | length == 1 %} - 1 commit was found in the role (``{{ older_sha }}`` › ``{{ latest_sha }}``): - -:: -{% elif commits | length > 1 %} - {{ commits | length }} commits were found in the role (``{{ older_sha }}`` › ``{{ latest_sha }}``): - -:: -{% endif %} - -{% for commit in commits if not commit.commit['message'].split('\n')[0][0:7] == 'Merge "' %} -{{ commit.sha[0:8] | indent(4, true)}} {{ commit.commit['message'].split('\n')[0] }} -{% endfor %}