Update tox config and add bashate E006, E040 exceptions

This patch updates the tox.ini the same bashate exceptions as are
currently in the OpenStack-Ansible playbook repo.

It also ensures that the linters and all lint targets work
appropriately and normalises the tox.ini configuration to use
uniform formatting.

The use of ansible.cfg is removed as there is no way of being
certain which paths can be used without reverting to an ugly
sed hack in the commands. This is why it is preferred to make
use of environment variables which make use of tox's default
substitutions instead. It's a more reliable way of achieving
the goal for the purpose of gating and testing.

The switch to using a git clone instead of ansible-galaxy to
download the plugins is due to the path spec not being able to
work in Ansible 2.x. [1]

[1] https://github.com/ansible/ansible/issues/13563

Change-Id: I077c5b79a95e78fd81c33382843273893d48bbb6

test-requirements.txt

@@ -1,5 +1,7 @@
 ansible-lint
 ansible>=1.9.1,<2.0.0
+bashate
+flake8
 
 # this is required for the docs build jobs
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2

tests/ansible-role-requirements.yml

@@ -1,8 +1,3 @@
-- name: plugins
-  src: https://github.com/os-cloud/openstack-ansible-plugins
-  path: ../../
-  scm: git
-  version: master
 - name: apt_package_pinning
   src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
   scm: git

tests/ansible.cfg

@@ -1,12 +0,0 @@
-[defaults]
-roles_path = ../../
-remote_tmp = ../.ansible/tmp/
-lookup_plugins = ../../plugins/lookups
-filter_plugins = ../../plugins/filters
-action_plugins = ../../plugins/actions
-library = ../../plugins/library
-
-host_key_checking = False
-
-[ssh_connection]
-control_path = /tmp/%%h-%%r

tests/test.yml

@@ -53,11 +53,6 @@
           name: "trusty.tgz"
           sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c"
           chroot_path: trusty/rootfs-amd64
-      # The $HOME directory is mocked to work with tox
-      #  by defining the 'ansible_env' hash. This should
-      #  NEVER be done outside of testing.
-      ansible_env:  ## NEVER DO THIS OUTSIDE OF TESTING
-        HOME: "/tmp"
     - role: "py_from_git"
       git_repo: "https://github.com/lxc/python2-lxc"
       git_dest: "/opt/lxc_python2"

tox.ini

@@ -1,96 +1,125 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = docs,pep8,ansible-syntax,ansible-lint,functional
+envlist = docs,linters,functional
+
 
 [testenv]
 usedevelop = True
 install_command = pip install -U {opts} {packages}
-setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
-commands =
-    /usr/bin/find . -type f -name "*.pyc" -delete
-    ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
-                   --ignore-errors \
-                   --force
+commands = /usr/bin/find . -type f -name "*.pyc" -delete
+passenv =
+    HOME
+whitelist_externals =
+    bash
+    git
+    rm
+setenv =
+    VIRTUAL_ENV={envdir}
+    ANSIBLE_HOST_KEY_CHECKING = False
+    ANSIBLE_SSH_CONTROL_PATH = /tmp/%%h-%%r
+    # TODO (odyssey4me) These are only here as they are non-standard folder
+    # names for Ansible 1.9.x. We are using the standard folder names for
+    # Ansible v2.x. We can remove this when we move to Ansible 2.x.
+    ANSIBLE_ACTION_PLUGINS = {homedir}/.ansible/plugins/action
+    ANSIBLE_CALLBACK_PLUGINS = {homedir}/.ansible/plugins/callback
+    ANSIBLE_FILTER_PLUGINS = {homedir}/.ansible/plugins/filter
+    ANSIBLE_LOOKUP_PLUGINS = {homedir}/.ansible/plugins/lookup
+    # This is required as the default is the current path or a path specified
+    # in ansible.cfg
+    ANSIBLE_LIBRARY = {homedir}/.ansible/plugins/library
+    # This is required as the default is '/etc/ansible/roles' or a path
+    # specified in ansible.cfg
+    ANSIBLE_ROLES_PATH = {homedir}/.ansible/roles:{toxinidir}/..
+
 
 [testenv:docs]
-commands = python setup.py build_sphinx
+commands=
+    python setup.py build_sphinx
+
 
 # environment used by the -infra templated docs job
 [testenv:venv]
 deps = -r{toxinidir}/test-requirements.txt
 commands = {posargs}
 
-# Run various lint checks
-[testenv:pep8]
-deps =
-    bashate
-    flake8
-whitelist_externals = bash
-# Run hacking/flake8 check for all python files
-commands =
-    bash -c "grep -Irl \
-        -e '!/usr/bin/env python' \
-        -e '!/bin/python' \
-        -e '!/usr/bin/python' \
-        --exclude-dir '.*' \
-        --exclude-dir 'doc' \
-        --exclude-dir '*.egg' \
-        --exclude-dir '*.egg-info' \
-        --exclude 'tox.ini' \
-        --exclude '*.sh' \
-        {toxinidir} | xargs flake8 --verbose"
 
-# Run bashate check for all bash scripts
-# Ignores the following rules:
-#     E003: Indent not multiple of 4 (we prefer to use multiples of 2)
-    bash -c "grep -Irl \
-        -e '!/usr/bin/env bash' \
-        -e '!/bin/bash' \
-        -e '!/bin/sh' \
-        --exclude-dir '.*' \
-        --exclude-dir '*.egg' \
-        --exclude-dir '*.egg-info' \
-        --exclude 'tox.ini' \
-        {toxinidir} | xargs bashate --verbose --ignore=E003"
+[testenv:pep8]
+commands =
+    # Run hacking/flake8 check for all python files
+    bash -c "grep --recursive --binary-files=without-match \
+        --files-with-match '^.!.*python$' \
+        --exclude-dir .eggs \
+        --exclude-dir .git \
+        --exclude-dir .tox \
+        --exclude-dir *.egg-info \
+        --exclude-dir doc \
+        {toxinidir} | xargs flake8 --verbose"
 
 [flake8]
 # Ignores the following rules due to how ansible modules work in general
-#     F403 'from ansible.module_utils.basic import *' used; unable to detect undefined names
-#     H303  No wildcard (*) import.
+#     F403 'from ansible.module_utils.basic import *' used;
+#          unable to detect undefined names
+#     H303 No wildcard (*) import.
 ignore=F403,H303
 
-[testenv:ansible-syntax]
-changedir = tests
+
+[testenv:bashate]
 commands =
+    # Run bashate check for all bash scripts
+    # Ignores the following rules:
+    #     E003: Indent not multiple of 4 (we prefer to use multiples of 2)
+    #     E006: Line longer than 79 columns (as many scripts use jinja
+    #           templating, this is very difficult)
+    #     E040: Syntax error determined using `bash -n` (as many scripts
+    #           use jinja templating, this will often fail and the syntax
+    #           error will be discovered in execution anyway)
+    bash -c "grep --recursive --binary-files=without-match \
+        --files-with-match '^.!.*\(ba\)\?sh$' \
+        --exclude-dir .tox \
+        --exclude-dir .git \
+        {toxinidir} | xargs bashate --error . --verbose --ignore=E003,E006,E040"
+
+
+[testenv:ansible-syntax]
+commands =
+    rm -rf {homedir}/.ansible
+    git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+              {homedir}/.ansible/plugins
     ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
+                   --role-file={toxinidir}/tests/ansible-role-requirements.yml \
                    --ignore-errors \
                    --force
-    ansible-playbook -i inventory \
+    ansible-playbook -i {toxinidir}/tests/inventory \
                      --syntax-check \
                      --list-tasks \
                      -e "rolename={toxinidir}" \
-                     test.yml
+                     {toxinidir}/tests/test.yml
+
 
 [testenv:ansible-lint]
-changedir = tests
 commands =
-    ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
-                   --ignore-errors \
-                   --force
-    ansible-lint test.yml
+    ansible-lint {toxinidir}/tests/test.yml
+
 
 [testenv:functional]
-changedir = tests
 commands =
+    rm -rf {homedir}/.ansible
+    git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+              {homedir}/.ansible/plugins
     ansible-galaxy install \
-                   --role-file=ansible-role-requirements.yml \
+                   --role-file={toxinidir}/tests/ansible-role-requirements.yml \
                    --ignore-errors \
                    --force
-    ansible-playbook -i inventory \
+    ansible-playbook -i {toxinidir}/tests/inventory \
                      -e "rolename={toxinidir}" \
-                     test.yml
+                     {toxinidir}/tests/test.yml
+
+
+[testenv:linters]
+commands =
+    {[testenv:pep8]commands}
+    {[testenv:bashate]commands}
+    {[testenv:ansible-lint]commands}
+    {[testenv:ansible-syntax]commands}