Merge "Scan templated python files with bandit"

keystone/templates/bin/_endpoint-update.py.tpl

@@ -73,9 +73,8 @@ except:
 # Set Internal Endpoint
 try:
     endpoint_url = os.environ['OS_BOOTSTRAP_INTERNAL_URL']
-    user_engine.execute(
+    cmd = "update endpoint set url = %s where interface ='internal' and service_id = (select id from service where service.type = 'identity')"
-        "update endpoint set url = '{0}' where interface ='internal' and service_id = (select id from service where service.type = 'identity')".
+    user_engine.execute(cmd, (endpoint_url,))
-        format(endpoint_url))
 except:
     logger.critical("Could not update internal endpoint")
     raise
@@ -83,9 +82,8 @@ except:
 # Set Admin Endpoint
 try:
     endpoint_url = os.environ['OS_BOOTSTRAP_ADMIN_URL']
-    user_engine.execute(
+    cmd = "update endpoint set url = %s where interface ='admin' and service_id = (select id from service where service.type = 'identity')"
-        "update endpoint set url = '{0}' where interface ='admin' and service_id = (select id from service where service.type = 'identity')".
+    user_engine.execute(cmd, (endpoint_url,))
-        format(endpoint_url))
 except:
     logger.critical("Could not update admin endpoint")
     raise
@@ -93,9 +91,8 @@ except:
 # Set Public Endpoint
 try:
     endpoint_url = os.environ['OS_BOOTSTRAP_PUBLIC_URL']
-    user_engine.execute(
+    cmd = "update endpoint set url = %s where interface ='public' and service_id = (select id from service where service.type = 'identity')"
-        "update endpoint set url = '{0}' where interface ='public' and service_id = (select id from service where service.type = 'identity')".
+    user_engine.execute(cmd, (endpoint_url,))
-        format(endpoint_url))
 except:
     logger.critical("Could not update public endpoint")
     raise

keystone/templates/bin/_fernet-manage.py.tpl

@@ -25,7 +25,7 @@ import os
 import pwd
 import re
 import six
-import subprocess
+import subprocess  #nosec
 import sys
 import time
 
@@ -127,7 +127,7 @@ def execute_command(cmd):
     LOG.info("Executing 'keystone-manage %s --keystone-user=%s "
              "--keystone-group=%s' command.",
              cmd, KEYSTONE_USER, KEYSTONE_GROUP)
-    subprocess.call(['keystone-manage', cmd,
+    subprocess.call(['keystone-manage', cmd,  #nosec
                      '--keystone-user=%s' % KEYSTONE_USER,
                      '--keystone-group=%s' % KEYSTONE_GROUP])
 

tools/gate/files/template-python.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+EXCLUDES="helm-toolkit doc tests tools logs tmp roles playbooks releasenotes zuul.d python-files"
+DIRS=`ls -d */ | cut -f1 -d'/'`
+
+for EX in $EXCLUDES; do
+  DIRS=`echo $DIRS | sed "s/\b$EX\b//g"`
+done
+
+for DIR in $DIRS; do
+  PYFILES=$(helm template $DIR | yq 'select(.data != null) | .data | to_entries | map(select(.key | test(".*\\.py"))) | select(length > 0) | values[] | {(.key) : (.value)}' | jq -s add)
+  PYKEYS=$(echo "$PYFILES" | jq -r 'select(. != null) | keys[]')
+  for KEY in $PYKEYS; do
+    echo "$PYFILES" | jq -r --arg KEY "$KEY" '.[$KEY]' > ./python-files/"$DIR-$KEY"
+  done
+done

tools/gate/playbooks/osh-bandit.yaml

@@ -0,0 +1,28 @@
+- hosts: all
+  name: openstack-helm-bandit
+  tasks:
+
+    - name: Install Required Packages and Setup Host
+      shell: |
+        set -xe;
+        ./tools/deployment/common/install-packages.sh
+        ./tools/deployment/common/deploy-k8s.sh
+        sudo -H pip install yq bandit
+      environment:
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Template out python files
+      shell: |
+        set -xe;
+        make all
+        mkdir -p python-files
+        ./tools/gate/files/template-python.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+    - name: Run bandit against python files
+      shell: bandit -r ./python-files
+      args:
+        chdir: "{{ zuul.project.src_dir }}"

zuul.d/jobs-openstack-helm.yaml

@@ -24,6 +24,20 @@
       - ^doc/.*$
       - ^releasenotes/.*$
 
+- job:
+    name: openstack-helm-bandit
+    timeout: 3600
+    run: tools/gate/playbooks/osh-bandit.yaml
+    required-projects:
+      - openstack/openstack-helm-infra
+    # NOTE(gagehugo): Look into only running this for py.tpl file changes
+    # files:
+    #  - ^.*\.py\.tpl$
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+
 - job:
     name: openstack-helm-chart-deploy
     parent: openstack-helm-functional-temp

zuul.d/project.yaml

@@ -24,6 +24,7 @@
     check:
       jobs:
         - openstack-helm-lint
+        - openstack-helm-bandit
         - openstack-helm-keystone
         - openstack-helm-keystone-ldap
         - openstack-helm-glance