Merge "Stop using backend hostname in zuul testinfra tests"

playbooks/zuul/run-base-pre.yaml

@@ -2,10 +2,10 @@
   roles:
     - ensure-tox
     - multi-node-known-hosts
-    - multi-node-hosts-file
     - copy-build-sshkey
     - use-docker-mirror
     - set-hostname
+    - multi-node-hosts-file
   tasks:
     - include_role:
         name: use-buildset-registry

playbooks/zuul/run-base.yaml

@@ -16,7 +16,7 @@
           - ansible_user
           - ansible_python_interpreter
         write_inventory_additional_hostvars:
-          public_v4: nodepool.public_ipv4
+          public_v4: nodepool.private_ipv4
           public_v6: nodepool.public_ipv6
     - name: Add groups config for test nodes
       template:

testinfra/test_base.py

@@ -12,22 +12,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import socket
+import util
 
 testinfra_hosts = ['all']
 
 
-def get_ips(value, family=None):
-    ret = set()
-    try:
-        addr_info = socket.getaddrinfo(value, None, family)
-    except socket.gaierror:
-        return ret
-    for addr in addr_info:
-        ret.add(addr[4][0])
-    return ret
-
-
 def test_exim_is_installed(host):
     if host.system_info.distribution in ['ubuntu', 'debian']:
         exim = host.package("exim4-base")
@@ -40,42 +29,13 @@ def test_exim_is_installed(host):
 
 
 def test_iptables(host):
-    rules = host.iptables.rules()
-    rules = [x.strip() for x in rules]
-
-    needed_rules = [
-        '-P INPUT ACCEPT',
-        '-P FORWARD DROP',
-        '-P OUTPUT ACCEPT',
-        '-N openstack-INPUT',
-        '-A INPUT -j openstack-INPUT',
-        '-A openstack-INPUT -i lo -j ACCEPT',
-        '-A openstack-INPUT -p icmp -m icmp --icmp-type any -j ACCEPT',
-        '-A openstack-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT',
-        '-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT',
-        '-A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited'
-    ]
-    for rule in needed_rules:
-        assert rule in rules
+    rules = util.verify_iptables(host)
 
     # Make sure that the zuul console stream rule is still present
     zuul = ('-A openstack-INPUT -p tcp -m state --state NEW'
             ' -m tcp --dport 19885 -j ACCEPT')
     assert zuul in rules
 
-    # Ensure all IPv4+6 addresses for cacti are allowed
-    for ip in get_ips('cacti.openstack.org', socket.AF_INET):
-        snmp = ('-A openstack-INPUT -s %s/32 -p udp -m udp'
-                ' --dport 161 -j ACCEPT' % ip)
-        assert snmp in rules
-
-    # TODO(ianw) add ip6tables support to testinfra iptables module
-    ip6rules = host.check_output('ip6tables -S')
-    for ip in get_ips('cacti.openstack.org', socket.AF_INET6):
-        snmp = ('-A openstack-INPUT -s %s/128 -p udp -m udp'
-                ' --dport 161 -j ACCEPT' % ip)
-        assert snmp in ip6rules
-
 
 def test_ntp(host):
     package = host.package("ntp")

testinfra/test_zuul_executor.py

@@ -0,0 +1,31 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import util
+
+testinfra_hosts = ['ze01.opendev.org']
+
+
+def test_iptables(host):
+    rules = util.verify_iptables(host)
+
+    needed_rules = [
+        '-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 79 -j ACCEPT',
+        '-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 7900 -j ACCEPT',
+    ]
+    for rule in needed_rules:
+        assert rule in rules
+
+    for rule in rules:
+        assert '--dport 4730' not in rule

testinfra/test_zuul_merger.py

@@ -0,0 +1,26 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import util
+
+testinfra_hosts = ['zm01.opendev.org']
+
+
+def test_iptables(host):
+    rules = util.verify_iptables(host)
+
+    for rule in rules:
+        assert '--dport 4730' not in rule
+        assert '--dport 79' not in rule
+        assert '--dport 7900' not in rule

testinfra/test_zuul_scheduler.py

@@ -0,0 +1,34 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import util
+import socket
+
+testinfra_hosts = ['zuul01.openstack.org']
+
+
+def test_iptables(host):
+    rules = util.verify_iptables(host)
+
+    ips = util.get_ips('ze01.opendev.org', socket.AF_INET)
+    assert len(ips) > 0
+
+    # Make sure that the gearman port is open to executors on the scheduler
+    for ip in util.get_ips('ze01.opendev.org', socket.AF_INET):
+        zuul = ('-A openstack-INPUT -s %s/32 -p tcp -m state --state NEW'
+                ' -m tcp --dport 4730 -j ACCEPT' % ip)
+        assert zuul in rules
+
+    for rule in rules:
+        assert '--dport 7900' not in rule

testinfra/util.py

@@ -14,14 +14,6 @@
 
 import socket
 
-
-testinfra_hosts = [
-    'ze01.opendev.org',
-    'zm01.openstack.org',
-    'zuul01.openstack.org',
-]
-
-
 def get_ips(value, family=None):
     ret = set()
     try:
@@ -33,7 +25,7 @@ def get_ips(value, family=None):
     return ret
 
 
-def test_iptables(host):
+def verify_iptables(host):
     rules = host.iptables.rules()
     rules = [x.strip() for x in rules]
 
@@ -52,13 +44,6 @@ def test_iptables(host):
     for rule in needed_rules:
         assert rule in rules
 
-    # Make sure that the gearman port is open to executors on the scheduler
-    if host.backend.get_hostname() == 'zuul01.openstack.org':
-        for ip in get_ips('ze01.opendev.org', socket.AF_INET):
-            zuul = ('-A openstack-INPUT -s %s/32 -p tcp -m state --state NEW'
-                    ' -m tcp --dport 4730 -j ACCEPT' % ip)
-            assert zuul in rules
-
     # Ensure all IPv4+6 addresses for cacti are allowed
     for ip in get_ips('cacti.openstack.org', socket.AF_INET):
         snmp = ('-A openstack-INPUT -s %s/32 -p udp -m udp'
@@ -71,3 +56,5 @@ def test_iptables(host):
         snmp = ('-A openstack-INPUT -s %s/128 -p udp -m udp'
                 ' --dport 161 -j ACCEPT' % ip)
         assert snmp in ip6rules
+
+    return rules

zuul.d/system-config-run.yaml

@@ -648,20 +648,27 @@
     host-vars:
       zm01.openstack.org:
         host_copy_output:
+          '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
           '/var/log/zuul/merger-debug.log': logs
       ze01.opendev.org:
         host_copy_output:
+          '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
           '/var/log/zuul/executor-debug.log': logs
       ze01.openstack.org:
         host_copy_output:
+          '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
           '/var/log/zuul/executor-debug.log': logs
       zuul01.openstack.org:
         host_copy_output:
+          '/etc/hosts': logs
           '/etc/zuul/zuul.conf': logs
           '/var/log/zuul/debug.log': logs
+      bridge.openstack.org:
+        host_copy_output:
+          '/etc/hosts': logs
     files:
       - playbooks/install-ansible.yaml
       - playbooks/service-zookeeper.yaml
@@ -673,6 +680,10 @@
       - playbooks/roles/zookeeper/
       - playbooks/roles/install-apt-repo
       - playbooks/roles/zuul
+      - testinfra/test_zuul_executor.py
+      - testinfra/test_zuul_scheduler.py
+      - testinfra/test_zuul_merger.py
+      - testinfra/util.py
 
 - job:
     name: system-config-run-review