diff --git a/inventory/groups.yaml b/inventory/groups.yaml index e4361a066c..4b659bdf64 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -250,8 +250,6 @@ groups: - zuul[0-9]*.open*.org zuul-executor: - ze[0-9]*.open*.org - zuul-executor-opendev: - - ze[0-9]*.opendev.org zuul-merger: - zm[0-9]*.open*.org zuul-preview: diff --git a/playbooks/group_vars/all.yaml b/playbooks/group_vars/all.yaml index 151c47fb52..d35e32b9d2 100644 --- a/playbooks/group_vars/all.yaml +++ b/playbooks/group_vars/all.yaml @@ -20,6 +20,10 @@ iptables_base_allowed_hosts: iptables_extra_allowed_hosts: [] iptables_allowed_hosts: "{{ iptables_base_allowed_hosts + iptables_extra_allowed_hosts }}" +iptables_base_allowed_groups: [] +iptables_extra_allowed_groups: [] +iptables_allowed_groups: "{{ iptables_base_allowed_groups + iptables_extra_allowed_groups }}" + iptables_base_public_tcp_ports: [] iptables_extra_public_tcp_ports: [] # iptables_test_public_tcp_ports is here only to allow the test @@ -181,11 +185,4 @@ disabled_users: - shrews - dmsimard -iptables_snmp_v4_hosts: - # cacti02.openstack.org - - 172.99.116.215 -iptables_snmp_v6_hosts: - # cacti02.openstack.org - - 2001:4800:7821:105:be76:4eff:fe04:b9a5 - gerrit_ssh_rsa_pubkey_contents: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+pCQlTAQYmCrOY6aPbvbyKQDcOCXibPNGIjnPPMuEItCS0vtRnqEBz7znWZS5Drq9yKpROh6uFF01ao2VnNjw6f+NdRNV19RWVe6mYN+qa2VrH2caLwBrKPiH0Xc/eK41D55dZU7IWwKYAw/NpiBaBfHavFwipI+rmEb68MH2hcimDdr/bji+0hkh3X+42dkNvmMdtkuCW6nKdAEhnXaHZc5SJR/EvzgRCfB8vbML13p46O9xhoJgn7ZWvMb3vaR5jxIkQwstUR36raEVhttBDEuWasWnHYbrM1zd3ooudbTEQf5vXISZKFygHyJFFqb4iQ76i+hDlb0VQKZCdaol gerrit-code-review@829f141b0fa5 diff --git a/playbooks/group_vars/elasticsearch.yaml b/playbooks/group_vars/elasticsearch.yaml index 4c6112c8b7..942bfb1e7b 100644 --- a/playbooks/group_vars/elasticsearch.yaml +++ b/playbooks/group_vars/elasticsearch.yaml @@ -1,82 +1,4 @@ -iptables_extra_allowed_hosts: - - protocol: tcp - port: 9200:9400 - hostname: elasticsearch02.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: elasticsearch03.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: elasticsearch04.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: elasticsearch05.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: elasticsearch06.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: elasticsearch07.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker01.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker02.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker03.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker04.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker05.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker06.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker07.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker08.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker09.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker10.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker11.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker12.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker13.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker14.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker15.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker16.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker17.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker18.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker19.openstack.org - - protocol: tcp - port: 9200:9400 - hostname: logstash-worker20.openstack.org +iptables_extra_allowed_groups: + - {'protocol': 'tcp', 'port': '9200:9400', 'group': 'elasticsearch'} + - {'protocol': 'tcp', 'port': '9200:9400', 'group': 'logstash'} + - {'protocol': 'tcp', 'port': '9200:9400', 'group': 'logstash-worker'} diff --git a/playbooks/group_vars/graphite.yaml b/playbooks/group_vars/graphite.yaml index 330be24897..bd5418eaf7 100644 --- a/playbooks/group_vars/graphite.yaml +++ b/playbooks/group_vars/graphite.yaml @@ -5,99 +5,13 @@ iptables_extra_allowed_hosts: - hostname: opendev.org port: 8125 protocol: udp - - hostname: firehose01.openstack.org - port: 8125 - protocol: udp - hostname: mirror-update01.openstack.org port: 8125 protocol: udp - - hostname: mirror-update01.opendev.org - port: 8125 - protocol: udp - - hostname: logstash.openstack.org - port: 8125 - protocol: udp - - hostname: nb01.opendev.org - port: 8125 - protocol: udp - - hostname: nb02.opendev.org - port: 8125 - protocol: udp - - hostname: nb03.openstack.org - port: 8125 - protocol: udp - - hostname: nl01.openstack.org - port: 8125 - protocol: udp - - hostname: nl02.openstack.org - port: 8125 - protocol: udp - - hostname: nl03.openstack.org - port: 8125 - protocol: udp - - hostname: nl04.openstack.org - port: 8125 - protocol: udp - - hostname: zuul01.openstack.org - port: 8125 - protocol: udp - - hostname: zm01.openstack.org - port: 8125 - protocol: udp - - hostname: zm02.openstack.org - port: 8125 - protocol: udp - - hostname: zm03.openstack.org - port: 8125 - protocol: udp - - hostname: zm04.openstack.org - port: 8125 - protocol: udp - - hostname: zm05.openstack.org - port: 8125 - protocol: udp - - hostname: zm06.openstack.org - port: 8125 - protocol: udp - - hostname: zm07.openstack.org - port: 8125 - protocol: udp - - hostname: zm08.openstack.org - port: 8125 - protocol: udp - - hostname: ze01.openstack.org - port: 8125 - protocol: udp - - hostname: ze02.openstack.org - port: 8125 - protocol: udp - - hostname: ze03.openstack.org - port: 8125 - protocol: udp - - hostname: ze04.openstack.org - port: 8125 - protocol: udp - - hostname: ze05.openstack.org - port: 8125 - protocol: udp - - hostname: ze06.openstack.org - port: 8125 - protocol: udp - - hostname: ze07.openstack.org - port: 8125 - protocol: udp - - hostname: ze08.openstack.org - port: 8125 - protocol: udp - - hostname: ze09.openstack.org - port: 8125 - protocol: udp - - hostname: ze10.openstack.org - port: 8125 - protocol: udp - - hostname: ze11.openstack.org - port: 8125 - protocol: udp - - hostname: ze12.openstack.org - port: 8125 - protocol: udp + +iptables_extra_allowed_groups: + - {'protocol': 'udp', 'port': '8125', 'group': 'firehose'} + - {'protocol': 'udp', 'port': '8125', 'group': 'mirror-update'} + - {'protocol': 'udp', 'port': '8125', 'group': 'logstash'} + - {'protocol': 'udp', 'port': '8125', 'group': 'nodepool'} + - {'protocol': 'udp', 'port': '8125', 'group': 'zuul'} diff --git a/playbooks/group_vars/logstash.yaml b/playbooks/group_vars/logstash.yaml index 62c1c7a447..91e4cf90d0 100644 --- a/playbooks/group_vars/logstash.yaml +++ b/playbooks/group_vars/logstash.yaml @@ -1,106 +1,7 @@ iptables_extra_public_tcp_ports: - 80 - 3306 -iptables_extra_allowed_hosts: - - protocol: tcp - port: '4730' - hostname: logstash-worker01.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker02.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker03.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker04.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker05.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker06.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker07.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker08.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker09.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker10.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker11.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker12.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker13.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker14.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker15.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker16.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker17.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker18.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker19.openstack.org - - protocol: tcp - port: '4730' - hostname: logstash-worker20.openstack.org - - protocol: tcp - port: '4730' - hostname: subunit-worker01.openstack.org - - protocol: tcp - port: '4730' - hostname: subunit-worker02.openstack.org - - protocol: tcp - port: '4730' - hostname: ze01.openstack.org - - protocol: tcp - port: '4730' - hostname: ze02.openstack.org - - protocol: tcp - port: '4730' - hostname: ze03.openstack.org - - protocol: tcp - port: '4730' - hostname: ze04.openstack.org - - protocol: tcp - port: '4730' - hostname: ze05.openstack.org - - protocol: tcp - port: '4730' - hostname: ze06.openstack.org - - protocol: tcp - port: '4730' - hostname: ze07.openstack.org - - protocol: tcp - port: '4730' - hostname: ze08.openstack.org - - protocol: tcp - port: '4730' - hostname: ze09.openstack.org - - protocol: tcp - port: '4730' - hostname: ze10.openstack.org - - protocol: tcp - port: '4730' - hostname: ze11.openstack.org - - protocol: tcp - port: '4730' - hostname: ze12.openstack.org +iptables_extra_allowed_groups: + - {'protocol': 'tcp', 'port': '4730', 'group': 'logstash-worker'} + - {'protocol': 'tcp', 'port': '4730', 'group': 'subunit-worker'} + - {'protocol': 'tcp', 'port': '4730', 'group': 'zuul-executor'} diff --git a/playbooks/group_vars/zookeeper.yaml b/playbooks/group_vars/zookeeper.yaml index f62df8548a..f74b283b0f 100644 --- a/playbooks/group_vars/zookeeper.yaml +++ b/playbooks/group_vars/zookeeper.yaml @@ -2,21 +2,10 @@ zookeeper_user: zookeeper zookeeper_group: zookeeper zookeeper_uid: 10001 zookeeper_gid: 10001 -iptables_extra_allowed_hosts: - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nb01.opendev.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nb02.opendev.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nb03.openstack.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nb04.opendev.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nl01.openstack.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nl02.openstack.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nl03.openstack.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'nl04.openstack.org'} - - {'protocol': 'tcp', 'port': '2181', 'hostname': 'zuul01.openstack.org'} +iptables_extra_allowed_groups: + - {'protocol': 'tcp', 'port': '2181', 'group': 'nodepool'} + - {'protocol': 'tcp', 'port': '2181', 'group': 'zuul'} # Zookeeper election - - {'protocol': 'tcp', 'port': '2888', 'hostname': 'zk01.openstack.org'} - - {'protocol': 'tcp', 'port': '2888', 'hostname': 'zk02.openstack.org'} - - {'protocol': 'tcp', 'port': '2888', 'hostname': 'zk03.openstack.org'} + - {'protocol': 'tcp', 'port': '2888', 'group': 'zookeeper'} # Zookeeper leader - - {'protocol': 'tcp', 'port': '3888', 'hostname': 'zk01.openstack.org'} - - {'protocol': 'tcp', 'port': '3888', 'hostname': 'zk02.openstack.org'} - - {'protocol': 'tcp', 'port': '3888', 'hostname': 'zk03.openstack.org'} + - {'protocol': 'tcp', 'port': '3888', 'group': 'zookeeper'} diff --git a/playbooks/group_vars/zuul-scheduler.yaml b/playbooks/group_vars/zuul-scheduler.yaml index b606dc0e90..e135217735 100644 --- a/playbooks/group_vars/zuul-scheduler.yaml +++ b/playbooks/group_vars/zuul-scheduler.yaml @@ -2,67 +2,8 @@ iptables_extra_public_tcp_ports: - 79 - 80 - 443 -iptables_extra_allowed_hosts: - - protocol: tcp - port: 4730 - hostname: ze01.openstack.org - - protocol: tcp - port: 4730 - hostname: ze02.openstack.org - - protocol: tcp - port: 4730 - hostname: ze03.openstack.org - - protocol: tcp - port: 4730 - hostname: ze04.openstack.org - - protocol: tcp - port: 4730 - hostname: ze05.openstack.org - - protocol: tcp - port: 4730 - hostname: ze06.openstack.org - - protocol: tcp - port: 4730 - hostname: ze07.openstack.org - - protocol: tcp - port: 4730 - hostname: ze08.openstack.org - - protocol: tcp - port: 4730 - hostname: ze09.openstack.org - - protocol: tcp - port: 4730 - hostname: ze10.openstack.org - - protocol: tcp - port: 4730 - hostname: ze11.openstack.org - - protocol: tcp - port: 4730 - hostname: ze12.openstack.org - - protocol: tcp - port: 4730 - hostname: zm01.openstack.org - - protocol: tcp - port: 4730 - hostname: zm02.openstack.org - - protocol: tcp - port: 4730 - hostname: zm03.openstack.org - - protocol: tcp - port: 4730 - hostname: zm04.openstack.org - - protocol: tcp - port: 4730 - hostname: zm05.openstack.org - - protocol: tcp - port: 4730 - hostname: zm06.openstack.org - - protocol: tcp - port: 4730 - hostname: zm07.openstack.org - - protocol: tcp - port: 4730 - hostname: zm08.openstack.org +iptables_extra_allowed_groups: + - {'protocol': 'tcp', 'port': '4730', 'group': 'zuul'} zuul_connections: - name: 'smtp' driver: 'smtp' diff --git a/playbooks/roles/iptables/README.rst b/playbooks/roles/iptables/README.rst index 71c30da5cf..b7368cd88a 100644 --- a/playbooks/roles/iptables/README.rst +++ b/playbooks/roles/iptables/README.rst @@ -11,7 +11,26 @@ Install and configure iptables .. zuul:rolevar:: hostname The hostname to allow. It will automatically be resolved, and - all IP addresses will be added to the firewall. + the inventory IP address will be added to the firewall. + + .. zuul:rolevar:: protocol + + One of "tcp" or "udp". + + .. zuul:rolevar:: port + + The port number. + +.. zuul:rolevar:: iptables_allowed_groups + :default: [] + + A list of dictionaries, each item in the list is a rule to add for + a host/port combination. The format of the dictionary is: + + .. zuul:rolevar:: group + + The ansible inventory group to add. Every host in the group will + be added to the firewall. .. zuul:rolevar:: protocol diff --git a/playbooks/roles/iptables/templates/rules.v4.j2 b/playbooks/roles/iptables/templates/rules.v4.j2 index ce845904ff..0b3c3f268c 100644 --- a/playbooks/roles/iptables/templates/rules.v4.j2 +++ b/playbooks/roles/iptables/templates/rules.v4.j2 @@ -27,5 +27,12 @@ -A openstack-INPUT {% if host.protocol == 'tcp' %}-m state --state NEW {% endif %} -m {{ host.protocol }} -p {{ host.protocol }} -s {{ addr }} --dport {{ host.port }} -j ACCEPT {% endfor -%} {% endfor -%} +{% for group in iptables_allowed_groups -%} +{% for addr in groups.get(group.group) | map('extract', hostvars, 'public_v4') -%} +{% if addr -%} +-A openstack-INPUT {% if group.protocol == 'tcp' %}-m state --state NEW {% endif %} -m {{ group.protocol }} -p {{ group.protocol }} -s {{ addr }} --dport {{ group.port }} -j ACCEPT +{% endif -%} +{% endfor -%} +{% endfor -%} -A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited COMMIT diff --git a/playbooks/roles/iptables/templates/rules.v6.j2 b/playbooks/roles/iptables/templates/rules.v6.j2 index da5d369402..d5a792b9df 100644 --- a/playbooks/roles/iptables/templates/rules.v6.j2 +++ b/playbooks/roles/iptables/templates/rules.v6.j2 @@ -26,5 +26,12 @@ -A openstack-INPUT {% if host.protocol == 'tcp' %}-m state --state NEW {% endif %}-m {{ host.protocol }} -p {{ host.protocol }} -s {{ addr }} --dport {{ host.port }} -j ACCEPT {% endfor -%} {% endfor -%} +{% for group in iptables_allowed_groups -%} +{% for addr in groups.get(group.group) | map('extract', hostvars, 'public_v6') -%} +{% if addr -%} +-A openstack-INPUT {% if group.protocol == 'tcp' %}-m state --state NEW {% endif %} -m {{ group.protocol }} -p {{ group.protocol }} -s {{ addr }} --dport {{ group.port }} -j ACCEPT +{% endif -%} +{% endfor -%} +{% endfor -%} -A openstack-INPUT -j REJECT --reject-with icmp6-adm-prohibited COMMIT diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 033c2c400c..66a6b99e28 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -15,6 +15,9 @@ write_inventory_exclude_hostvars: - ansible_user - ansible_python_interpreter + write_inventory_additional_hostvars: + public_v4: nodepool.public_ipv4 + public_v6: nodepool.public_ipv6 - name: Add groups config for test nodes template: src: "templates/gate-groups.yaml.j2" diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 7a9685e72f..b22d1140e9 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -16,6 +16,8 @@ '/var/log/syslog': logs_txt '/var/log/messages': logs_txt '/var/log/docker': logs + '/etc/iptables/rules.v4': logs_txt + '/etc/iptables/rules.v6': logs_txt host-vars: bridge.openstack.org: host_copy_output: