Add lists exim config to ansible

The mailing list servers have a more complex exim config. Put the
routers and transports into ansible variables.

While we're doing it, role variables with an exim_ prefix - since 'routers'
as a global variable might be a little broad.

iteritems isn't a thing in python3, only items.

We need to escape the exim config with ${if or{{ - because of the {{
which looks like jinja. Wrap it in a {% raw %} block.

Getting the yaml indentation right for things here is non-trivial. Make
them strings instead.

Add a README.rst file - and use the zuul:rolevar construct in it,
because it's nice.

Change-Id: Ieccfce99a1d278440c5baa207479a1887898298e

playbooks/base.yaml

@@ -8,6 +8,4 @@
 - hosts: "!disabled"
   roles:
     - base-server
-    - role: exim
+    - exim
-      aliases:
-        root: "{{ ','.join(sysadmins|default([])) }}"

playbooks/group_vars/all.yaml

@@ -11,6 +11,12 @@ distro_lookup_path:
   - "{{ ansible_facts.os_family }}.yaml"
   - default.yaml
 
+exim_sysadmins: "{{ ','.join(sysadmins|default([])) }}"
+exim_extra_aliases: {}
+exim_base_aliases:
+  root: "{{ exim_sysadmins }}"
+exim_aliases: "{{ exim_base_aliases|combine(exim_extra_aliases) }}"
+
 puppet:
   logdest: syslog
   copy_hieradata: true

playbooks/group_vars/mailman.yaml

@@ -0,0 +1,4 @@
+exim_queue_interval: '1m'
+exim_queue_run_max: '50'
+exim_smtp_accept_max: '100'
+exim_smtp_accept_max_per_host: '10'

playbooks/host_vars/lists.katacontainers.io.yaml

@@ -0,0 +1,55 @@
+mm_domains: lists.katacontainers.io
+exim_local_domains: "@:{{ mm_domains }}"
+exim_aliases:
+  root: "{{ ','.join(listadmins|default([])) }}"
+exim_routers:
+  - mailman_verp_router: |
+      {% raw -%}
+      driver = dnslookup
+      # we only consider messages sent in through loopback
+      condition = ${if or{{eq{$sender_host_address}{127.0.0.1}}\
+                              {eq{$sender_host_address}{::1}}}{yes}{no}}
+      {% endraw %}
+      # we do not do this for traffic going to the local machine
+      domains = !+local_domains
+      ignore_target_hosts = <; 0.0.0.0; \
+                                        64.94.110.11; \
+                                        127.0.0.0/8; \
+                                        ::1/128;fe80::/10;fe \
+                                        c0::/10;ff00::/8
+      # only the un-VERPed bounce addresses are handled
+      senders = "*-bounces@*"
+      transport = mailman_verp_smtp
+  - mailman_router: |
+      driver            = accept
+      domains           = {{ mm_domains }}
+      require_files     = /var/lib/mailman/lists/${lc::$local_part}/config.pck
+      local_part_suffix_optional
+      local_part_suffix = -admin     : \
+             -bounces   : -bounces+* : \
+             -confirm   : -confirm+* : \
+             -join      : -leave     : \
+             -owner     : -request   : \
+             -subscribe : -unsubscribe
+      transport         = mailman_transport
+exim_transports:
+  - mailman_transport: |
+      driver = pipe
+      command = /var/lib/mailman/mail/mailman \
+          '${if def:local_part_suffix \
+                {${sg{$local_part_suffix}{-(\\w+)(\\+.*)?}{\$1}}} \
+                {post}}' \
+          $local_part
+      current_directory = /var/lib/mailman
+      home_directory = /var/lib/mailman
+      user = list
+      group = list
+  - mailman_verp_smtp: |
+      driver = smtp
+      # put recipient address into return_path
+      return_path = \
+        ${local_part:$return_path}+$local_part=$domain@${domain:$return_path}
+      max_rcpt = 1
+      # Errors-To: may carry old return_path
+      headers_remove = Errors-To
+      headers_add = Errors-To: ${return_path}

playbooks/host_vars/lists.openstack.org.yaml

@@ -0,0 +1,75 @@
+mm_domains: 'lists.openstack.org:lists.zuul-ci.org:lists.airshipit.org:lists.starlingx.io'
+exim_local_domains: "@:{{ mm_domains }}"
+exim_aliases:
+  root: "{{ ','.join(listadmins|default([])) }}"
+  ambassadors-owner: spam
+  community-owner: spam
+  foundation-board-confidential-owner: spam
+  foundation-board-owner: spam
+  foundation-owner: spam
+  legal-discuss-owner: spam
+  mailman-owner: spam
+  marketing-owner: spam
+  openstack-announce-owner: spam
+  openstack-dev-owner: spam
+  openstack-docs-owner: spam
+  openstack-fr-owner: spam
+  openstack-i18n-owner: spam
+  openstack-infra-owner: spam
+  openstack-operators-owner: spam
+  openstack-owner: spam
+  openstack-qa-owner: spam
+  openstack-security-owner: spam
+  openstack-tc-owner: spam
+  openstack-vi-owner: spam
+  product-wg-owner: spam
+  superuser-owner: spam
+  user-committee-owner: spam
+  women-of-openstack-owner: spam
+  spam: ':fail: delivery temporarily disabled due to ongoing spam flood'
+exim_routers:
+  - mailman_verp_router: |
+      {% raw -%}
+      driver = dnslookup
+      condition = ${if or{{eq{$sender_host_address}{127.0.0.1}}\
+                              {eq{$sender_host_address}{::1}}}{yes}{no}}
+      {% endraw %}
+      domains = !+local_domains
+      ignore_target_hosts = <; 0.0.0.0; \
+                                        64.94.110.11; \
+                                        127.0.0.0/8; \
+                                        ::1/128;fe80::/10;fe \
+                                        c0::/10;ff00::/8
+      senders = "*-bounces@*"
+      transport = mailman_verp_smtp
+  - mailman_router: |
+      driver = accept
+      domains = {{ mm_domains }}
+      local_part_suffix = -admin     : \
+                                    -bounces   : -bounces+* : \
+                                    -confirm   : -confirm+* : \
+                                    -join      : -leave     : \
+                                    -owner     : -request   : \
+                                    -subscribe : -unsubscribe
+      local_part_suffix_optional
+      require_files = ${lookup{${lc::$domain}}lsearch{/etc/mailman/sites}}/lists/${lc::$local_part}/config.pck
+      transport = mailman_transport
+exim_transports:
+  - mailman_transport: |
+      driver = pipe
+      command = /var/lib/mailman/mail/mailman \
+                              '${if def:local_part_suffix \
+                                     {${sg{$local_part_suffix}{-(\\w+)(\\+.*)?}{\$1}}} \
+                                     {post}}' \
+                              $local_part
+      current_directory = /var/lib/mailman
+      environment = MAILMAN_SITE_DIR=${lookup{${lc:$domain}}lsearch{/etc/mailman/sites}}
+      group = list
+      home_directory = /var/lib/mailman
+      user = list
+  - mailman_verp_smtp: |
+      driver = smtp
+      headers_add = Errors-To: ${return_path}
+      headers_remove = Errors-To
+      max_rcpt = 1
+      return_path = ${local_part:$return_path}+$local_part=$domain@${domain:$return_path}

playbooks/roles/exim/README.rst

@@ -0,0 +1,54 @@
+Installs and configures the exim mail server
+
+**Role Variables**
+
+.. zuul:rolevar:: exim_aliases
+   :default: {}
+
+   A dictionary with keys being the email alias and the value being the
+   address or comma separated list of addresses.
+
+.. zuul:rolevar:: exim_routers
+   :default: []
+
+   A list of additional exim routers to define.
+
+.. zuul:rolevar:: exim_transports
+   :default: []
+
+   A list of additional exim transports to define.
+
+.. zuul:rolevar:: exim_local_domains
+   :default: "@"
+
+   Colon separated list of local domains.
+
+.. zuul:rolevar:: exim_queue_interval
+   :default: 30m
+
+   How often should we run the queue.
+
+.. zuul:rolevar:: exim_queue_run_max
+   :default: 5
+
+   Number of simultaneous queue runners.
+
+.. zuul:rolevar:: exim_smtp_accept_max
+   :default: null
+
+   The maximum number of simultaneous incoming SMTP calls that Exim will
+   accept. If the value is set to zero, no limit is applied. However, it
+   is required to be non-zero if
+   :zuul:rolevar:`exim_smtp_accept_max_per_host` is set.
+
+.. zuul:rolevar:: exim_smtp_accept_max_per_host
+   :default: null
+
+   Restrict the number of simultaneous IP connections from a single host
+   (strictly, from a single IP address) to the Exim daemon. The option is
+   expanded, to enable different limits to be applied to different hosts
+   by reference to ``$sender_host_address``. Once the limit is reached,
+   additional connection attempts from the same host are rejected with error
+   code 421. The option’s default value imposes no limit. If this option is
+   set greater than zero, it is required that
+   :zuul:rolevar:`exim_smtp_accept_max` be non-zero.

playbooks/roles/exim/defaults/main.yaml

@@ -0,0 +1,6 @@
+exim_aliases: {}
+exim_local_domains: '@'
+exim_queue_interval: 30m
+exim_queue_run_max: 5
+exim_routers: []
+exim_transports: []

playbooks/roles/exim/templates/aliases.j2

@@ -12,7 +12,7 @@ abuse: root
 noc: root
 security: root
 
-{% for k, v in aliases|dictsort %}
+{% for k, v in exim_aliases|dictsort %}
 {% if v %}
 {{ k }}: {{ v }}
 {% endif %}

playbooks/roles/exim/templates/exim.sysconfig.j2

@@ -1,2 +1,2 @@
 DAEMON=yes
-QUEUE={{ queue_interval }}
+QUEUE={{ exim_queue_interval }}

playbooks/roles/exim/templates/exim4.conf.j2

@@ -61,7 +61,7 @@ keep_environment =
 # +local_domains, +relay_to_domains, and +relay_from_hosts, respectively. They
 # are all colon-separated lists:
 
-domainlist local_domains = {{ local_domains }}
+domainlist local_domains = {{ exim_local_domains }}
 domainlist relay_to_domains =
 hostlist   relay_from_hosts = 127.0.0.1
 
@@ -303,7 +303,7 @@ timeout_frozen_after = 7d
 # interlock with other processes, so additional queue runners can be
 # started by other means, or by killing and restarting the daemon.
 
-queue_run_max = {{ queue_run_max }}
+queue_run_max = {{ exim_queue_run_max }}
 
 # When this option is set, a delivery process is started whenever a
 # message is received, routing is performed, and local deliveries take
@@ -324,14 +324,14 @@ queue_run_max = {{ queue_run_max }}
 
 dns_ipv4_lookup = *.gmail.com : *.google.com
 
-{% if smtp_accept_max is defined %}
+{% if exim_smtp_accept_max is defined %}
 # simultaneous incoming connections
-smtp_accept_max = {{ smtp_accept_max }}
+smtp_accept_max = {{ exim_smtp_accept_max }}
 {% endif %}
 
-{% if smtp_accept_max_per_host is defined %}
+{% if exim_smtp_accept_max_per_host is defined %}
 # connections from one host
-smtp_accept_max_per_host = {{ smtp_accept_max_per_host }}
+smtp_accept_max_per_host = {{ exim_smtp_accept_max_per_host }}
 {% endif %}
 
 ######################################################################
@@ -543,26 +543,6 @@ acl_check_data:
 
 begin routers
 
-{% for router in routers %}
-{% for name, values in router.iteritems() %}
-
-{{ name }}:
-{% if 'driver' in values %}
-  driver = {{ values.driver }}
-{% endif %}
-
-{% for k, v in values|dictsort %}
-{% if k != 'driver' %}
-{% if v|true %}
-  {{ k }}
-{% else %}
-  {{ k }} = {{ v }}
-{% endif %}
-{% endif %}
-{% endfor %}
-{% endfor %}
-{% endfor %}
-
 # This router routes to remote hosts over SMTP by explicit IP address,
 # when an email address is given in "domain literal" form, for example,
 # <user@[192.168.35.64]>. The RFCs require this facility. However, it is
@@ -649,6 +629,15 @@ localuser:
   transport = local_delivery
   cannot_route_message = Unknown user
 
+{% for router in exim_routers %}
+{% for name, values in router.items() %}
+
+{{ name }}:
+{% for value in values.split('\n') %}
+  {{ value }}
+{% endfor %}
+{% endfor %}
+{% endfor %}
 
 ######################################################################
 #                      TRANSPORTS CONFIGURATION                      #
@@ -716,22 +705,12 @@ address_reply:
   driver = autoreply
 
 
-{% for transport in transports %}
+{% for transport in exim_transports %}
-{% for name, values in transport.iteritems() %}
+{% for name, values in transport.items() %}
 
 {{ name }}:
-{% if 'driver' in values %}
+{% for value in values.split('\n') %}
-  driver = {{ values.driver }}
+  {{ value }}
-{% endif %}
-
-{% for k, v in values|dictsort %}
-{% if k != 'driver' %}
-{% if v|true %}
-  {{ k }}
-{% else %}
-  {{ k }} = {{ v }}
-{% endif %}
-{% endif %}
 {% endfor %}
 {% endfor %}
 {% endfor %}

playbooks/roles/exim/templates/exim4.default.j2

@@ -9,7 +9,7 @@ EX4DEF_VERSION=''
 # setting this to 'no' will also disable queueruns from /etc/ppp/ip-up.d/exim4
 QUEUERUNNER='combined'
 # how often should we run the queue
-QUEUEINTERVAL='{{ queue_interval }}'
+QUEUEINTERVAL='{{ exim_queue_interval }}'
 # options common to quez-runner and listening daemon
 COMMONOPTIONS=''
 # more options for the daemon/process running the queue (applies to the one

playbooks/roles/exim/vars/main.yaml

@@ -1,6 +0,0 @@
-queue_interval: 30m
-queue_run_max: 5
-local_domains: '@'
-routers: []
-transports: []
-aliases: {}