diff --git a/inventory/service/group_vars/mailman3.yaml b/inventory/service/group_vars/mailman3.yaml
new file mode 100644
index 0000000000..a721f7e356
--- /dev/null
+++ b/inventory/service/group_vars/mailman3.yaml
@@ -0,0 +1,89 @@
+exim_queue_interval: '1m'
+exim_queue_run_max: '50'
+exim_smtp_accept_max: '100'
+exim_smtp_accept_max_per_host: '10'
+iptables_extra_public_tcp_ports:
+ - 25
+ - 80
+ - 443
+ - 465
+letsencrypt_certs:
+ lists-opendev-org-main:
+ - "{{ inventory_hostname }}"
+ - lists.opendev.org
+ - lists.airshipit.org
+ - lists.katacontainers.io
+ - lists.openinfra.dev
+ - lists.openstack.org
+ - lists.starlingx.io
+ - lists.zuul-ci.org
+borg_backup_excludes_extra:
+ # db is backed up in dumps, don't capture live files
+ - /var/lib/mailman/database
+ # backed up by streaming backup
+ - /var/backups/mailman-mariadb
+ # Can regenerate indexes from source email files
+ - /var/lib/mailman/web-data/fulltext_index
+exim_routers:
+ - mailman_verp_router: |
+ {% raw -%}
+ driver = dnslookup
+ condition = ${if or{{eq{$sender_host_address}{127.0.0.1}}\
+ {eq{$sender_host_address}{::1}}}{yes}{no}}
+ {% endraw %}
+ domains = !+local_domains
+ ignore_target_hosts = <; 0.0.0.0; \
+ 127.0.0.0/8; \
+ ::1/128;fe80::/10;fe \
+ c0::/10;ff00::/8
+ senders = "*-bounces@*"
+ transport = mailman_verp_smtp
+ - dnslookup: '{{ exim_dnslookup_router }}'
+ - system_aliases: '{{ exim_system_aliases_router }}'
+ - domain_aliases: |
+ driver = redirect
+ allow_fail
+ allow_defer
+ data = ${lookup{$local_part@$domain}lsearch{/etc/aliases.domain}}
+ file_transport = address_file
+ pipe_transport = address_pipe
+ - localuser: '{{ exim_localuser_router }}'
+ - mailman_copy: |
+ driver = accept
+ domains = lists.openstack.org
+ local_parts = openstack-discuss
+ transport = local_copy
+ unseen
+ - mailman_router: |
+ driver = accept
+ domains = {{ mm_domains }}
+ local_part_suffix = -admin : \
+ -bounces : -bounces+* : \
+ -confirm : -confirm+* : \
+ -join : -leave : \
+ -owner : -request : \
+ -subscribe : -unsubscribe
+ local_part_suffix_optional
+ require_files = /var/lib/mailman/core/var/lists/${local_part}.${domain}
+ transport = mailman_transport
+exim_transports:
+ - local_copy: |
+ driver = appendfile
+ file = /var/mail/$local_part
+ group = mail
+ mode = 0660
+ - mailman_transport: |
+ debug_print = "Email for mailman"
+ driver = smtp
+ protocol = lmtp
+ allow_localhost
+ hosts = localhost
+ port = 8024
+ rcpt_include_affixes = true
+ - mailman_verp_smtp: |
+ driver = smtp
+ headers_add = Errors-To: ${return_path}
+ headers_remove = Errors-To
+ max_rcpt = 1
+ return_path = ${local_part:$return_path}+$local_part=$domain@${domain:$return_path}
+mailman_multihost: true
diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml
index 35fd4b652f..b74fdab347 100644
--- a/inventory/service/groups.yaml
+++ b/inventory/service/groups.yaml
@@ -24,11 +24,13 @@ groups:
- kdc03.openstack.org
- eavesdrop01.opendev.org
- paste01.opendev.org
+ - lists01.opendev.org
# These are test specific hosts that we add to the backup
# group to mimic as much as possible what their prod version
# end up doing.
- gitea99.opendev.org
- review99.opendev.org
+ - lists99.opendev.org
# All these servers are "special-cased" in specifically
# as they are puppet and should be replaced "soon"
- lists.openstack.org
@@ -89,6 +91,7 @@ groups:
- keycloak[0-9]*.opendev.org
- lists.katacontainers.io
- lists.openstack.org
+ - lists[0-9]*.opendev.org
- meetpad[0-9]*.opendev.org
- mirror[0-9]*.opendev.org
- nb[0-9]*.opendev.org
@@ -100,8 +103,10 @@ groups:
- translate[0-9]*.open*.org
- zuul[0-9]*.opendev.org
mailman:
- - lists*.katacontainers.io
- - lists*.open*.org
+ - lists.katacontainers.io
+ - lists.openstack.org
+ mailman3:
+ - lists[0-9]*.opendev.org
meetpad:
- meetpad[0-9]*.opendev.org
mirror:
diff --git a/inventory/service/host_vars/lists01.opendev.org.yaml b/inventory/service/host_vars/lists01.opendev.org.yaml
new file mode 100644
index 0000000000..57390c8732
--- /dev/null
+++ b/inventory/service/host_vars/lists01.opendev.org.yaml
@@ -0,0 +1,225 @@
+mm_domains: 'lists.openstack.org:lists.zuul-ci.org:lists.airshipit.org:lists.starlingx.io:lists.opendev.org:lists.openinfra.dev:lists.katacontainers.io'
+exim_local_domains: "@:{{ mm_domains }}"
+exim_enable_spf: true
+exim_aliases:
+ root: "{{ ','.join(listadmins|default([])) }}"
+ interop-wg: openstack-discuss
+ openstack: openstack-discuss
+ openstack-dev: openstack-discuss
+ openstack-infra: openstack-discuss
+ openstack-operators: openstack-discuss
+ openstack-security: openstack-discuss
+ openstack-sigs: openstack-discuss
+ openstack-tc: openstack-discuss
+ user-committee: openstack-discuss
+ airship-discuss-owner: spam
+ community-owner: spam
+ edge-computing-owner: spam
+ foundation-board-confidential-owner: spam
+ foundation-board-owner: spam
+ foundation-owner: spam
+ legal-discuss-owner: spam
+ mailman-owner: spam
+ marketing-owner: spam
+ openstack-announce-owner: spam
+ openstack-docs-owner: spam
+ openstack-fr-owner: spam
+ openstack-i18n-owner: spam
+ openstack-infra-owner: spam
+ openstack-ko-owner: spam
+ openstack-qa-owner: spam
+ product-wg-owner: spam
+ user-committee-owner: spam
+ spam: ':fail: delivery temporarily disabled due to ongoing spam flood'
+ # TODO It would be better to bypass verification for postorius@listdomain
+ # and set a :fail: rule for anyone trying to send email to this addr.
+ # But that requires updating our main exim config so that needs more thought.
+ postorius: ':blackhole: outgoing email only from this address'
+exim_domain_aliases:
+ community@lists.openstack.org: community@lists.openinfra.dev
+ edge-computing@lists.openstack.org: edge-computing@lists.opendev.org
+ foundation@lists.openstack.org: foundation@lists.openinfra.dev
+ foundation-board@lists.openstack.org: foundation-board@lists.openinfra.dev
+ foundation-board-confidential@lists.openstack.org: foundation-board-confidential@lists.openinfra.dev
+ goldmembers@lists.openstack.org: goldmembers@lists.openinfra.dev
+ marketing@lists.openstack.org: marketing@lists.openinfra.dev
+ staff@lists.openstack.org: staff@lists.openinfra.dev
+ summit-programming-committee@lists.openinfra.dev: summit-track-chairs@lists.openinfra.dev
+ summitsponsors@lists.openstack.org: summitsponsors@lists.openinfra.dev
+mailman_sites:
+ # First entry in this list is the primary web domain
+ - listdomain: lists.opendev.org
+ install_languages: ['en']
+ lists:
+ - name: computing-force-network
+ description: 'Organizing efforts around Computing Force Network related area'
+ owner: 'niujie@outlook.com'
+ - name: edge-computing
+ description: 'Organizing efforts around the edge-computing focus area.'
+ owner: 'ildiko@openinfra.dev'
+ - name: floss-mooc
+ description: 'Discussions & Coordination around the FLOSS MOOC being collaboratively developed here: https://gitlab.com/mooc-floss/mooc-floss'
+ owner: 'knelson@openinfra.dev'
+ - name: nbmp-discuss
+ description: 'Collaborating on Network Based Media Processing related platform and infrastructure systems usage and development.'
+ owner: 'ildiko@openstack.org'
+ - name: openinfralabs
+ description: 'Discussion of the OpenInfra Labs academic and research resource sharing effort'
+ owner: 'mnaser@vexxhost.com'
+ - name: rust-vmm
+ description: 'Collaborating on Rust-based virtual machine monitors.'
+ owner: 'claire@openstack.org'
+ - name: rustyk8s
+ description: 'Collaborating on Rust-based Kubernetes API.'
+ owner: 'allison@lohutok.net'
+ - name: service-announce
+ description: 'Announcement list for OpenDev services.'
+ owner: 'cboylan@sapwetik.org'
+ - name: service-discuss
+ description: 'Discussion list for OpenDev services.'
+ owner: 'cboylan@sapwetik.org'
+ - name: service-incident
+ description: 'Private list for OpenDev incident coordination.'
+ owner: 'cboylan@sapwetik.org'
+ private: true
+ # The domains and lists below are currently commented out as we intend on
+ # deploying a single domain and its lists at a time starting with
+ # lists.opendev.org. As we deploy other domains we can uncomment these
+ # blocks. Double check no new lists are been added or removed first.
+ #- listdomain: lists.airshipit.org
+ # install_languages: ['en']
+ # lists:
+ # - name: airship-announce
+ # description: 'Announcements of Airship releases and other important information.'
+ # owner: 'jonathan@openstack.org'
+ # - name: airship-discuss
+ # description: 'Discussion of Airship usage and development.'
+ # owner: 'jonathan@openstack.org'
+ # - name: airship-embargo-notice
+ # description: 'Embargoed security vulnerability announcements for Airship consumers.'
+ # owner: 'andrew.walters@att.com'
+ # private: true
+ # - name: airship-job-failures
+ # description: 'Notification messages for failures from CICD jobs.'
+ # owner: 'roman.gorshunov@att.com'
+ # - name: airship-security
+ # description: 'Public Airship security advisories.'
+ # owner: 'andrew.walters@att.com'
+ #- listdomain: lists.katacontainers.io
+ # install_languages: ['en']
+ # lists:
+ # - name: embargo-notice
+ # description: 'Announcements of embargoed notices for the Kata Containers project'
+ # owner: 'jonathan@openstack.org'
+ # private: true
+ # - name: kata-dev
+ # description: 'Kata Containers Development Mailing List (not for usage questions)'
+ # owner: 'jonathan@openstack.org'
+ # - name: kata-hypervisor
+ # description: 'Discussion of security and virtualization targeted at container use cases'
+ # owner: 'jonathan@openstack.org'
+ #- listdomain: lists.openinfra.dev
+ # install_languages: ['en']
+ # lists:
+ # - name: community
+ # description: 'The OpenInfra Community team is the main contact point for anybody running a local OpenInfra Group.'
+ # owner: 'allison@openinfra.dev'
+ # - name: foundation
+ # description: 'General discussion list for activities of the OpenInfra Foundation'
+ # owner: 'jonathan@openinfra.dev'
+ # - name: foundation-board
+ # description: 'OpenInfra Foundation Board of Directors'
+ # owner: 'jonathan@openinfra.dev'
+ # - name: foundation-board-confidential
+ # description: 'OpenInfra Foundation Board of Directors'
+ # owner: 'jonathan@openinfra.dev'
+ # private: true
+ # - name: goldmembers
+ # description: 'The discussion list for Gold Members of the OpenInfra Foundation'
+ # owner: 'jonathan@openinfra.dev'
+ # private: true
+ # - name: marketing
+ # description: 'The OpenInfra Marketing list is the meant to facilitate discussion and best practice sharing among marketers and event organizers in the OpenInfra community.'
+ # owner: 'allison@openinfra.dev'
+ # - name: staff
+ # description: 'Private list for OpenInfra Foundation staff members'
+ # owner: 'mark@openinfra.dev'
+ # private: true
+ # - name: summit-track-chairs
+ # description: 'OpenInfra Summit track chair communications'
+ # owner: 'erin@openinfra.dev'
+ # private: true
+ # - name: summitsponsors
+ # description: 'Coordination among OpenInfra Summit event sponsors'
+ # owner: 'erin@openinfra.dev'
+ # private: true
+ #- listdomain: lists.openstack.org
+ # install_languages: ['de', 'fr', 'it', 'ko', 'ru', 'vi', 'zh_TW']
+ # lists:
+ # - name: embargo-notice
+ # description: 'Announcements to stakeholders for embargoed security vulnerabilities.'
+ # owner: 'fungi@yuggoth.org'
+ # private: true
+ # - name: legal-discuss
+ # description: 'Discussions on legal matters related to the project'
+ # owner: 'thierry@openinfra.dev'
+ # - name: openstack-announce
+ # description: 'Key announcements about OpenStack & Security advisories'
+ # owner: 'fungi@yuggoth.org'
+ # - name: openstack-discuss
+ # description: 'Discussion of OpenStack usage and development.'
+ # owner: 'fungi@yuggoth.org'
+ # - name: openstack-es
+ # description: 'Lista de correo acerca de OpenStack en español'
+ # owner: 'flavio@redhat.com'
+ # - name: openstack-fr
+ # description: 'List of the OpenStack french user group'
+ # owner: 'erwan@erwan.com'
+ # - name: openstack-hpc
+ # description: 'High-Performance Computing OpenStack List'
+ # owner: 'brian.schott@nimbisservices.com'
+ # - name: openstack-i18n
+ # description: 'List of the OpenStack Internationalization team.'
+ # owner: 'guoyingc@cn.ibm.com'
+ # - name: openstack-it
+ # description: 'Discussioni su OpenStack in italiano'
+ # owner: 'stefano@openstack.org'
+ # - name: openstack-ko
+ # description: 'OpenStack Korea Community Discussions in Korean (오픈스택 한국 커뮤니티 메일링리스트)'
+ # owner: 'ianyrchoi@gmail.com'
+ # - name: openstack-mentoring
+ # description: 'List to coordinate interactions between mentors and mentees of the OpenStack mentoring program. Also for questions about the mentoring program (i.e. how to get involved, how it works, etc.'
+ # owner: 'amy@demarco.com'
+ # - name: openstack-stable-maint
+ # description: 'A mailing list for the OpenStack Stable Branch test reports.'
+ # owner: 'tony@bakeyournoodle.com'
+ # - name: openstack-zh
+ # description: 'OpenStack社区中文讨论群组'
+ # owner: 'yeluaiesec@gmail.com'
+ # - name: release-announce
+ # description: 'Announcement of official OpenStack releases.'
+ # owner: 'thierry@openstack.org'
+ # - name: release-job-failures
+ # description: 'Notification messages for failures from release-related build jobs.'
+ # owner: 'doug@doughellmann.com'
+ #- listdomain: lists.starlingx.io
+ # install_languages: ['en']
+ # lists:
+ # - name: starlingx-announce
+ # description: 'Announcements of StarlingX releases and other important information.'
+ # owner: 'jonathan@openstack.org'
+ # - name: starlingx-discuss
+ # description: 'Discussion of StarlingX usage and development.'
+ # owner: 'jonathan@openstack.org'
+ #- listdomain: lists.zuul-ci.org
+ # install_languages: ['en']
+ # lists:
+ # - name: zuul-announce
+ # description: 'Announcements of Zuul releases and other important information.'
+ # owner: 'corvus@inaugust.com'
+ # - name: zuul-discuss
+ # description: 'Discussion of Zuul usage and development.'
+ # owner: 'corvus@inaugust.com'
+ # - name: zuul-jobs-failures
+ # description: 'Gets notifications about zuul-jobs periodic job failures.'
+ # owner: 'ssbarnea@redhat.com'
diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
index 1d5cf9d2a7..c71885db35 100644
--- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
+++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
@@ -45,6 +45,9 @@
- name: letsencrypt updated lists-openstack-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+- name: letsencrypt updated lists-opendev-org-main
+ include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
# Static
- name: letsencrypt updated static-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
diff --git a/playbooks/roles/mailman3/README.rst b/playbooks/roles/mailman3/README.rst
new file mode 100644
index 0000000000..7fd9224854
--- /dev/null
+++ b/playbooks/roles/mailman3/README.rst
@@ -0,0 +1 @@
+Role to configure mailman3.
diff --git a/playbooks/roles/mailman3/files/99-max_allowed_packet.cnf b/playbooks/roles/mailman3/files/99-max_allowed_packet.cnf
new file mode 100644
index 0000000000..0cc9b8630c
--- /dev/null
+++ b/playbooks/roles/mailman3/files/99-max_allowed_packet.cnf
@@ -0,0 +1,10 @@
+[mysqldump]
+# Default is 24MB which is not large enough for all mailman attachments.
+# This affects clients including mysqldump. It is larger than the server
+# side because mysqldump apparently can do larger packets to insert blobs.
+max_allowed_packet=256M
+
+[mysqld]
+# Default is 16MB which is not large enough for all mailman attachments.
+# This affects the server side.
+max_allowed_packet=128M
diff --git a/playbooks/roles/mailman3/files/web-settings.py b/playbooks/roles/mailman3/files/web-settings.py
new file mode 100644
index 0000000000..8e2905e2d4
--- /dev/null
+++ b/playbooks/roles/mailman3/files/web-settings.py
@@ -0,0 +1,429 @@
+# This file has been copied from:
+# https://github.com/maxking/docker-mailman/blob/2693386453ff3865b7c106c6aa456b683bd3bf08/web/mailman-web/settings.py
+# In order to override the ALLOWED_HOSTS setting.
+#
+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Mailman Suite.
+#
+# Mailman Suite is free sofware: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Mailman Suite is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+
+# You should have received a copy of the GNU General Public License along
+# with Mailman Suite. If not, see .
+"""
+Django Settings for Mailman Suite (hyperkitty + postorius)
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.8/ref/settings/
+"""
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+import dj_database_url
+import sys
+from socket import gethostbyname
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = os.environ.get('SECRET_KEY')
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ADMINS = (
+ ('Mailman Suite Admin', 'root@localhost'),
+)
+
+SITE_ID = 1
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [
+ "localhost", # Archiving API from Mailman, keep it.
+ "127.0.0.1", # Archiving API from Mailman, keep it. OpenDev edit
+ # "lists.your-domain.org",
+ # Add here all production URLs you may have.
+ # The next two entries are commented out to prevent name resolution
+ # problems. This is an opendev local edit.
+ # Note we cannot use settings_local.py as this entry is evaluated at
+ # import time.
+ #"mailman-web",
+ #gethostbyname("mailman-web"),
+ os.environ.get('SERVE_FROM_DOMAIN'),
+ #os.environ.get('DJANGO_ALLOWED_HOSTS'),
+]
+
+# We have modified handling of DJANGO_ALLOWED_HOSTS here to deserialize a
+# list of hosts into a python list of strings.
+django_allowed_hosts = os.environ.get('DJANGO_ALLOWED_HOSTS')
+if django_allowed_hosts:
+ ALLOWED_HOSTS.extend(django_allowed_hosts.split(':'))
+
+# Mailman API credentials
+MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001')
+MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin')
+MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass')
+MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY')
+MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),)
+
+# Application definition
+
+INSTALLED_APPS = []
+DEFAULT_APPS = [
+ 'hyperkitty',
+ 'postorius',
+ 'django_mailman3',
+ # Uncomment the next line to enable the admin:
+ 'django.contrib.admin',
+ # Uncomment the next line to enable admin documentation:
+ # 'django.contrib.admindocs',
+ 'django.contrib.auth',
+ 'django.contrib.contenttypes',
+ 'django.contrib.sessions',
+ 'django.contrib.sites',
+ 'django.contrib.messages',
+ 'django.contrib.staticfiles',
+ 'rest_framework',
+ 'django_gravatar',
+ 'compressor',
+ 'haystack',
+ 'django_extensions',
+ 'django_q',
+ 'allauth',
+ 'allauth.account',
+ 'allauth.socialaccount',
+]
+
+MAILMAN_WEB_SOCIAL_AUTH = [
+ 'django_mailman3.lib.auth.fedora',
+ 'allauth.socialaccount.providers.openid',
+ 'allauth.socialaccount.providers.github',
+ 'allauth.socialaccount.providers.gitlab',
+ 'allauth.socialaccount.providers.google',
+]
+
+MIDDLEWARE = (
+ 'django.contrib.sessions.middleware.SessionMiddleware',
+ 'django.middleware.common.CommonMiddleware',
+ 'django.middleware.csrf.CsrfViewMiddleware',
+ 'django.middleware.locale.LocaleMiddleware',
+ 'django.contrib.auth.middleware.AuthenticationMiddleware',
+ 'django.contrib.messages.middleware.MessageMiddleware',
+ 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+ 'django.middleware.security.SecurityMiddleware',
+ 'django_mailman3.middleware.TimezoneMiddleware',
+ 'postorius.middleware.PostoriusMiddleware',
+)
+
+ROOT_URLCONF = 'urls'
+
+TEMPLATES = [
+ {
+ 'BACKEND': 'django.template.backends.django.DjangoTemplates',
+ 'DIRS': [],
+ 'APP_DIRS': True,
+ 'OPTIONS': {
+ 'context_processors': [
+ 'django.template.context_processors.debug',
+ 'django.template.context_processors.i18n',
+ 'django.template.context_processors.media',
+ 'django.template.context_processors.static',
+ 'django.template.context_processors.tz',
+ 'django.template.context_processors.csrf',
+ 'django.template.context_processors.request',
+ 'django.contrib.auth.context_processors.auth',
+ 'django.contrib.messages.context_processors.messages',
+ 'django_mailman3.context_processors.common',
+ 'hyperkitty.context_processors.common',
+ 'postorius.context_processors.postorius',
+ ],
+ },
+ },
+]
+
+WSGI_APPLICATION = 'wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+# dj_database_url uses $DATABASE_URL environment variable to create a
+# django-style-config-dict.
+# https://github.com/kennethreitz/dj-database-url
+DATABASES = {
+ 'default': dj_database_url.config(conn_max_age=600)
+}
+
+# If you're behind a proxy, use the X-Forwarded-Host header
+# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
+USE_X_FORWARDED_HOST = True
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
+AUTH_PASSWORD_VALIDATORS = [
+ {
+ 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+ },
+ {
+ 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+ },
+]
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.8/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+STATIC_ROOT = '/opt/mailman-web-data/static'
+
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+ 'django.contrib.staticfiles.finders.FileSystemFinder',
+ 'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+ 'compressor.finders.CompressorFinder',
+)
+
+
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
+
+LOGIN_URL = 'account_login'
+LOGIN_REDIRECT_URL = 'list_index'
+LOGOUT_URL = 'account_logout'
+
+
+# Use SERVE_FROM_DOMAIN as the default domain in the email.
+hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local')
+DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname)
+SERVER_EMAIL = 'root@{}'.format(hostname)
+
+# Change this when you have a real email backend
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST = os.environ.get('SMTP_HOST', '')
+EMAIL_PORT = os.environ.get('SMTP_PORT', 25)
+EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '')
+EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '')
+EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False)
+EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False)
+
+# Compatibility with Bootstrap 3
+from django.contrib.messages import constants as messages # flake8: noqa
+MESSAGE_TAGS = {
+ messages.ERROR: 'danger'
+}
+
+
+#
+# Social auth
+#
+AUTHENTICATION_BACKENDS = (
+ 'django.contrib.auth.backends.ModelBackend',
+ 'allauth.account.auth_backends.AuthenticationBackend',
+)
+
+# Django Allauth
+ACCOUNT_AUTHENTICATION_METHOD = "username_email"
+ACCOUNT_EMAIL_REQUIRED = True
+ACCOUNT_EMAIL_VERIFICATION = "mandatory"
+# You probably want https in production, but this is a dev setup file
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
+ACCOUNT_UNIQUE_EMAIL = True
+
+SOCIALACCOUNT_PROVIDERS = {
+ 'openid': {
+ 'SERVERS': [
+ dict(id='yahoo',
+ name='Yahoo',
+ openid_url='http://me.yahoo.com'),
+ ],
+ },
+ 'google': {
+ 'SCOPE': ['profile', 'email'],
+ 'AUTH_PARAMS': {'access_type': 'online'},
+ },
+ 'facebook': {
+ 'METHOD': 'oauth2',
+ 'SCOPE': ['email'],
+ 'FIELDS': [
+ 'email',
+ 'name',
+ 'first_name',
+ 'last_name',
+ 'locale',
+ 'timezone',
+ ],
+ 'VERSION': 'v2.4',
+ },
+}
+
+
+# django-compressor
+# https://pypi.python.org/pypi/django_compressor
+#
+COMPRESS_PRECOMPILERS = (
+ ('text/less', 'lessc {infile} {outfile}'),
+ ('text/x-scss', 'sassc -t compressed {infile} {outfile}'),
+ ('text/x-sass', 'sassc -t compressed {infile} {outfile}'),
+)
+
+# On a production setup, setting COMPRESS_OFFLINE to True will bring a
+# significant performance improvement, as CSS files will not need to be
+# recompiled on each requests. It means running an additional "compress"
+# management command after each code upgrade.
+# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression
+# COMPRESS_OFFLINE = True
+
+#
+# Full-text search engine
+#
+HAYSTACK_CONNECTIONS = {
+ 'default': {
+ 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+ 'PATH': "/opt/mailman-web-data/fulltext_index",
+ # You can also use the Xapian engine, it's faster and more accurate,
+ # but requires another library.
+ # http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian
+ # Example configuration for Xapian:
+ #'ENGINE': 'xapian_backend.XapianEngine'
+ },
+}
+
+import sys
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+ 'version': 1,
+ 'disable_existing_loggers': False,
+ 'filters': {
+ 'require_debug_false': {
+ '()': 'django.utils.log.RequireDebugFalse'
+ }
+ },
+ 'handlers': {
+ 'mail_admins': {
+ 'level': 'ERROR',
+ 'filters': ['require_debug_false'],
+ 'class': 'django.utils.log.AdminEmailHandler'
+ },
+ 'file':{
+ 'level': 'INFO',
+ 'class': 'logging.handlers.RotatingFileHandler',
+ #'class': 'logging.handlers.WatchedFileHandler',
+ 'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'),
+ 'formatter': 'verbose',
+ },
+ 'console': {
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'simple',
+ 'level': 'INFO',
+ 'stream': sys.stdout,
+ },
+ },
+ 'loggers': {
+ 'django.request': {
+ 'handlers': ['mail_admins', 'file'],
+ 'level': 'INFO',
+ 'propagate': True,
+ },
+ 'django': {
+ 'handlers': ['file'],
+ 'level': 'INFO',
+ 'propagate': True,
+ },
+ 'hyperkitty': {
+ 'handlers': ['file'],
+ 'level': 'INFO',
+ 'propagate': True,
+ },
+ 'postorius': {
+ 'handlers': ['file'],
+ 'level': 'INFO',
+ 'propagate': True
+ },
+ },
+ 'formatters': {
+ 'verbose': {
+ 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
+ },
+ 'simple': {
+ 'format': '%(levelname)s %(message)s'
+ },
+ },
+}
+
+
+if os.environ.get('LOG_TO_CONSOLE') == 'yes':
+ LOGGING['loggers']['django']['handlers'].append('console')
+ LOGGING['loggers']['django.request']['handlers'].append('console')
+
+# HyperKitty-specific
+#
+# Only display mailing-lists from the same virtual host as the webserver
+FILTER_VHOST = False
+
+
+Q_CLUSTER = {
+ 'timeout': 300,
+ 'retry': 300,
+ 'save_limit': 100,
+ 'orm': 'default',
+}
+
+POSTORIUS_TEMPLATE_BASE_URL = os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000')
+
+DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache')
+DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte
+
+CACHES = {
+ 'default': {
+ 'BACKEND': 'diskcache.DjangoCache',
+ 'LOCATION': DISKCACHE_PATH,
+ 'OPTIONS': {
+ 'size_limit': DISKCACHE_SIZE,
+ },
+ },
+}
+
+try:
+ from settings_local import *
+except ImportError:
+ pass
+
+# Compatibility for older installs that override INSTALLED_APPS
+if not INSTALLED_APPS:
+ INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH
diff --git a/playbooks/roles/mailman3/files/web-settings_local.py b/playbooks/roles/mailman3/files/web-settings_local.py
new file mode 100644
index 0000000000..a755892a8a
--- /dev/null
+++ b/playbooks/roles/mailman3/files/web-settings_local.py
@@ -0,0 +1,21 @@
+# Override default index system. Xapian will be the default in the next
+# major release of mailman3, but is currently recommended.
+HAYSTACK_CONNECTIONS = {
+ 'default': {
+ 'ENGINE': 'xapian_backend.XapianEngine',
+ 'PATH': "/opt/mailman-web-data/fulltext_index",
+ },
+}
+
+# Disable Gravatar integration since it violates privacy expectations
+HYPERKITTY_ENABLE_GRAVATAR = False
+
+# This disables web auth using Google, GitHub, Gitlab, Yahoo,
+# Fedora, and generic OpenID.
+# TODO: In the future we will want to enable this specifically for
+# our keycloak server only.
+MAILMAN_WEB_SOCIAL_AUTH = []
+
+FILTER_VHOST = True
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
diff --git a/playbooks/roles/mailman3/handlers/main.yaml b/playbooks/roles/mailman3/handlers/main.yaml
new file mode 100644
index 0000000000..18f5b3bbc9
--- /dev/null
+++ b/playbooks/roles/mailman3/handlers/main.yaml
@@ -0,0 +1,9 @@
+- name: mailman restart apache2
+ service:
+ name: apache2
+ state: restarted
+
+- name: mailman reload apache2
+ service:
+ name: apache2
+ state: reloaded
diff --git a/playbooks/roles/mailman3/tasks/create_lists.yaml b/playbooks/roles/mailman3/tasks/create_lists.yaml
new file mode 100644
index 0000000000..349f0c27ba
--- /dev/null
+++ b/playbooks/roles/mailman3/tasks/create_lists.yaml
@@ -0,0 +1,114 @@
+- name: Check if domain exists
+ uri:
+ url: 'http://localhost:8001/3.1/domains/{{ mm_site.listdomain }}'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: GET
+ body_format: json
+ status_code: [200, 404]
+ register: domain_exists
+ no_log: true
+
+- name: Create list domain in mm3
+ when: domain_exists.status == 404
+ uri:
+ url: 'http://localhost:8001/3.1/domains'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: POST
+ body_format: json
+ body:
+ mail_host: "{{ mm_site.listdomain }}"
+ status_code: [201]
+ no_log: true
+
+- name: Check if list exists
+ uri:
+ url: 'http://localhost:8001/3.1/lists/{{ mm_list.name }}@{{ mm_site.listdomain }}'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: GET
+ body_format: json
+ status_code: [200, 404]
+ register: list_exists
+ loop: "{{ mm_site.lists }}"
+ loop_control:
+ loop_var: mm_list
+ no_log: true
+
+- name: Create lists in mm3
+ when: list_exists.results[exists_idx].status == 404
+ uri:
+ url: 'http://localhost:8001/3.1/lists'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: POST
+ body_format: json
+ body:
+ fqdn_listname: "{{ mm_list.name }}@{{ mm_site.listdomain }}"
+ style_name: "{{ mm_list.private | default('false') | bool | ternary('private-default', 'legacy-default') }}"
+ status_code: [201]
+ loop: "{{ mm_site.lists }}"
+ loop_control:
+ loop_var: mm_list
+ index_var: exists_idx
+ no_log: true
+
+- name: Set list properties in mm3
+ when: list_exists.results[exists_idx].status == 404
+ uri:
+ url: 'http://localhost:8001/3.1/lists/{{ mm_list.name }}@{{ mm_site.listdomain }}/config'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: PATCH
+ body_format: json
+ body:
+ description: "{{ mm_list.description }}"
+ advertised: "{{ mm_list.private | default('false') | bool | ternary('false', 'true') }}"
+ # TODO enable this when lynx is present on the container images
+ # convert_html_to_plaintext: "true"
+ process_bounces: "false"
+ filter_extensions:
+ - "exe"
+ - "bat"
+ - "cmd"
+ - "com"
+ - "pif"
+ - "scr"
+ - "vbs"
+ - "cpl"
+ pass_types:
+ - "multipart/mixed"
+ - "multipart/alternative"
+ - "text/plain"
+ status_code: [204]
+ loop: "{{ mm_site.lists }}"
+ loop_control:
+ loop_var: mm_list
+ index_var: exists_idx
+ no_log: true
+
+- name: Set list owner in mm3
+ when: list_exists.results[exists_idx].status == 404
+ uri:
+ url: 'http://localhost:8001/3.1/members'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: POST
+ body_format: json
+ body:
+ list_id: "{{ mm_list.name }}.{{ mm_site.listdomain }}"
+ subscriber: "{{ mm_list.owner }}"
+ role: "owner"
+ status_code: [201]
+ loop: "{{ mm_site.lists }}"
+ loop_control:
+ loop_var: mm_list
+ index_var: exists_idx
+ no_log: true
diff --git a/playbooks/roles/mailman3/tasks/main.yaml b/playbooks/roles/mailman3/tasks/main.yaml
new file mode 100644
index 0000000000..07360635f4
--- /dev/null
+++ b/playbooks/roles/mailman3/tasks/main.yaml
@@ -0,0 +1,278 @@
+# The old mailman2 exim config refers to this file. Write it out
+# to make basic testing happy, but we may need to clean it up or
+# modify it for mailman3.
+- name: Write /etc/aliases.domain
+ template:
+ src: "domain_aliases.j2"
+ dest: "/etc/aliases.domain"
+ mode: '0444'
+
+- name: Create Mailman Group
+ group:
+ name: mailman
+ gid: 10010
+ system: yes
+
+- name: Create Mailman User
+ user:
+ name: mailman
+ uid: 10010
+ comment: Mailman User
+ shell: /bin/bash
+ home: /var/lib/mailman
+ group: mailman
+ create_home: yes
+ system: yes
+
+#### Install Mailman ####
+
+- name: Ensure Mailman core volume directory exists
+ file:
+ state: directory
+ path: "/var/lib/mailman/core"
+ # TODO: undo for https://github.com/maxking/docker-mailman/issues/550
+ owner: 100
+ group: 65533
+ mode: '0755'
+
+- name: Ensure Mailman database volume directory exists
+ file:
+ state: directory
+ path: "/var/lib/mailman/database"
+ # TODO: undo for https://github.com/maxking/docker-mailman/issues/550
+ owner: 999
+ group: 999
+ mode: '0755'
+
+- name: Ensure Mailman web volume directories exist
+ file:
+ state: directory
+ path: "/var/lib/mailman/{{ item }}"
+ # TODO: undo for https://github.com/maxking/docker-mailman/issues/550
+ owner: 100
+ group: 101
+ mode: '0755'
+ loop:
+ - import
+ - web
+ - web-data
+ - web-data/fulltext_index
+ - web-data/mm2archives
+
+- name: Copy our overridden settings.py for mailman-web
+ copy:
+ src: web-settings.py
+ dest: /var/lib/mailman/web/settings.py
+ # TODO: undo for https://github.com/maxking/docker-mailman/issues/550
+ owner: 100
+ group: 101
+ mode: '0644'
+
+- name: Copy our settings_local.py for mailman-web
+ copy:
+ src: web-settings_local.py
+ dest: /var/lib/mailman/web-data/settings_local.py
+ # TODO: undo for https://github.com/maxking/docker-mailman/issues/550
+ owner: 100
+ group: 101
+ mode: '0644'
+
+- name: Copy our max_allowed_packet override config
+ copy:
+ src: 99-max_allowed_packet.cnf
+ dest: /var/lib/mailman/99-max_allowed_packet.cnf
+ owner: 999
+ group: 999
+ mode: '0644'
+
+- name: Ensure /etc/mailman-compose directory
+ file:
+ state: directory
+ path: /etc/mailman-compose
+ mode: '0755'
+
+- name: Put docker-compose file in place
+ template:
+ src: docker-compose.yaml.j2
+ dest: /etc/mailman-compose/docker-compose.yaml
+ mode: '0600'
+
+- name: Run docker-compose pull
+ shell:
+ cmd: docker-compose pull
+ chdir: /etc/mailman-compose/
+
+- name: Run docker-compose up
+ shell:
+ cmd: docker-compose up -d
+ chdir: /etc/mailman-compose/
+
+- name: Run docker prune to cleanup unneeded images
+ shell:
+ cmd: docker image prune -f
+
+- name: Install apache2
+ package:
+ name:
+ - apache2
+ - apache2-utils
+ state: present
+
+- name: Apache modules
+ apache2_module:
+ state: present
+ name: "{{ a2_mod }}"
+ loop:
+ - authz_host
+ - proxy
+ - proxy_uwsgi
+ - ssl
+ - rewrite
+ loop_control:
+ loop_var: a2_mod
+ notify: mailman restart apache2
+
+- name: Make sure packaged default site disabled
+ command: a2dissite 000-default.conf
+ args:
+ removes: /etc/apache2/sites-enabled/000-default.conf
+
+- name: Create mailman vhost config
+ template:
+ src: mailman.vhost.j2
+ dest: "/etc/apache2/sites-enabled/50-{{ mailman_sites.0.listdomain }}.conf"
+ owner: root
+ group: root
+ mode: '0644'
+ notify: mailman reload apache2
+
+- name: Enable apache2 server
+ service:
+ name: "apache2"
+ enabled: yes
+
+#### Configure Mailman Services ####
+
+- name: Wait for mm3 REST API to be up and running
+ uri:
+ url: 'http://localhost:8001/3.1/domains'
+ url_username: restadmin
+ url_password: "{{ mailman3_rest_password }}"
+ force_basic_auth: yes
+ method: GET
+ register: mm_rest_api_up
+ delay: 1
+ retries: 300
+ until: mm_rest_api_up and mm_rest_api_up.status == 200
+ no_log: true
+
+# It has been difficult to nail down a reliable mathod for determining
+# when the database is sufficiently populated that we can create the django
+# admin user. We apply a number of approaches in response to this. If we
+# can identify a single method that is reliable this list can be trimmed.
+- name: Wait for DB to be populated
+ command: >
+ docker exec mailman-compose_database_1 bash -c
+ 'mysql -u mailman -p"$MYSQL_PASSWORD" -D mailmandb -e
+ "SHOW TABLES LIKE \"auth_user\";"'
+ register: django_db_exists
+ delay: 1
+ retries: 300
+ until: django_db_exists.stdout_lines | length > 1 and django_db_exists.stdout_lines[1] == "auth_user"
+
+- name: Wait for DB to be populated second approach
+ command: >
+ docker exec mailman-core alembic -c /usr/lib/python3.9/site-packages/mailman/config/alembic.cfg current
+ register: alembic_version
+ delay: 1
+ retries: 300
+ until: alembic_version.stdout_lines | length > 0 and "(head)" in alembic_version.stdout_lines[0]
+
+- name: Wait for DB to be populated third approach
+ shell: >
+ docker exec mailman-web bash -c
+ 'python3 manage.py showmigrations' |
+ grep -q '^ \[ \] [0-9]\+_.*'
+ register: django_db_migrations
+ delay: 1
+ retries: 300
+ failed_when: false
+ # When grep stops matching the empty '[ ]' that indicates all migrations
+ # are marked with '[X]' and are complete. Grep returns non zero when we
+ # reach this point.
+ until: django_db_migrations.rc != 0
+
+- name: Check if django admin user exists
+ command: >
+ docker exec mailman-compose_database_1 bash -c
+ 'mysql -u mailman -p"$MYSQL_PASSWORD" -D mailmandb -e
+ "SELECT COUNT(id) FROM auth_user WHERE id = 1 AND is_superuser = 1;"'
+ register: django_admin_exists
+
+- name: Create django admin user
+ when: django_admin_exists.stdout_lines[1] == "0"
+ command: >
+ docker exec mailman-web bash -c
+ "DJANGO_SUPERUSER_PASSWORD={{ mailman3_admin_password }}
+ python3 manage.py createsuperuser --no-input
+ --username {{ mailman3_admin_user }}
+ --email '{{ mailman3_admin_email }}'"
+ no_log: true
+
+- name: Create lists in mm3
+ include_tasks: create_lists.yaml
+ loop: "{{ mailman_sites }}"
+ loop_control:
+ loop_var: mm_site
+
+#### Logrotate for service logs ####
+
+- name: Rotate mailman logs
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_rotate: 90
+ logrotate_file_name: '/var/lib/mailman/web-data/logs/*.log'
+
+#### Database Backups ####
+
+- name: Create db backup dest
+ file:
+ state: directory
+ path: /var/backups/mailman-mariadb
+ mode: 0700
+ owner: root
+ group: root
+
+- name: Set up cron job to backup the database
+ cron:
+ name: mailman-db-backup
+ state: present
+ user: root
+ job: >
+ /usr/local/bin/docker-compose -f /etc/mailman-compose/docker-compose.yaml exec -T database
+ bash -c '/usr/bin/mysqldump --opt --databases mailmandb --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
+ gzip -9 > /var/backups/mailman-mariadb/mailman-mariadb.sql.gz
+ minute: 14
+ hour: 5
+
+- name: Rotate db backups
+ include_role:
+ name: logrotate
+ vars:
+ logrotate_file_name: /var/backups/mailman-mariadb/mailman-mariadb.sql.gz
+ logrotate_compress: false
+
+- name: Setup db backup streaming job
+ block:
+ - name: Create backup streaming config dir
+ file:
+ path: /etc/borg-streams
+ state: directory
+
+ - name: Create db streaming file
+ copy:
+ content: >-
+ /usr/local/bin/docker-compose -f /etc/mailman-compose/docker-compose.yaml exec -T mariadb
+ bash -c '/usr/bin/mysqldump --skip-extended-insert --databases mailmandb --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"'
+ dest: /etc/borg-streams/mysql
diff --git a/playbooks/roles/mailman3/templates/docker-compose.yaml.j2 b/playbooks/roles/mailman3/templates/docker-compose.yaml.j2
new file mode 100644
index 0000000000..a53ad65412
--- /dev/null
+++ b/playbooks/roles/mailman3/templates/docker-compose.yaml.j2
@@ -0,0 +1,71 @@
+# Adapted from https://github.com/maxking/docker-mailman/blob/2693386453ff3865b7c106c6aa456b683bd3bf08/docker-compose-mysql.yaml
+# which is an MIT licensed repo.
+
+version: '2'
+services:
+ mailman-core:
+ image: docker.io/maxking/mailman-core:0.4
+ restart: always
+ container_name: mailman-core
+ volumes:
+ - /var/lib/mailman/core:/opt/mailman/
+ - /var/lib/mailman/import:/opt/import
+ stop_grace_period: 30s
+ depends_on:
+ - database
+ environment:
+ - DATABASE_URL=mysql+pymysql://mailman:{{ mailman3_db_password }}@127.0.0.1:3306/mailmandb?charset=utf8mb4&use_unicode=1
+ - DATABASE_TYPE=mysql
+ - DATABASE_CLASS=mailman.database.mysql.MySQLDatabase
+ - HYPERKITTY_URL=http://127.0.0.1:8000/hyperkitty
+ - HYPERKITTY_API_KEY={{ mailman3_hyperkitty_api_key }}
+ - SMTP_HOST=localhost
+ - MM_HOSTNAME=localhost
+ - MAILMAN_REST_USER=restadmin
+ - MAILMAN_REST_PASSWORD={{ mailman3_rest_password }}
+ network_mode: host
+ #user: mailman
+
+ mailman-web:
+ image: docker.io/maxking/mailman-web:0.4
+ restart: always
+ container_name: mailman-web
+ depends_on:
+ - database
+ volumes:
+ - /var/lib/mailman/import:/opt/import
+ - /var/lib/mailman/web-data:/opt/mailman-web-data
+ - /var/lib/mailman/web/settings.py:/opt/mailman-web/settings.py
+ environment:
+ # Testing to see if these are really necessary
+ #- MAILMAN_ADMIN_USER={{ mailman3_admin_user }}
+ #- MAILMAN_ADMIN_EMAIL={{ mailman3_admin_email }}
+ - SERVE_FROM_DOMAIN=lists.opendev.org
+ - DJANGO_ALLOWED_HOSTS={{ mm_domains }}
+ - DATABASE_TYPE=mysql
+ - DATABASE_URL=mysql://mailman:{{ mailman3_db_password }}@127.0.0.1:3306/mailmandb?charset=utf8mb4
+ - HYPERKITTY_API_KEY={{ mailman3_hyperkitty_api_key }}
+ - SECRET_KEY={{ mailman3_django_secret_key }}
+ - DYLD_LIBRARY_PATH=/usr/local/mysql/lib/
+ - MAILMAN_HOSTNAME=localhost
+ - MAILMAN_REST_URL=http://127.0.0.1:8001
+ - MAILMAN_REST_USER=restadmin
+ - MAILMAN_REST_PASSWORD={{ mailman3_rest_password }}
+ - POSTORIUS_TEMPLATE_BASE_URL=http://127.0.0.1:8000
+ - SMTP_HOST=localhost
+ network_mode: host
+ #user: mailman
+
+ database:
+ environment:
+ MYSQL_DATABASE: mailmandb
+ MYSQL_USER: mailman
+ MYSQL_PASSWORD: {{ mailman3_db_password }}
+ MYSQL_ROOT_PASSWORD: {{ mailman3_db_root_password }}
+ image: docker.io/library/mariadb:10.6
+ restart: always
+ command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+ volumes:
+ - /var/lib/mailman/database:/var/lib/mysql
+ - /var/lib/mailman/99-max_allowed_packet.cnf:/etc/mysql/conf.d/99-max_allowed_packet.cnf:ro
+ network_mode: host
diff --git a/playbooks/roles/mailman3/templates/domain_aliases.j2 b/playbooks/roles/mailman3/templates/domain_aliases.j2
new file mode 100644
index 0000000000..05e531b03f
--- /dev/null
+++ b/playbooks/roles/mailman3/templates/domain_aliases.j2
@@ -0,0 +1,6 @@
+# /etc/aliases.domain
+{% for k, v in exim_domain_aliases|dictsort %}
+{% if v %}
+{{ k }}: {{ v }}
+{% endif %}
+{% endfor %}
diff --git a/playbooks/roles/mailman3/templates/mailman.vhost.j2 b/playbooks/roles/mailman3/templates/mailman.vhost.j2
new file mode 100644
index 0000000000..811b61c87c
--- /dev/null
+++ b/playbooks/roles/mailman3/templates/mailman.vhost.j2
@@ -0,0 +1,69 @@
+
+ ServerName {{ mailman_sites.0.listdomain }}
+ {% for site in mailman_sites[1:] -%}
+ ServerAlias {{ site.listdomain }}
+ {% endfor -%}
+
+ ErrorLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-error.log
+
+ # Possible values include: debug, info, notice, warn, error, crit,
+ # alert, emerg.
+ LogLevel warn
+
+ CustomLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-access.log combined
+
+ # Use mod rewrite to redirect as we want to preserve the FQDN for each
+ # mm3 vhost.
+ RewriteEngine On
+ RewriteRule "/(.*)" "https://%{HTTP_HOST}/$1" [R=301]
+
+
+
+ ServerName {{ mailman_sites.0.listdomain }}
+ {% for site in mailman_sites[1:] -%}
+ ServerAlias {{ site.listdomain }}
+ {% endfor -%}
+ ServerAdmin webmaster@openstack.org
+ ErrorLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-ssl-error.log
+ LogLevel warn
+ CustomLog ${APACHE_LOG_DIR}/{{ mailman_sites.0.listdomain }}-ssl-access.log combined
+
+ SSLEngine on
+ SSLProtocol All -SSLv2 -SSLv3
+ # Note: this list should ensure ciphers that provide forward secrecy
+ SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+ SSLHonorCipherOrder on
+
+ SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
+ SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
+
+ Alias /static /var/lib/mailman/web-data/static
+ Alias /favicon.ico /var/lib/mailman/web-data/static/hyperkitty/img/favicon.ico
+
+
+ Require local
+
+
+ RewriteEngine On
+ RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1"
+ RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/postorius/lists/$1.%{HTTP_HOST}/"
+ RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/postorius/lists/"
+
+ ProxyPassMatch ^/static/ !
+ ProxyPass "/" "uwsgi://localhost:8080/"
+
+
+ AllowOverride None
+ Order allow,deny
+ Allow from all
+ Require all granted
+
+
+
+ AllowOverride None
+ Order allow,deny
+ Allow from all
+ Require all granted
+
+
diff --git a/playbooks/service-lists3.yaml b/playbooks/service-lists3.yaml
new file mode 100644
index 0000000000..6d79d7172f
--- /dev/null
+++ b/playbooks/service-lists3.yaml
@@ -0,0 +1,10 @@
+# Maintaing a todo list here as a central accounting file
+# TODO Test mailman dmarc settings
+# TODO fix container uid/gid mismatch with bind mounted contents
+# this breaks xapian
+- hosts: "mailman3:!disabled"
+ name: "Configure mailman3 servers"
+ roles:
+ - iptables
+ - install-docker
+ - mailman3
diff --git a/playbooks/zuul/files/host_vars/lists99.opendev.org.yaml b/playbooks/zuul/files/host_vars/lists99.opendev.org.yaml
new file mode 100644
index 0000000000..03570ab354
--- /dev/null
+++ b/playbooks/zuul/files/host_vars/lists99.opendev.org.yaml
@@ -0,0 +1,294 @@
+mailman_list_password: notarealpassword
+mailman3_db_password: Eith5vii5beezohc
+mailman3_db_root_password: eiloh9Edohngaeri
+mailman3_hyperkitty_api_key: Thosai4Xomeque9e
+mailman3_django_secret_key: ohki3ohWusai8tee
+mailman3_rest_password: OhTo3doh5ohsuope
+mailman3_admin_user: admin
+mailman3_admin_email: infra-root@openstack.org
+mailman3_admin_password: AeNie8vegeiquei1
+mm_domains: 'lists.openstack.org:lists.zuul-ci.org:lists.airshipit.org:lists.starlingx.io:lists.opendev.org:lists.openinfra.dev:lists.katacontainers.io'
+exim_local_domains: "@:{{ mm_domains }}"
+exim_enable_spf: true
+exim_aliases:
+ root: "{{ ','.join(listadmins|default([])) }}"
+ interop-wg: openstack-discuss
+ openstack: openstack-discuss
+ openstack-dev: openstack-discuss
+ openstack-infra: openstack-discuss
+ openstack-operators: openstack-discuss
+ openstack-security: openstack-discuss
+ openstack-sigs: openstack-discuss
+ openstack-tc: openstack-discuss
+ user-committee: openstack-discuss
+ airship-discuss-owner: spam
+ community-owner: spam
+ edge-computing-owner: spam
+ foundation-board-confidential-owner: spam
+ foundation-board-owner: spam
+ foundation-owner: spam
+ legal-discuss-owner: spam
+ mailman-owner: spam
+ marketing-owner: spam
+ openstack-announce-owner: spam
+ openstack-docs-owner: spam
+ openstack-fr-owner: spam
+ openstack-i18n-owner: spam
+ openstack-infra-owner: spam
+ openstack-ko-owner: spam
+ openstack-qa-owner: spam
+ product-wg-owner: spam
+ user-committee-owner: spam
+ spam: ':fail: delivery temporarily disabled due to ongoing spam flood'
+ # TODO It would be better to bypass verification for postorius@listdomain
+ # and set a :fail: rule for anyone trying to send email to this addr.
+ # But that requires updating our main exim config so that needs more thought.
+ postorius: ':blackhole: outgoing email only from this address'
+exim_domain_aliases:
+ community@lists.openstack.org: community@lists.openinfra.dev
+ edge-computing@lists.openstack.org: edge-computing@lists.opendev.org
+ foundation@lists.openstack.org: foundation@lists.openinfra.dev
+ foundation-board@lists.openstack.org: foundation-board@lists.openinfra.dev
+ foundation-board-confidential@lists.openstack.org: foundation-board-confidential@lists.openinfra.dev
+ goldmembers@lists.openstack.org: goldmembers@lists.openinfra.dev
+ marketing@lists.openstack.org: marketing@lists.openinfra.dev
+ staff@lists.openstack.org: staff@lists.openinfra.dev
+ summit-programming-committee@lists.openinfra.dev: summit-track-chairs@lists.openinfra.dev
+ summitsponsors@lists.openstack.org: summitsponsors@lists.openinfra.dev
+exim_routers:
+ - mailman_verp_router: |
+ {% raw -%}
+ driver = dnslookup
+ condition = ${if or{{eq{$sender_host_address}{127.0.0.1}}\
+ {eq{$sender_host_address}{::1}}}{yes}{no}}
+ {% endraw %}
+ domains = !+local_domains
+ ignore_target_hosts = <; 0.0.0.0; \
+ 64.94.110.11; \
+ 127.0.0.0/8; \
+ ::1/128;fe80::/10;fe \
+ c0::/10;ff00::/8
+ senders = "*-bounces@*"
+ transport = mailman_verp_smtp
+ - dnslookup: '{{ exim_dnslookup_router }}'
+ - system_aliases: '{{ exim_system_aliases_router }}'
+ - domain_aliases: |
+ driver = redirect
+ allow_fail
+ allow_defer
+ data = ${lookup{$local_part@$domain}lsearch{/etc/aliases.domain}}
+ file_transport = address_file
+ pipe_transport = address_pipe
+ - localuser: '{{ exim_localuser_router }}'
+ - mailman_copy: |
+ driver = accept
+ domains = lists.openstack.org
+ local_parts = openstack-discuss
+ transport = local_copy
+ unseen
+ - mailman_router: |
+ driver = accept
+ domains = {{ mm_domains }}
+ local_part_suffix = -admin : \
+ -bounces : -bounces+* : \
+ -confirm : -confirm+* : \
+ -join : -leave : \
+ -owner : -request : \
+ -subscribe : -unsubscribe
+ local_part_suffix_optional
+ require_files = /var/lib/mailman/core/var/lists/${local_part}.${domain}
+ transport = mailman_transport
+exim_transports:
+ - local_copy: |
+ driver = appendfile
+ file = /var/mail/$local_part
+ group = mail
+ mode = 0660
+ - mailman_transport: |
+ debug_print = "Email for mailman"
+ driver = smtp
+ protocol = lmtp
+ allow_localhost
+ hosts = localhost
+ port = 8024
+ rcpt_include_affixes = true
+ - mailman_verp_smtp: |
+ driver = smtp
+ headers_add = Errors-To: ${return_path}
+ headers_remove = Errors-To
+ max_rcpt = 1
+ return_path = ${local_part:$return_path}+$local_part=$domain@${domain:$return_path}
+mailman_multihost: true
+mailman_sites:
+ # First entry in this list is the primary web domain
+ - listdomain: lists.opendev.org
+ install_languages: ['en']
+ lists:
+ - name: computing-force-network
+ description: 'Organizing efforts around Computing Force Network related area'
+ owner: 'niujie@outlook.com'
+ - name: edge-computing
+ description: 'Organizing efforts around the edge-computing focus area.'
+ owner: 'ildiko@openinfra.dev'
+ - name: floss-mooc
+ description: 'Discussions & Coordination around the FLOSS MOOC being collaboratively developed here: https://gitlab.com/mooc-floss/mooc-floss'
+ owner: 'knelson@openinfra.dev'
+ - name: nbmp-discuss
+ description: 'Collaborating on Network Based Media Processing related platform and infrastructure systems usage and development.'
+ owner: 'ildiko@openstack.org'
+ - name: openinfralabs
+ description: 'Discussion of the OpenInfra Labs academic and research resource sharing effort'
+ owner: 'mnaser@vexxhost.com'
+ - name: rust-vmm
+ description: 'Collaborating on Rust-based virtual machine monitors.'
+ owner: 'claire@openstack.org'
+ - name: rustyk8s
+ description: 'Collaborating on Rust-based Kubernetes API.'
+ owner: 'allison@lohutok.net'
+ - name: service-announce
+ description: 'Announcement list for OpenDev services.'
+ owner: 'cboylan@sapwetik.org'
+ - name: service-discuss
+ description: 'Discussion list for OpenDev services.'
+ owner: 'cboylan@sapwetik.org'
+ - name: service-incident
+ description: 'Private list for OpenDev incident coordination.'
+ owner: 'cboylan@sapwetik.org'
+ private: true
+ - listdomain: lists.airshipit.org
+ install_languages: ['en']
+ lists:
+ - name: airship-announce
+ description: 'Announcements of Airship releases and other important information.'
+ owner: 'jonathan@openstack.org'
+ - name: airship-discuss
+ description: 'Discussion of Airship usage and development.'
+ owner: 'jonathan@openstack.org'
+ - name: airship-embargo-notice
+ description: 'Embargoed security vulnerability announcements for Airship consumers.'
+ owner: 'andrew.walters@att.com'
+ private: true
+ - name: airship-job-failures
+ description: 'Notification messages for failures from CICD jobs.'
+ owner: 'roman.gorshunov@att.com'
+ - name: airship-security
+ description: 'Public Airship security advisories.'
+ owner: 'andrew.walters@att.com'
+ - listdomain: lists.katacontainers.io
+ install_languages: ['en']
+ lists:
+ - name: embargo-notice
+ description: 'Announcements of embargoed notices for the Kata Containers project'
+ owner: 'jonathan@openstack.org'
+ private: true
+ - name: kata-dev
+ description: 'Kata Containers Development Mailing List (not for usage questions)'
+ owner: 'jonathan@openstack.org'
+ - name: kata-hypervisor
+ description: 'Discussion of security and virtualization targeted at container use cases'
+ owner: 'jonathan@openstack.org'
+ - listdomain: lists.openinfra.dev
+ install_languages: ['en']
+ lists:
+ - name: community
+ description: 'The OpenInfra Community team is the main contact point for anybody running a local OpenInfra Group.'
+ owner: 'allison@openinfra.dev'
+ - name: foundation
+ description: 'General discussion list for activities of the OpenInfra Foundation'
+ owner: 'jonathan@openinfra.dev'
+ - name: foundation-board
+ description: 'OpenInfra Foundation Board of Directors'
+ owner: 'jonathan@openinfra.dev'
+ - name: foundation-board-confidential
+ description: 'OpenInfra Foundation Board of Directors'
+ owner: 'jonathan@openinfra.dev'
+ private: true
+ - name: goldmembers
+ description: 'The discussion list for Gold Members of the OpenInfra Foundation'
+ owner: 'jonathan@openinfra.dev'
+ private: true
+ - name: marketing
+ description: 'The OpenInfra Marketing list is the meant to facilitate discussion and best practice sharing among marketers and event organizers in the OpenInfra community.'
+ owner: 'allison@openinfra.dev'
+ - name: staff
+ description: 'Private list for OpenInfra Foundation staff members'
+ owner: 'mark@openinfra.dev'
+ private: true
+ - name: summit-track-chairs
+ description: 'OpenInfra Summit track chair communications'
+ owner: 'erin@openinfra.dev'
+ private: true
+ - name: summitsponsors
+ description: 'Coordination among OpenInfra Summit event sponsors'
+ owner: 'erin@openinfra.dev'
+ private: true
+ - listdomain: lists.openstack.org
+ install_languages: ['de', 'fr', 'it', 'ko', 'ru', 'vi', 'zh_TW']
+ lists:
+ - name: embargo-notice
+ description: 'Announcements to stakeholders for embargoed security vulnerabilities.'
+ owner: 'fungi@yuggoth.org'
+ private: true
+ - name: legal-discuss
+ description: 'Discussions on legal matters related to the project'
+ owner: 'thierry@openinfra.dev'
+ - name: openstack-announce
+ description: 'Key announcements about OpenStack & Security advisories'
+ owner: 'fungi@yuggoth.org'
+ - name: openstack-discuss
+ description: 'Discussion of OpenStack usage and development.'
+ owner: 'fungi@yuggoth.org'
+ - name: openstack-es
+ description: 'Lista de correo acerca de OpenStack en español'
+ owner: 'flavio@redhat.com'
+ - name: openstack-fr
+ description: 'List of the OpenStack french user group'
+ owner: 'erwan@erwan.com'
+ - name: openstack-hpc
+ description: 'High-Performance Computing OpenStack List'
+ owner: 'brian.schott@nimbisservices.com'
+ - name: openstack-i18n
+ description: 'List of the OpenStack Internationalization team.'
+ owner: 'guoyingc@cn.ibm.com'
+ - name: openstack-it
+ description: 'Discussioni su OpenStack in italiano'
+ owner: 'stefano@openstack.org'
+ - name: openstack-ko
+ description: 'OpenStack Korea Community Discussions in Korean (오픈스택 한국 커뮤니티 메일링리스트)'
+ owner: 'ianyrchoi@gmail.com'
+ - name: openstack-mentoring
+ description: 'List to coordinate interactions between mentors and mentees of the OpenStack mentoring program. Also for questions about the mentoring program (i.e. how to get involved, how it works, etc.'
+ owner: 'amy@demarco.com'
+ - name: openstack-stable-maint
+ description: 'A mailing list for the OpenStack Stable Branch test reports.'
+ owner: 'tony@bakeyournoodle.com'
+ - name: openstack-zh
+ description: 'OpenStack社区中文讨论群组'
+ owner: 'yeluaiesec@gmail.com'
+ - name: release-announce
+ description: 'Announcement of official OpenStack releases.'
+ owner: 'thierry@openstack.org'
+ - name: release-job-failures
+ description: 'Notification messages for failures from release-related build jobs.'
+ owner: 'doug@doughellmann.com'
+ - listdomain: lists.starlingx.io
+ install_languages: ['en']
+ lists:
+ - name: starlingx-announce
+ description: 'Announcements of StarlingX releases and other important information.'
+ owner: 'jonathan@openstack.org'
+ - name: starlingx-discuss
+ description: 'Discussion of StarlingX usage and development.'
+ owner: 'jonathan@openstack.org'
+ - listdomain: lists.zuul-ci.org
+ install_languages: ['en']
+ lists:
+ - name: zuul-announce
+ description: 'Announcements of Zuul releases and other important information.'
+ owner: 'corvus@inaugust.com'
+ - name: zuul-discuss
+ description: 'Discussion of Zuul usage and development.'
+ owner: 'corvus@inaugust.com'
+ - name: zuul-jobs-failures
+ description: 'Gets notifications about zuul-jobs periodic job failures.'
+ owner: 'ssbarnea@redhat.com'
diff --git a/playbooks/zuul/lists3-alias-logs.yaml b/playbooks/zuul/lists3-alias-logs.yaml
new file mode 100644
index 0000000000..dfc28aab0d
--- /dev/null
+++ b/playbooks/zuul/lists3-alias-logs.yaml
@@ -0,0 +1,16 @@
+# We need to do this in a run playbook so that we copy logs even
+# when the run playbooks fail. Failure in the run playbooks prevents the
+# test playbook from running.
+- hosts: "mailman3"
+ tasks:
+ - name: Alias mailman-web logs
+ file:
+ state: link
+ src: /var/lib/mailman/web-data/logs
+ path: /var/lib/mailman/mailman-web-logs
+
+ - name: Alias mailman-core logs
+ file:
+ state: link
+ src: /var/lib/mailman/core/var/logs
+ path: /var/lib/mailman/mailman-core-logs
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index b51ba2b977..582b1ea684 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -151,6 +151,14 @@
- host_vars/paste99.opendev.org.yaml
- host_vars/refstack01.openstack.org.yaml
- host_vars/review99.opendev.org.yaml
+ - name: Write lists99 host_vars.
+ # This file is special because it has raw tags in it that we need to
+ # carry through. I can't figure out a better way to do that then copying
+ # it directly rather than treating it as a template.
+ copy:
+ src: "files/host_vars/lists99.opendev.org.yaml"
+ dest: "/etc/ansible/hosts/host_vars/lists99.opendev.org.yaml"
+
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
- name: Run base.yaml
diff --git a/playbooks/zuul/test-lists3.yaml b/playbooks/zuul/test-lists3.yaml
new file mode 100644
index 0000000000..03bd422547
--- /dev/null
+++ b/playbooks/zuul/test-lists3.yaml
@@ -0,0 +1,19 @@
+- hosts: "mailman3"
+ tasks:
+ - name: Force hyperkitty to populate info about the new lists we created
+ command: docker exec mailman-web ./manage.py runjobs hourly
+
+ - name: Update /etc/hosts in order to test mm3 vhosts
+ lineinfile:
+ state: present
+ backrefs: yes
+ path: /etc/hosts
+ regexp: '^127\.0\.0\.1\s+(.*)$'
+ line: '127.0.0.1 \1 {{ mm_site.listdomain }}'
+ loop: "{{ mailman_sites }}"
+ loop_control:
+ loop_var: mm_site
+
+ - name: Run selenium container
+ include_role:
+ name: run-selenium
diff --git a/testinfra/test_lists_opendev_org.py b/testinfra/test_lists_opendev_org.py
new file mode 100644
index 0000000000..5b1ff89fc9
--- /dev/null
+++ b/testinfra/test_lists_opendev_org.py
@@ -0,0 +1,61 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from util import take_screenshots
+
+testinfra_hosts = ['lists99.opendev.org']
+
+def test_mariadb_listening(host):
+ mariadb_port = host.socket("tcp://127.0.0.1:3306")
+ assert mariadb_port.is_listening
+
+def test_mailman3_web_listening(host):
+ mailman_web = host.socket("tcp://0.0.0.0:8000")
+ assert mailman_web.is_listening
+ mailman_uwsgi = host.socket("tcp://0.0.0.0:8080")
+ assert mailman_uwsgi.is_listening
+
+def test_mailman3_core_listening(host):
+ mailman_rest = host.socket("tcp://127.0.0.1:8001")
+ assert mailman_rest.is_listening
+ mailman_lmtp = host.socket("tcp://127.0.0.1:8024")
+ assert mailman_lmtp.is_listening
+
+def test_apache2_listening(host):
+ apache2_http = host.socket("tcp://0.0.0.0:80")
+ assert apache2_http.is_listening
+ apache2_https = host.socket("tcp://0.0.0.0:443")
+ assert apache2_https.is_listening
+
+def test_mailman3_screenshots(host):
+ shots = (
+ ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),
+ ("https://lists.opendev.org:443/hyperkitty/",
+ None, "mm3-opendev-archives.png"),
+ ("https://lists.opendev.org:443"
+ "/hyperkitty/list/service-discuss@lists.opendev.org/",
+ None, "mm3-opendev-list.png"),
+ ("https://lists.opendev.org:443"
+ "/accounts/login/?next=/postorius/lists/",
+ None, "mm3-opendev-login.png"),
+ ("https://lists.openstack.org:443", None, "mm3-openstack-main.png"),
+ ("https://lists.openstack.org:443/hyperkitty/",
+ None, "mm3-openstack-archives.png"),
+ ("https://lists.openstack.org:443"
+ "/hyperkitty/list/openstack-discuss@lists.openstack.org/",
+ None, "mm3-openstack-list.png"),
+ ("https://lists.openstack.org:443"
+ "/accounts/login/?next=/postorius/lists/",
+ None, "mm3-openstack-login.png"),
+ )
+
+ take_screenshots(host, shots)
diff --git a/tools/mm3-migrate.sh b/tools/mm3-migrate.sh
new file mode 100644
index 0000000000..6559acd0a2
--- /dev/null
+++ b/tools/mm3-migrate.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+s=$1
+test -z "$s" && echo 'Specify the list FQDN to migrate!' && exit 1
+echo "*** starting at $(date -Is) ***"
+set -x
+for m in $(
+ ls -d /var/lib/mailman/import/$s/lists/* \
+ | grep -v /mailman$ \
+ | cut -d/ -f8
+); do
+ time sudo docker-compose -f /etc/mailman-compose/docker-compose.yaml \
+ exec -T -u mailman mailman-core mailman import21 $m@$s \
+ /opt/import/$s/lists/$m/config.pck
+ time sudo docker-compose -f /etc/mailman-compose/docker-compose.yaml \
+ exec -T -u mailman mailman-web python3 manage.py hyperkitty_import -l \
+ $m@$s /opt/import/$s/archives/private/$m.mbox/$m.mbox
+ time sudo docker-compose -f /etc/mailman-compose/docker-compose.yaml \
+ exec -T -u mailman mailman-web python3 manage.py \
+ update_index_one_list $m@$s
+done
+sudo mv /var/lib/mailman/import/$s/archives \
+ /var/lib/mailman/web-data/mm2archives/$s
+for a in /var/lib/mailman/web-data/mm2archives/$s/public/*; do
+ sudo ln -fs ../private/`basename $a` $a
+done
+set +x
+echo "*** completed at $(date -Is) ***"
diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml
index c190ac7f2e..be3fbf0471 100644
--- a/zuul.d/infra-prod.yaml
+++ b/zuul.d/infra-prod.yaml
@@ -563,9 +563,23 @@
- inventory/service/host_vars/lists.katacontainers.io.yaml
- playbooks/roles/iptables/
- playbooks/roles/base/exim
- - playbooks/roles/mailman
+ - playbooks/roles/mailman/
- playbooks/service-lists.yaml
+- job:
+ name: infra-prod-service-lists3
+ parent: infra-prod-service-base
+ description: Run service-lists3.yaml playbook.
+ vars:
+ playbook_name: service-lists3.yaml
+ files:
+ - inventory/base
+ - inventory/service/host_vars/lists01.opendev.org.yaml
+ - playbooks/roles/iptables/
+ - playbooks/roles/base/exim
+ - playbooks/roles/mailman3/
+ - playbooks/service-lists3.yaml
+
# Run AFS changes separately so we can make sure to only do one at a time
# (turns out quorum is nice to have)
- job:
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index 80c513bc55..4024ed5e94 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -30,6 +30,7 @@
soft: true
- system-config-run-kerberos
- system-config-run-lists
+ - system-config-run-lists3
- system-config-run-nodepool:
dependencies:
- name: opendev-buildset-registry
@@ -180,6 +181,7 @@
soft: true
- system-config-run-kerberos
- system-config-run-lists
+ - system-config-run-lists3
- system-config-run-nodepool:
dependencies:
- name: opendev-buildset-registry
@@ -461,6 +463,12 @@
soft: true
- name: infra-prod-letsencrypt
soft: true
+ - infra-prod-service-lists3: &infra-prod-service-lists3
+ dependencies:
+ - name: infra-prod-service-borg-backup
+ soft: true
+ - name: infra-prod-letsencrypt
+ soft: true
- infra-prod-service-mirror: &infra-prod-service-mirror
dependencies:
- name: infra-prod-letsencrypt
@@ -599,6 +607,7 @@
- infra-prod-service-keycloak: *infra-prod-service-keycloak
- infra-prod-service-meetpad: *infra-prod-service-meetpad
- infra-prod-service-lists: *infra-prod-service-lists
+ - infra-prod-service-lists3: *infra-prod-service-lists3
- infra-prod-service-mirror: *infra-prod-service-mirror
- infra-prod-service-nodepool: *infra-prod-service-nodepool
- infra-prod-service-static: *infra-prod-service-static
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 655d3c3219..d428909849 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -258,7 +258,7 @@
- inventory/service/host_vars/lists.katacontainers.io.yaml
- inventory/service/group_vars/mailman.yaml
- playbooks/roles/base/exim
- - playbooks/roles/mailman
+ - playbooks/roles/mailman/
- playbooks/service-lists.yaml
- playbooks/test-lists.yaml
- playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2
@@ -286,6 +286,49 @@
'/var/log/apache2': logs
'/var/log/mailman': logs
+- job:
+ name: system-config-run-lists3
+ # We don't use the system-config-run-containers base job because we
+ # are consuming upstream containers only.
+ parent: system-config-run
+ description: |
+ Run the playbook for a mailman3 list server.
+ timeout: 3600
+ nodeset:
+ nodes:
+ - <<: *bridge_node_x86
+ - name: lists99.opendev.org
+ label: ubuntu-jammy
+ groups:
+ - <<: *bastion_group
+ required-projects:
+ - opendev/system-config
+ files:
+ - playbooks/bootstrap-bridge.yaml
+ - inventory/service/host_vars/lists01.opendev.org.yaml
+ - inventory/service/group_vars/mailman3.yaml
+ - playbooks/roles/base/exim
+ - playbooks/roles/mailman3
+ - playbooks/service-lists3.yaml
+ - playbooks/test-lists3.yaml
+ - playbooks/zuul/files/host_vars/lists99.opendev.org.yaml
+ - testinfra/test_lists_opendev_org.py
+ vars:
+ run_playbooks:
+ - playbooks/letsencrypt.yaml
+ - playbooks/service-lists3.yaml
+ # Run this twice to check idempotency
+ - playbooks/service-lists3.yaml
+ - playbooks/zuul/lists3-alias-logs.yaml
+ run_test_playbook: playbooks/zuul/test-lists3.yaml
+ host-vars:
+ lists99.opendev.org:
+ host_copy_output:
+ '/var/log/acme.sh': logs
+ '/var/log/apache2': logs
+ '/var/lib/mailman/mailman-web-logs': logs
+ '/var/lib/mailman/mailman-core-logs': logs
+
- job:
name: system-config-run-nodepool
parent: system-config-run