Merge "Fork the maxking/docker-mailman images"

docker/mailman/core/Dockerfile

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.3
+# Use 3.15 for Core since it has Python 3.9
+FROM alpine:3.15
+
+#Add startup script to container
+COPY docker-entrypoint.sh /usr/local/bin/
+
+# Add requirements file.
+COPY requirements.txt /tmp/
+
+#Install all required packages, add user for executing mailman and set execution rights for startup script
+RUN --mount=type=cache,target=/root/.cache \
+    apk update \
+    && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \
+       libffi-dev \
+  # Mailman html to plaintext conversion uses lynx.
+  # psutil needs linux-headers to compile on musl c library.
+    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx \
+    && python3 -m pip install -U pip setuptools wheel \
+        && python3 -m pip install psycopg2 \
+                   gunicorn==19.9.0 \
+                   pymysql \
+                   sqlalchemy \
+                   -r /tmp/requirements.txt \
+    && apk del build-deps \
+    && adduser -S mailman
+
+# Change the working directory.
+WORKDIR /opt/mailman
+
+#Expose the ports for the api (8001) and lmtp (8024)
+EXPOSE 8001 8024
+
+ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["master", "--force"]

docker/mailman/core/README.md

@@ -0,0 +1,6 @@
+This is forked from https://github.com/maxking/docker-mailman/tree/main/core
+and you should refer to that repo for information. The upstream repo is MIT
+licensed.
+
+The contents in this dir are based on commit
+22b3b7d4024a00e9896837fa04883cbaeef38b20.

docker/mailman/core/docker-entrypoint.sh

@@ -0,0 +1,233 @@
+#! /bin/bash
+set -e
+
+function wait_for_postgres () {
+	# Check if the postgres database is up and accepting connections before
+	# moving forward.
+	# TODO: Use python3's psycopg2 module to do this in python3 instead of
+	# installing postgres-client in the image.
+	until psql $DATABASE_URL -c '\l'; do
+		>&2 echo "Postgres is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "Postgres is up - continuing"
+}
+
+function wait_for_mysql () {
+	# Check if MySQL is up and accepting connections.
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
+		>&2 echo "MySQL is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "MySQL is up - continuing"
+}
+
+# Empty the config file.
+echo "# This file is autogenerated at container startup." > /etc/mailman.cfg
+
+# Check if $MM_HOSTNAME is set, if not, set it to the value returned by
+# `hostname -i` command to set it to whatever IP address is assigned to the
+# container.
+if [[ ! -v MM_HOSTNAME ]]; then
+	export MM_HOSTNAME=`hostname -i`
+fi
+
+# SMTP_HOST defaults to the gateway
+if [[ ! -v SMTP_HOST ]]; then
+	export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }')
+	echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default"
+fi
+
+if [[ ! -v SMTP_PORT ]]; then
+	export SMTP_PORT=25
+fi
+
+# Check if REST port, username, and password are set, if not, set them
+# to default values.
+if [[ ! -v MAILMAN_REST_PORT ]]; then
+	export MAILMAN_REST_PORT='8001'
+fi
+
+if [[ ! -v MAILMAN_REST_USER ]]; then
+	export MAILMAN_REST_USER='restadmin'
+fi
+
+if [[ ! -v MAILMAN_REST_PASSWORD ]]; then
+	export MAILMAN_REST_PASSWORD='restpass'
+fi
+
+function setup_database () {
+	if [[ ! -v DATABASE_URL ]]
+	then
+		echo "Environment variable DATABASE_URL should be defined..."
+		exit 1
+	fi
+
+	# Translate mysql:// urls to mysql+mysql:// backend:
+	if [[ "$DATABASE_URL" == mysql://* ]]; then
+		DATABASE_URL="mysql+pymysql://${DATABASE_URL:8}"
+		echo "Database URL was automatically rewritten to: $DATABASE_URL"
+	fi
+
+	# If DATABASE_CLASS is not set, guess it for common databases:
+	if [ -z "$DATABASE_CLASS" ]; then
+		if [[ ("$DATABASE_URL" == mysql:*) ||
+				("$DATABASE_URL" == mysql+*) ]]; then
+			DATABASE_CLASS=mailman.database.mysql.MySQLDatabase
+		fi
+		if [[ ("$DATABASE_URL" == postgres:*) ||
+				("$DATABASE_URL" == postgres+*) ]]; then
+			DATABASE_CLASS=mailman.database.postgresql.PostgreSQLDatabase
+		fi
+	fi
+
+	cat >> /etc/mailman.cfg <<EOF
+[database]
+class: $DATABASE_CLASS
+url: $DATABASE_URL
+EOF
+}
+
+
+# Check if $DATABASE_URL is defined, if not, use a standard sqlite database.
+#
+# If the $DATABASE_URL is defined and is postgres, check if it is available
+# yet. Do not start the container before the postgresql boots up.
+#
+# TODO: If the $DATABASE_URL is defined and is mysql, check if the database is
+# available before the container boots up.
+#
+# TODO: Check the database type and detect if it is up based on that. For now,
+# assume that postgres is being used if DATABASE_URL is defined.
+if [[ ! -v DATABASE_URL ]]; then
+	echo "DATABASE_URL is not defined. Using sqlite database..."
+else
+	setup_database
+fi
+
+
+if [[ "$DATABASE_TYPE" = 'postgres' ]]
+then
+	wait_for_postgres
+elif [[ "$DATABASE_TYPE" = 'mysql' ]]
+then
+	wait_for_mysql
+fi
+
+# Generate a basic mailman.cfg.
+cat >> /etc/mailman.cfg << EOF
+[runner.retry]
+sleep_time: 10s
+
+[webservice]
+hostname: $MM_HOSTNAME
+port: $MAILMAN_REST_PORT
+admin_user: $MAILMAN_REST_USER
+admin_pass: $MAILMAN_REST_PASSWORD
+configuration: /etc/gunicorn.cfg
+
+EOF
+
+# Generate a basic gunicorn.cfg.
+SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])')
+cp "${SITE_DIR}/mailman/config/gunicorn.cfg" /etc/gunicorn.cfg
+
+# Generate a basic configuration to use exim
+cat > /tmp/exim-mailman.cfg <<EOF
+[mta]
+incoming: mailman.mta.exim4.LMTP
+outgoing: mailman.mta.deliver.deliver
+lmtp_host: $MM_HOSTNAME
+lmtp_port: 8024
+smtp_host: $SMTP_HOST
+smtp_port: $SMTP_PORT
+configuration: python:mailman.config.exim4
+
+EOF
+
+cat > /etc/postfix-mailman.cfg << EOF
+[postfix]
+transport_file_type: regex
+# While in regex mode, postmap_command is never used, a placeholder
+# is added here so that it doesn't break anything.
+postmap_command: true
+EOF
+
+# Generate a basic configuration to use postfix.
+cat > /tmp/postfix-mailman.cfg <<EOF
+[mta]
+incoming: mailman.mta.postfix.LMTP
+outgoing: mailman.mta.deliver.deliver
+lmtp_host: $MM_HOSTNAME
+lmtp_port: 8024
+smtp_host: $SMTP_HOST
+smtp_port: $SMTP_PORT
+configuration: /etc/postfix-mailman.cfg
+
+EOF
+
+if [ "$MTA" == "exim" ]
+then
+	echo "Using Exim configuration"
+	cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg
+elif [ "$MTA" == "postfix" ]
+then
+	echo "Using Postfix configuration"
+	cat /tmp/postfix-mailman.cfg >> /etc/mailman.cfg
+else
+	echo "No MTA environment variable found, defaulting to Exim"
+	cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg
+fi
+
+rm -f /tmp/{postfix,exim}-mailman.cfg
+
+if [[ -e /opt/mailman/mailman-extra.cfg ]]
+then
+	echo "Found configuration file at /opt/mailman/mailman-extra.cfg"
+	cat /opt/mailman/mailman-extra.cfg >> /etc/mailman.cfg
+fi
+
+if [[ -e /opt/mailman/gunicorn-extra.cfg ]]
+then
+       echo "Found [webserver] configuration file at /opt/mailman/gunicorn-extra.cfg"
+       cat /opt/mailman/gunicorn-extra.cfg > /etc/gunicorn.cfg
+fi
+
+if [[ -v HYPERKITTY_API_KEY ]]; then
+
+echo "HYPERKITTY_API_KEY found, setting up HyperKitty archiver..."
+
+cat >> /etc/mailman.cfg << EOF
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
+enable: yes
+configuration: /etc/mailman-hyperkitty.cfg
+
+EOF
+
+if [[ ! -v HYPERKITTY_URL ]]; then
+	echo "HYPERKITTY_URL not set, using the default value of http://mailman-web:8000/hyperkitty"
+	export HYPERKITTY_URL="http://mailman-web:8000/hyperkitty/"
+fi
+
+# Generate a basic mailman-hyperkitty.cfg.
+cat > /etc/mailman-hyperkitty.cfg <<EOF
+[general]
+base_url: $HYPERKITTY_URL
+api_key: $HYPERKITTY_API_KEY
+EOF
+
+else
+
+echo "HYPERKITTY_API_KEY not defined, skipping HyperKitty setup..."
+
+fi
+
+# Now chown the places where mailman wants to write stuff.
+chown -R mailman /opt/mailman
+
+# Generate the LMTP files for postfix if needed.
+su-exec mailman mailman aliases
+
+exec su-exec mailman "$@"

docker/mailman/core/requirements.txt

@@ -0,0 +1,5 @@
+# This is a separate file from Dockerfile so that we can use dependabot
+# for version updates that isn't supported for contents inside the
+# Dockerfile.
+mailman==3.3.7
+mailman-hyperkitty==1.2.0

docker/mailman/web/Dockerfile

@@ -0,0 +1,52 @@
+# syntax = docker/dockerfile:1.3
+FROM alpine:3.16.2
+
+# Add needed files for uwsgi server + settings for django
+COPY mailman-web /opt/mailman-web
+# Add startup script to container
+COPY docker-entrypoint.sh /usr/local/bin/
+# Add requirements file.
+COPY requirements.txt /tmp/
+
+# Install packages and dependencies for postorius and hyperkitty Add user for
+# executing apps, change ownership for uwsgi+django files and set execution
+# rights for management script
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
+	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
+           postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
+	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
+	   postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \
+	   python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \
+	&& python3 -m pip install -U 'Django<4.1' pip setuptools wheel \
+	&& pip install -r /tmp/requirements.txt \
+		whoosh \
+		uwsgi \
+		'psycopg2<2.9' \
+		dj-database-url \
+		mysqlclient \
+		typing \
+		xapian-haystack \
+		django-auth-ldap \
+		python-memcached \
+		diskcache \
+		django-utils-six \
+		tzdata \
+	&& apk del .build-deps \
+	&& addgroup -S mailman \
+	&& adduser -S -G mailman mailman \
+	&& chown -R mailman /opt/mailman-web/ \
+	&& chmod u+x /opt/mailman-web/manage.py
+
+WORKDIR /opt/mailman-web
+
+# Expose port 8000 for http and port 8080 for uwsgi
+# (see web/mailman-web/uwsgi.ini#L2-L4)
+EXPOSE 8000 8080
+
+# Use stop signal for uwsgi server
+STOPSIGNAL SIGINT
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"]

docker/mailman/web/README.md

@@ -0,0 +1,6 @@
+This is forked from https://github.com/maxking/docker-mailman/tree/main/web
+and you should refer to that repo for information. The upstream repo is MIT
+licensed.
+
+The contents in this dir are based on commit
+22b3b7d4024a00e9896837fa04883cbaeef38b20.

docker/mailman/web/docker-entrypoint.sh

@@ -0,0 +1,154 @@
+#! /bin/bash
+set -e
+
+
+function wait_for_postgres () {
+	# Check if the postgres database is up and accepting connections before
+	# moving forward.
+	# TODO: Use python's psycopg2 module to do this in python instead of
+	# installing postgres-client in the image.
+	until psql $DATABASE_URL -c '\l'; do
+		>&2 echo "Postgres is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "Postgres is up - continuing"
+}
+
+function wait_for_mysql () {
+	# Check if MySQL is up and accepting connections.
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
+		>&2 echo "MySQL is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "MySQL is up - continuing"
+}
+
+function check_or_create () {
+	# Check if the path exists, if not, create the directory.
+	if [[ ! -e dir ]]; then
+		echo "$1 does not exist, creating ..."
+		mkdir "$1"
+	fi
+}
+
+# function postgres_ready(){
+# python << END
+# import sys
+# import psycopg2
+# try:
+#     conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres")
+# except psycopg2.OperationalError:
+#     sys.exit(-1)
+# sys.exit(0)
+# END
+# }
+
+# SMTP_HOST defaults to the gateway
+if [[ ! -v SMTP_HOST ]]; then
+	export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }')
+fi
+
+# Check if $SECRET_KEY is defined, if not, bail out.
+if [[ ! -v SECRET_KEY ]]; then
+	echo "SECRET_KEY is not defined. Aborting."
+	exit 1
+fi
+
+# Check if $DATABASE_URL is defined, if not, use a standard sqlite database.
+#
+# If the $DATABASE_URL is defined and is postgres, check if it is available
+# yet. Do not start the container before the postgresql boots up.
+#
+# If the $DATABASE_URL is defined and is mysql, check if the database is
+# available before the container boots up.
+#
+# TODO: Check the database type and detect if it is up based on that. For now,
+# assume that postgres is being used if DATABASE_URL is defined.
+
+if [[ ! -v DATABASE_URL ]]; then
+	echo "DATABASE_URL is not defined. Using sqlite database..."
+	export DATABASE_URL=sqlite://mailmanweb.db
+	export DATABASE_TYPE='sqlite'
+fi
+
+if [[ "$DATABASE_TYPE" = 'postgres' ]]
+then
+	wait_for_postgres
+elif [[ "$DATABASE_TYPE" = 'mysql' ]]
+then
+	wait_for_mysql
+fi
+
+# Check if we are in the correct directory before running commands.
+if [[ ! $(pwd) == '/opt/mailman-web' ]]; then
+	echo "Running in the wrong directory...switching to /opt/mailman-web"
+	cd /opt/mailman-web
+fi
+
+# Check if the logs directory is setup.
+if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then
+	echo "Creating log file for mailman web"
+	mkdir -p /opt/mailman-web-data/logs/
+	touch /opt/mailman-web-data/logs/mailmanweb.log
+fi
+
+if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then
+	echo "Creating log file for uwsgi.."
+	touch /opt/mailman-web-data/logs/uwsgi.log
+fi
+
+# Check if the settings_local.py file exists, if yes, copy it too.
+if [[ -e /opt/mailman-web-data/settings_local.py ]]; then
+	echo "Copying settings_local.py ..."
+	cp /opt/mailman-web-data/settings_local.py /opt/mailman-web/settings_local.py
+	chown mailman:mailman /opt/mailman-web/settings_local.py
+else
+	echo "settings_local.py not found, it is highly recommended that you provide one"
+	echo "Using default configuration to run."
+fi
+
+# Collect static for the django installation.
+python3 manage.py collectstatic --noinput --clear --verbosity 0
+
+
+# Compile all the installed po files to mo.
+SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])')
+echo "Compiling locale files in $SITE_DIR"
+cd $SITE_DIR && /opt/mailman-web/manage.py compilemessages &&  cd -
+
+# Compress static files.
+python3 manage.py compress --force
+
+
+# Migrate all the data to the database if this is a new installation, otherwise
+# this command will upgrade the database.
+python3 manage.py migrate
+
+# If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL is defined create a new
+# superuser for Django. There is no password setup so it can't login yet unless
+# the password is reset.
+if [[ -v MAILMAN_ADMIN_USER ]] && [[ -v MAILMAN_ADMIN_EMAIL ]];
+then
+	echo "Creating admin user $MAILMAN_ADMIN_USER ..."
+	python3 manage.py createsuperuser --noinput --username "$MAILMAN_ADMIN_USER"\
+		   --email "$MAILMAN_ADMIN_EMAIL" 2> /dev/null || \
+		echo "Superuser $MAILMAN_ADMIN_USER already exists"
+fi
+
+# If SERVE_FROM_DOMAIN is defined then rename the default `example.com`
+# domain to the defined domain.
+if [[ -v SERVE_FROM_DOMAIN ]];
+then
+	echo "Setting $SERVE_FROM_DOMAIN as the default domain ..."
+	python3 manage.py shell -c \
+	"from django.contrib.sites.models import Site; Site.objects.filter(domain='example.com').update(domain='$SERVE_FROM_DOMAIN', name='$SERVE_FROM_DOMAIN')"
+fi
+
+# Create a mailman user with the specific UID and GID and do not create home
+# directory for it. Also chown the logs directory to write the files.
+chown mailman:mailman /opt/mailman-web-data -R
+
+[[ -v DISKCACHE_PATH ]] && chown mailman:mailman "${DISKCACHE_PATH}" -R
+
+exec $@

docker/mailman/web/mailman-web/manage.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

docker/mailman/web/mailman-web/settings.py

@@ -0,0 +1,414 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Mailman Suite.
+#
+# Mailman Suite is free sofware: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Mailman Suite is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+
+# You should have received a copy of the GNU General Public License along
+# with Mailman Suite.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Django Settings for Mailman Suite (hyperkitty + postorius)
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.8/ref/settings/
+"""
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+import dj_database_url
+import sys
+from socket import gethostbyname
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = os.environ.get('SECRET_KEY')
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ADMINS = (
+     ('Mailman Suite Admin', 'root@localhost'),
+)
+
+SITE_ID = 1
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [
+    "localhost",  # Archiving API from Mailman, keep it.
+    # "lists.your-domain.org",
+    # Add here all production URLs you may have.
+    "mailman-web",
+    gethostbyname("mailman-web"),
+    os.environ.get('SERVE_FROM_DOMAIN'),
+    os.environ.get('DJANGO_ALLOWED_HOSTS'),
+]
+
+# Mailman API credentials
+MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001')
+MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin')
+MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass')
+MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY')
+MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),)
+
+# Application definition
+
+INSTALLED_APPS = []
+DEFAULT_APPS = [
+    'hyperkitty',
+    'postorius',
+    'django_mailman3',
+    # Uncomment the next line to enable the admin:
+    'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'rest_framework',
+    'django_gravatar',
+    'compressor',
+    'haystack',
+    'django_extensions',
+    'django_q',
+    'allauth',
+    'allauth.account',
+    'allauth.socialaccount',
+]
+
+MAILMAN_WEB_SOCIAL_AUTH = [
+    'django_mailman3.lib.auth.fedora',
+    'allauth.socialaccount.providers.openid',
+    'allauth.socialaccount.providers.github',
+    'allauth.socialaccount.providers.gitlab',
+    'allauth.socialaccount.providers.google',
+]
+
+MIDDLEWARE = (
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django_mailman3.middleware.TimezoneMiddleware',
+    'postorius.middleware.PostoriusMiddleware',
+)
+
+ROOT_URLCONF = 'urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.media',
+                'django.template.context_processors.static',
+                'django.template.context_processors.tz',
+                'django.template.context_processors.csrf',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+                'django_mailman3.context_processors.common',
+                'hyperkitty.context_processors.common',
+                'postorius.context_processors.postorius',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+# dj_database_url uses $DATABASE_URL environment variable to create a
+# django-style-config-dict.
+# https://github.com/kennethreitz/dj-database-url
+DATABASES = {
+    'default': dj_database_url.config(conn_max_age=600)
+}
+
+# If you're behind a proxy, use the X-Forwarded-Host header
+# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
+USE_X_FORWARDED_HOST = True
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.8/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+STATIC_ROOT = '/opt/mailman-web-data/static'
+
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'compressor.finders.CompressorFinder',
+)
+
+
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
+
+LOGIN_URL = 'account_login'
+LOGIN_REDIRECT_URL = 'list_index'
+LOGOUT_URL = 'account_logout'
+
+
+# Use SERVE_FROM_DOMAIN as the default domain in the email.
+hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local')
+DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname)
+SERVER_EMAIL = 'root@{}'.format(hostname)
+
+# Change this when you have a real email backend
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST = os.environ.get('SMTP_HOST', '')
+EMAIL_PORT = os.environ.get('SMTP_PORT', 25)
+EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '')
+EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '')
+EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False)
+EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False)
+
+# Compatibility with Bootstrap 3
+from django.contrib.messages import constants as messages  # flake8: noqa
+MESSAGE_TAGS = {
+    messages.ERROR: 'danger'
+}
+
+
+#
+# Social auth
+#
+AUTHENTICATION_BACKENDS = (
+    'django.contrib.auth.backends.ModelBackend',
+    'allauth.account.auth_backends.AuthenticationBackend',
+)
+
+# Django Allauth
+ACCOUNT_AUTHENTICATION_METHOD = "username_email"
+ACCOUNT_EMAIL_REQUIRED = True
+ACCOUNT_EMAIL_VERIFICATION = "mandatory"
+# You probably want https in production, but this is a dev setup file
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
+ACCOUNT_UNIQUE_EMAIL  = True
+
+SOCIALACCOUNT_PROVIDERS = {
+    'openid': {
+        'SERVERS': [
+            dict(id='yahoo',
+                 name='Yahoo',
+                 openid_url='http://me.yahoo.com'),
+        ],
+    },
+    'google': {
+        'SCOPE': ['profile', 'email'],
+        'AUTH_PARAMS': {'access_type': 'online'},
+    },
+    'facebook': {
+       'METHOD': 'oauth2',
+       'SCOPE': ['email'],
+       'FIELDS': [
+           'email',
+           'name',
+           'first_name',
+           'last_name',
+           'locale',
+           'timezone',
+           ],
+       'VERSION': 'v2.4',
+    },
+}
+
+
+# django-compressor
+# https://pypi.python.org/pypi/django_compressor
+#
+COMPRESS_PRECOMPILERS = (
+   ('text/less', 'lessc {infile} {outfile}'),
+   ('text/x-scss', 'sassc -t compressed {infile} {outfile}'),
+   ('text/x-sass', 'sassc -t compressed {infile} {outfile}'),
+)
+
+# On a production setup, setting COMPRESS_OFFLINE to True will bring a
+# significant performance improvement, as CSS files will not need to be
+# recompiled on each requests. It means running an additional "compress"
+# management command after each code upgrade.
+# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression
+# COMPRESS_OFFLINE = True
+
+#
+# Full-text search engine
+#
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': "/opt/mailman-web-data/fulltext_index",
+        # You can also use the Xapian engine, it's faster and more accurate,
+        # but requires another library.
+        # http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian
+        # Example configuration for Xapian:
+        #'ENGINE': 'xapian_backend.XapianEngine'
+    },
+}
+
+import sys
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'file':{
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            #'class': 'logging.handlers.WatchedFileHandler',
+            'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'),
+            'formatter': 'verbose',
+        },
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple',
+            'level': 'INFO',
+            'stream': sys.stdout,
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins', 'file'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'hyperkitty': {
+            'handlers': ['file'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'postorius': {
+            'handlers': ['file'],
+            'level': 'INFO',
+            'propagate': True
+        },
+    },
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+    },
+}
+
+
+if os.environ.get('LOG_TO_CONSOLE') == 'yes':
+    LOGGING['loggers']['django']['handlers'].append('console')
+    LOGGING['loggers']['django.request']['handlers'].append('console')
+
+# HyperKitty-specific
+#
+# Only display mailing-lists from the same virtual host as the webserver
+FILTER_VHOST = False
+
+
+Q_CLUSTER = {
+    'timeout': 300,
+    'retry': 300,
+    'save_limit': 100,
+    'orm': 'default',
+}
+
+POSTORIUS_TEMPLATE_BASE_URL =  os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000')
+
+DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache')
+DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte
+
+CACHES = {
+    'default': {
+        'BACKEND': 'diskcache.DjangoCache',
+        'LOCATION': DISKCACHE_PATH,
+        'OPTIONS': {
+            'size_limit': DISKCACHE_SIZE,
+        },
+    },
+}
+
+try:
+    from settings_local import *
+except ImportError:
+    pass
+
+# Compatibility for older installs that override INSTALLED_APPS
+if not INSTALLED_APPS:
+    INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH

docker/mailman/web/mailman-web/urls.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Postorius.
+#
+# Postorius is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Postorius is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Postorius.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import include
+from django.contrib import admin
+from django.urls import path, reverse_lazy
+from django.views.generic import RedirectView
+
+urlpatterns = [
+    path(r'', RedirectView.as_view(
+        url=reverse_lazy('list_index'),
+        permanent=True)),
+    path(r'mailman3/', include('postorius.urls')),
+    path(r'archives/', include('hyperkitty.urls')),
+    path(r'', include('django_mailman3.urls')),
+    path(r'accounts/', include('allauth.urls')),
+    # Django admin
+    path(r'admin/', admin.site.urls),
+]

docker/mailman/web/mailman-web/uwsgi.ini

@@ -0,0 +1,53 @@
+[uwsgi]
+# Port on which uwsgi will be listening.
+uwsgi-socket = 0.0.0.0:8080
+http-socket = 0.0.0.0:8000
+
+#Enable threading for python
+enable-threads = true
+
+# Setting uwsgi buffer size to what Apache2 supports.
+buffer-size = 8190
+
+# Move to the directory wher the django files are.
+chdir = /opt/mailman-web
+
+# Use the wsgi file provided with the django project.
+wsgi-file = wsgi.py
+
+# Setup default number of processes and threads per process.
+master = true
+processes = 2
+threads = 2
+
+# Drop privielges and don't run as root.
+uid = mailman
+gid = mailman
+
+# Setup the django_q related worker processes.
+attach-daemon = ./manage.py qcluster
+
+# Setup hyperkitty's cron jobs.
+# 'minutely' jobs are run hourly for perf reasons.
+# See https://github.com/maxking/docker-mailman/issues/327
+unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs minutely
+unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly
+unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly
+unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily
+unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly
+unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly
+unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly
+
+# Setup the request log.
+req-logger = file:/opt/mailman-web-data/logs/uwsgi.log
+
+# Log cron seperately.
+logger = cron file:/opt/mailman-web-data/logs/uwsgi-cron.log
+log-route = cron uwsgi-cron
+
+# Log qcluster commands seperately.
+logger = qcluster file:/opt/mailman-web-data/logs/uwsgi-qcluster.log
+log-route = qcluster uwsgi-daemons
+
+# Last log and it logs the rest of the stuff.
+logger = file:/opt/mailman-web-data/logs/uwsgi-error.log

docker/mailman/web/mailman-web/wsgi.py

@@ -0,0 +1,38 @@
+"""
+WSGI config for HyperKitty project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/
+"""
+
+import os
+
+# import sys
+# import site
+
+# For some unknown reason, sometimes mod_wsgi fails to set the python paths to
+# the virtualenv, with the 'python-path' option. You can do it here too.
+#
+# # Remember original sys.path.
+# prev_sys_path = list(sys.path)
+# # Add here, for the settings module
+# site.addsitedir(os.path.abspath(os.path.dirname(__file__)))
+# # Add the virtualenv
+# venv = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+#                     '..', 'lib', 'python2.6', 'site-packages')
+# site.addsitedir(venv)
+# # Reorder sys.path so new directories at the front.
+# new_sys_path = []
+# for item in list(sys.path):
+#     if item not in prev_sys_path:
+#         new_sys_path.append(item)
+#         sys.path.remove(item)
+#         sys.path[:0] = new_sys_path
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+
+application = get_wsgi_application()

docker/mailman/web/requirements.txt

@@ -0,0 +1,4 @@
+mailmanclient==3.3.4
+postorius==1.3.7
+hyperkitty==1.3.6
+django-mailman3==1.3.8

playbooks/roles/mailman3/templates/docker-compose.yaml.j2

@@ -4,7 +4,7 @@
 version: '2'
 services:
   mailman-core:
-    image: docker.io/maxking/mailman-core:0.4
+    image: docker.io/opendevorg/mailman-core:latest
     restart: always
     container_name: mailman-core
     volumes:
@@ -27,7 +27,7 @@ services:
     #user: mailman
 
   mailman-web:
-    image: docker.io/maxking/mailman-web:0.4
+    image: docker.io/opendevorg/mailman-web:latest
     restart: always
     container_name: mailman-web
     depends_on:

playbooks/roles/mailman3/templates/mailman.vhost.j2

@@ -39,7 +39,7 @@
   SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
 
   Alias /static /var/lib/mailman/web-data/static
-  Alias /favicon.ico /var/lib/mailman/web-data/static/hyperkitty/img/favicon.ico
+  Alias /favicon.ico /var/lib/mailman/web-data/static/archives/img/favicon.ico
 
   <Location "/admin">
     Require local
@@ -47,8 +47,8 @@
 
   RewriteEngine On
   RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1"
-  RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/postorius/lists/$1.%{HTTP_HOST}/"
+  RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/mailman3/lists/$1.%{HTTP_HOST}/"
-  RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/postorius/lists/"
+  RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/mailman3/lists/"
 
   ProxyPassMatch ^/static/ !
   ProxyPass "/" "uwsgi://localhost:8080/"

testinfra/test_lists_opendev_org.py

@@ -39,22 +39,22 @@ def test_apache2_listening(host):
 def test_mailman3_screenshots(host):
     shots = (
         ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),
-        ("https://lists.opendev.org:443/hyperkitty/",
+        ("https://lists.opendev.org:443/archives/",
          None, "mm3-opendev-archives.png"),
         ("https://lists.opendev.org:443"
-         "/hyperkitty/list/service-discuss@lists.opendev.org/",
+         "/archives/list/service-discuss@lists.opendev.org/",
          None, "mm3-opendev-list.png"),
         ("https://lists.opendev.org:443"
-         "/accounts/login/?next=/postorius/lists/",
+         "/accounts/login/?next=/mailman3/lists/",
          None, "mm3-opendev-login.png"),
         ("https://lists.openstack.org:443", None, "mm3-openstack-main.png"),
-        ("https://lists.openstack.org:443/hyperkitty/",
+        ("https://lists.openstack.org:443/archives/",
          None, "mm3-openstack-archives.png"),
         ("https://lists.openstack.org:443"
-         "/hyperkitty/list/openstack-discuss@lists.openstack.org/",
+         "/archives/list/openstack-discuss@lists.openstack.org/",
          None, "mm3-openstack-list.png"),
         ("https://lists.openstack.org:443"
-         "/accounts/login/?next=/postorius/lists/",
+         "/accounts/login/?next=/mailman3/lists/",
          None, "mm3-openstack-login.png"),
     )
 

tools/run-bashate.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 ROOT=$(readlink -fn $(dirname $0)/.. )
-find $ROOT -type f -not -wholename \*.tox/\* -and \( -name \*.sh \
+find $ROOT -type f -not -wholename \*.tox/\* -and \
-    -or -name \*rc -or -name functions\* \) -print0 \
+    \( -name \*.sh -or -name \*rc -or -name functions\* \) -and \
+    -not -path \*/docker/mailman/\* \
+    -print0 \
     | xargs -0 bashate -i E006,E010 -v

zuul.d/docker-images/mailman.yaml

@@ -0,0 +1,29 @@
+- job:
+    name: system-config-build-image-mailman
+    description: Build mailman docker images
+    provides: mailman-container-images
+    parent: system-config-build-image
+    vars: &mailman_vars
+      docker_images:
+        - context: docker/mailman/core
+          repository: opendevorg/mailman-core
+        - context: docker/mailman/web
+          repository: opendevorg/mailman-web
+      docker_use_buildkit: true
+    files: &mailman_files
+      - docker/mailman/
+
+- job:
+    name: system-config-upload-image-mailman
+    description: Build and upload mailman images.
+    provides: mailman-container-images
+    parent: system-config-upload-image
+    vars: *mailman_vars
+    files: *mailman_files
+
+- job:
+    name: system-config-promote-image-mailman
+    description: Promote previously published mailman images to latest.
+    parent: system-config-promote-image
+    vars: *mailman_vars
+    files: *mailman_files

zuul.d/project.yaml

@@ -30,7 +30,11 @@
                 soft: true
         - system-config-run-kerberos
         - system-config-run-lists
-        - system-config-run-lists3
+        - system-config-run-lists3:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-build-image-mailman
+                soft: true
         - system-config-run-nodepool:
             dependencies:
               - name: opendev-buildset-registry
@@ -114,6 +118,7 @@
               - name: system-config-build-image-python-base-3.9-bullseye
                 soft: true
         - system-config-build-image-etherpad
+        - system-config-build-image-mailman
         - system-config-build-image-gitea:
             dependencies:
               - name: opendev-buildset-registry
@@ -182,7 +187,11 @@
                 soft: true
         - system-config-run-kerberos
         - system-config-run-lists
-        - system-config-run-lists3
+        - system-config-run-lists3:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-upload-image-mailman
+                soft: true
         - system-config-run-nodepool:
             dependencies:
               - name: opendev-buildset-registry
@@ -254,6 +263,7 @@
         - system-config-upload-image-hound
         - system-config-upload-image-assets
         - system-config-upload-image-etherpad
+        - system-config-upload-image-mailman
         - system-config-upload-image-gitea:
             dependencies:
               - name: opendev-buildset-registry
@@ -305,6 +315,7 @@
         - system-config-promote-image-gitea-init
         - system-config-promote-image-gitea
         - system-config-promote-image-etherpad
+        - system-config-promote-image-mailman
         - system-config-promote-image-haproxy-statsd
         - system-config-promote-image-zookeeper-statsd
         - system-config-promote-image-accessbot
@@ -471,6 +482,8 @@
                 soft: true
               - name: infra-prod-letsencrypt
                 soft: true
+              - name: system-config-promote-image-mailman
+                soft: true
         - infra-prod-service-mirror: &infra-prod-service-mirror
             dependencies:
               - name: infra-prod-letsencrypt

zuul.d/system-config-run.yaml

@@ -308,7 +308,8 @@
     name: system-config-run-lists3
     # We don't use the system-config-run-containers base job because we
     # are consuming upstream containers only.
-    parent: system-config-run
+    parent: system-config-run-containers
+    requires: mailman-container-images
     description: |
       Run the playbook for a mailman3 list server.
     timeout: 3600