Add robots.txt to our list servers

We've noticed that our uwsgi queues are filling up and a lot of requests
are being made to robots.txt which ends up 500/503 erroring. Add a
robots.txt file which allows crawling of our lists and archives with a
delay value in hopes this will cause bots to cache results and not fill
up the queue with repetetive requests.

Change-Id: I660d8d43f6b2d96663212d93ec48e67d86e9e761

playbooks/roles/mailman3/files/robots.txt

@@ -0,0 +1,7 @@
+User-agent: *
+
+Disallow: /accounts/*
+Allow: /archives/*
+Allow: /mailman3/lists/*
+
+Crawl-delay: 2

playbooks/roles/mailman3/tasks/main.yaml

@@ -141,6 +141,22 @@
   shell:
     cmd: docker image prune -f
 
+- name: Create robots.txt location dir
+  file:
+    path: /var/www/robots
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Copy the robots.txt
+  copy:
+    src: robots.txt
+    dest: /var/www/robots/robots.txt
+    owner: root
+    group: root
+    mode: '0644'
+
 - name: Install apache2
   package:
     name:

playbooks/roles/mailman3/templates/mailman.vhost.j2

@@ -46,6 +46,7 @@
   </Location>
 
   RewriteEngine On
+  RewriteRule ^/robots.txt$ /var/www/robots/robots.txt [L]
   RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1"
   RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/mailman3/lists/$1.%{HTTP_HOST}/"
   RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/mailman3/lists/"
@@ -66,4 +67,8 @@
     Allow from all
     Require all granted
   </Directory>
+
+  <Directory "/var/www/robots">
+    Require all granted
+  </Directory>
 </VirtualHost>

testinfra/test_lists_opendev_org.py

@@ -36,6 +36,13 @@ def test_apache2_listening(host):
     apache2_https = host.socket("tcp://0.0.0.0:443")
     assert apache2_https.is_listening
 
+def test_robots(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve lists.opendev.org:443:127.0.0.1 '
+                   'https://lists.opendev.org/robots.txt')
+    assert 'Disallow: /accounts/*' in cmd.stdout
+    assert 'Allow: /archives/*' in cmd.stdout
+
 def test_mailman3_screenshots(host):
     shots = (
         ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),