Stop redirecting for the paste site

The pastebinit command-line tool hard-codes an allowed list of
pastebin URLs, one of which is "http://paste.openstack.org" so
redirecting to HTTPS and to other hostnames seems to break it.

It has a specific user-agent, so allow plain HTTP access for this
tool, but redirect others.

Change-Id: Ia7c983986e6e9c08299ded5282a83761448b35bb

playbooks/roles/lodgeit/templates/paste.vhost.j2

@@ -2,13 +2,42 @@
   ServerName {{ inventory_hostname }}
   ServerAdmin webmaster@openstack.org
 
+  AllowEncodedSlashes On
+
   ErrorLog ${APACHE_LOG_DIR}/paste-error.log
 
   LogLevel warn
 
   CustomLog ${APACHE_LOG_DIR}/paste-access.log combined
 
-  Redirect / https://paste.opendev.org/
+  RewriteEngine On
+
+  # NOTE(fungi) 2021-08-13 : The pastebinit command-line tool
+  # hard-codes an allowed list of pastebin URLs, one of which is
+  # "http://paste.openstack.org" so redirecting to HTTPS and to other
+  # hostnames seems to break it.  We allow it to access the http site
+  # for compatability, but redirect anything that doesn't look like it
+  # over to the https site.
+  RewriteCond %{HTTP_USER_AGENT} !"Pastebinit" [NC]
+  RewriteRule ^(.*)$ https://paste.opendev.org$1 [R=301,L]
+
+  # NOTE(ianw) 2021-08-04 : block GET /json/?method=pastes.getRecent
+  # This a) gives Python 3 unicode errors we haven't looked into and b)
+  # is only used by bots as a vector for scraping things.  Just
+  # disable it.
+  RewriteCond %{QUERY_STRING} "method=pastes.getRecent"
+  RewriteRule .* - [F,L]
+
+  ProxyPass  / http://localhost:9000/ retry=0
+  ProxyPassReverse / http://localhost:9000/
+
+  <Location "/robots.txt">
+    ProxyPass !
+  </Location>
+  <Directory "/var/lib/lodgeit/www">
+    Require all granted
+  </Directory>
+  Alias /robots.txt /var/lib/lodgeit/www/robots.txt
 
 </VirtualHost>
 

testinfra/test_paste.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import requests
 from selenium import webdriver
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.common.exceptions import TimeoutException
@@ -33,6 +34,19 @@ def test_paste(host):
                    'https://paste.opendev.org')
     assert 'New Paste' in cmd.stdout
 
+def test_paste_redirects(host):
+    # http site should redirect all agents but Pastebinit
+    r = requests.get(
+        'http://%s' % host.backend.get_hostname(), allow_redirects=False)
+    assert r.status_code == 301
+    assert r.headers['Location'] == 'https://paste.opendev.org/'
+
+    headers = {
+        'User-Agent': 'Pastebinit v1.2.3'
+    }
+    r = requests.get('http://%s' % (host.backend.get_hostname()))
+    assert r.status_code == 200
+
 def test_paste_robots(host):
     cmd = host.run('curl --insecure '
                    '--resolve paste.opendev.org:443:127.0.0.1 '

tox.ini

@@ -41,6 +41,7 @@ deps =
   pytest-html # MPL-2.0
   pytest-testinfra>=6.0.0
   selenium
+  requests
 
 # This environment assumes a gate-hosts.yaml file has been written.
 passenv =