Clark Boylan 7ad3125814 Add robots.txt to Zuul web
This adds a robots.txt that kindly asks bots to not crawl anything on
zuul. We've seen soem bots crawling which leads to them trolling the
build logs which seems like overkill and increases bandwidth usage in
our donor clouds. Ask them to stop and quiet everything down a bit.

Change-Id: I88d85c7a51159b5b020aa179e24acec55fb42931
2024-04-03 13:31:06 -07:00

82 lines
2.4 KiB
Django/Jinja

<VirtualHost *:80>
ServerName zuul.openstack.org
ServerAdmin webmaster@openstack.org
ErrorLog ${APACHE_LOG_DIR}/zuul-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/zuul-access.log combined-cache
Use UserAgentFilter
Redirect / https://zuul.openstack.org/
</VirtualHost>
<VirtualHost *:443>
ServerName zuul.openstack.org
ServerAdmin webmaster@openstack.org
AllowEncodedSlashes On
ErrorLog ${APACHE_LOG_DIR}/zuul-ssl-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/zuul-ssl-access.log combined-cache
SSLEngine on
SSLProtocol All -SSLv2 -SSLv3
# Note: this list should ensure ciphers that provide forward secrecy
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
SSLHonorCipherOrder on
SSLCertificateFile /etc/letsencrypt-certs/zuul.opendev.org/zuul.opendev.org.cer
SSLCertificateKeyFile /etc/letsencrypt-certs/zuul.opendev.org/zuul.opendev.org.key
SSLCertificateChainFile /etc/letsencrypt-certs/zuul.opendev.org/ca.cer
BrowserMatch "MSIE [2-6]" \
nokeepalive ssl-unclean-shutdown \
downgrade-1.0 force-response-1.0
# MSIE 7 and newer should be able to use keepalive
BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
# This macro relies on mod rewrite rules so put it before any of the
# actual rewrites we want to perform.
Use UserAgentFilter
RewriteEngine on
RewriteRule ^/robots.txt$ /var/www/robots/robots.txt [L]
RewriteRule ^/api/connection/(.*)$ http://127.0.0.1:9000/api/connection/$1 [P,L]
RewriteRule ^/api/console-stream ws://127.0.0.1:9000/api/tenant/openstack/console-stream [P,L]
RewriteRule ^/api/(.*)$ http://127.0.0.1:9000/api/tenant/openstack/$1 [P,L]
RewriteRule ^/(.*)$ http://127.0.0.1:9000/$1 [P,L]
AddOutputFilterByType DEFLATE application/json text/css text/javascript application/javascript
# Enable SHM backend for socache
CacheSocache shmcb
# Anything bigger should fall through to disk
CacheSocacheMaxSize 102400
# This is required to match on rewrites correctly
CacheQuickHandler off
# Disk cache settings
CacheRoot /var/cache/apache2/mod_cache_disk
CacheMaxFileSize 10000000
<LocationMatch "^/api/status">
CacheEnable socache
CacheEnable disk
</LocationMatch>
<Location "/static">
CacheEnable socache
CacheEnable disk
</Location>
<Directory "/var/www/robots">
Require all granted
</Directory>
</VirtualHost>