From 8fb306a22b5f6ed4f308d767fc64c6400a295c91 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 10 Mar 2020 09:43:16 -0700
Subject: [PATCH] Add periodic job to run goaccess reporting

This adds a simple periodic job that runs against static.opendev.org and
produces opendev reports. To start we don't publish these reports
publicly so that we can double check their contents first. Assuming that
all goes well we'll be able to apply this to all of our
static.opendev.org hosted sites and publish these reports through the
zuul logging system.

Change-Id: I66705808d435c16bf0da6989296c896099964aaa
Story: 2007387
---
 .zuul.yaml                             | 22 ++++++++++
 playbooks/periodic/goaccess.yaml       | 60 ++++++++++++++++++++++++++
 playbooks/roles/static/tasks/main.yaml |  7 ++-
 3 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 playbooks/periodic/goaccess.yaml

diff --git a/.zuul.yaml b/.zuul.yaml
index c89a05c039..c67fb62599 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1189,6 +1189,27 @@
       nodes: []
     run: playbooks/periodic/404.yaml
 
+- job:
+    name: system-config-goaccess-report
+    description: |
+      Run goaccess tool on static.opendev.org hosted site.
+    abstract: true
+    protected: true
+    nodeset:
+      nodes: []
+    run: playbooks/periodic/goaccess.yaml
+    # TODO add publishing step once output is checked and clean.
+
+- job:
+    name: zuul-ci-goaccess-report
+    parent: system-config-goaccess-report
+    description: |
+      Run goaccess tool on zuul-ci.org.
+    final: true
+    protected: true
+    vars:
+      goaccess_site: zuul-ci.org
+
 - project:
     templates:
       - system-config-zuul-role-integration
@@ -1359,3 +1380,4 @@
     periodic:
       jobs:
         - system-config-static-404-checker
+        - zuul-ci-goaccess-report
diff --git a/playbooks/periodic/goaccess.yaml b/playbooks/periodic/goaccess.yaml
new file mode 100644
index 0000000000..9b0dce80fc
--- /dev/null
+++ b/playbooks/periodic/goaccess.yaml
@@ -0,0 +1,60 @@
+- hosts: localhost
+  tasks:
+    - name: Add static.opendev.org to inventory
+      add_host:
+        name: static.opendev.org
+        ansible_connection: ssh
+        ansible_host: static.opendev.org
+        ansible_port: 22
+        ansible_user: zuul
+
+    - name: Add static.opendev.org host key
+      known_hosts:
+        name: static.opendev.org
+        key: static.opendev.org,23.253.245.150,2001:4800:7818:101:be76:4eff:fe04:7c28 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMu3PnnkNhPS2d5Z2uPju3Qqcbbc0lwHA1j9MgHlLnbK3bx1O2Kfez6RJUGl2i6nshdzkKwPBvN2vehQKiw1oSk=
+
+- hosts: static.opendev.org
+  tasks:
+    - name: Configure goaccess
+      copy:
+        dest: /home/zuul/goaccess.conf
+        owner: zuul
+        group: zuul
+        mode: '0444'
+        content: |
+          log-format COMBINED
+
+          ignore-panel VISITORS
+          ignore-panel REQUESTS
+          ignore-panel REQUESTS_STATIC
+          ignore-panel NOT_FOUND
+          ignore-panel HOSTS
+          ignore-panel OS
+          ignore-panel BROWSERS
+          ignore-panel VISIT_TIMES
+          ignore-panel VIRTUAL_HOSTS
+          ignore-panel REFERRERS
+          ignore-panel REFERRING_SITES
+          ignore-panel KEYPHRASES
+          ignore-panel STATUS_CODES
+          ignore-panel REMOTE_USER
+          ignore-panel GEO_LOCATION
+
+          enable-panel VISITORS
+          enable-panel REQUESTS
+          enable-panel REQUESTS_STATIC
+          enable-panel NOT_FOUND
+          enable-panel STATUS_CODES
+    - name: Run goaccess
+      become: yes
+      shell: |
+        # We will publish to the zuul homedir for now as that allows us to
+        # double check the data produced is clean before making it
+        # public.
+        OUT_FILE="/home/zuul/{{ goaccess_site }}_goaccess_report.html"
+        SOURCE_FILE="/var/log/apache2/{{ goaccess_site }}_access.log"
+        # This pipes in all the compressed logs as well as the current
+        # uncompressed logs. Giving us a window as wide as our log retention.
+        zcat ${SOURCE_FILE}.*.gz | goaccess -o $OUT_FILE -p /home/zuul/goaccess.conf $SOURCE_FILE ${SOURCE_FILE}.1 -
+      args:
+        executable: /bin/bash
diff --git a/playbooks/roles/static/tasks/main.yaml b/playbooks/roles/static/tasks/main.yaml
index d85e56b38e..2dfa82c5c2 100644
--- a/playbooks/roles/static/tasks/main.yaml
+++ b/playbooks/roles/static/tasks/main.yaml
@@ -21,6 +21,11 @@
       - apache2-utils
     state: present
 
+- name: Install goaccess
+  apt:
+    name: goaccess
+    state: present
+
 - name: Rewrite module
   apache2_module:
     state: present
@@ -95,4 +100,4 @@
     - git.starlingx.io
     - git.zuul-ci.org
   loop_control:
-    loop_var: hostname
\ No newline at end of file
+    loop_var: hostname