From e0427cfbb7f7638da6741a2fb918cbaf111259b6 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 19 Sep 2013 16:11:28 -0700 Subject: [PATCH] Deploy elastic-recheck on logstash.openstack.org. This elastic recheck thing that classifies failures based on logstash queries is awesome. We should run it on the OpenStack developer infrastructure. Deploy it on logstash.o.o to keep related services together. Change-Id: I16ae7a32c2c48474454df17931779d0f60c3ed85 --- manifests/site.pp | 15 +- .../files/elastic-recheck.init | 150 ++++++++++++++++++ .../files/recheckwatchbot.yaml | 4 + modules/elastic_recheck/manifests/init.pp | 131 +++++++++++++++ .../templates/elastic-recheck.conf.erb | 13 ++ .../openstack_project/manifests/logstash.pp | 16 ++ 6 files changed, 325 insertions(+), 4 deletions(-) create mode 100644 modules/elastic_recheck/files/elastic-recheck.init create mode 100644 modules/elastic_recheck/files/recheckwatchbot.yaml create mode 100644 modules/elastic_recheck/manifests/init.pp create mode 100644 modules/elastic_recheck/templates/elastic-recheck.conf.erb diff --git a/manifests/site.pp b/manifests/site.pp index 11ef6b3805..1390f1bf87 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -257,9 +257,9 @@ $elasticsearch_nodes = [ node 'logstash.openstack.org' { class { 'openstack_project::logstash': - sysadmins => hiera('sysadmins'), - elasticsearch_nodes => $elasticsearch_nodes, - gearman_workers => [ + sysadmins => hiera('sysadmins'), + elasticsearch_nodes => $elasticsearch_nodes, + gearman_workers => [ 'logstash-worker1.openstack.org', 'logstash-worker2.openstack.org', 'logstash-worker3.openstack.org', @@ -269,7 +269,7 @@ node 'logstash.openstack.org' { 'logstash-worker7.openstack.org', 'logstash-worker8.openstack.org', ], - discover_nodes => [ + discover_nodes => [ 'elasticsearch.openstack.org:9200', 'elasticsearch2.openstack.org:9200', 'elasticsearch3.openstack.org:9200', @@ -277,6 +277,13 @@ node 'logstash.openstack.org' { 'elasticsearch5.openstack.org:9200', 'elasticsearch6.openstack.org:9200', ], + # Config for elastic-recheck + gerrit_ssh_private_key => '/etc/elastic-recheck/id_rsa', + gerrit_ssh_private_key_contents => hiera('elastic-recheck_gerrit_ssh_private_key'), + recheck_bot_nick => 'openstackrecheck', + recheck_bot_passwd => hiera('elastic-recheck_ircbot_password'), + gerrit_host => 'review.openstack.org', + elasticsearch_url => 'http://logstash.openstack.org/elasticsearch/', } } diff --git a/modules/elastic_recheck/files/elastic-recheck.init b/modules/elastic_recheck/files/elastic-recheck.init new file mode 100644 index 0000000000..edaf5d17c8 --- /dev/null +++ b/modules/elastic_recheck/files/elastic-recheck.init @@ -0,0 +1,150 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: elastic-recheck +# Required-Start: $remote_fs $syslog +# Required-Stop: $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Elastic Recheck +# Description: Classify tempest-devstack failures using ElasticSearch +### END INIT INFO + +# Jeremy Stanley + +# Do NOT "set -e" + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Elastic Recheck" +NAME=elastic-recheck +#TODO: rework the following two lines once we have executable entrypoints +DAEMON=/usr/bin/python +DAEMON_ARGS="/opt/$NAME/elastic_recheck/bot.py /etc/$NAME/$NAME.conf" +PIDFILE=/var/run/$NAME/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +USER=recheck + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Load the VERBOSE setting and other rcS variables +. /lib/init/vars.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + + mkdir -p /var/run/$NAME + chown $USER /var/run/$NAME + start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 + # Add code here, if necessary, that waits for the process to be ready + # to handle requests from services started subsequently which depend + # on this one. As a last resort, sleep for some time. +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --signal 9 --pidfile $PIDFILE + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + rm -f /var/run/$NAME/* + return "$RETVAL" +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + # + # If the daemon can reload its configuration without + # restarting (for example, when it is sent a SIGHUP), + # then implement that here. + # + start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name $NAME + return 0 +} + +case "$1" in + start) + [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + stop) + [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; + 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; + esac + ;; + status) + status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + #reload|force-reload) + # + # If do_reload() is not implemented then leave this commented out + # and leave 'force-reload' as an alias for 'restart'. + # + #log_daemon_msg "Reloading $DESC" "$NAME" + #do_reload + #log_end_msg $? + #;; + restart|force-reload) + # + # If the "reload" option is implemented then remove the + # 'force-reload' alias + # + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + +: diff --git a/modules/elastic_recheck/files/recheckwatchbot.yaml b/modules/elastic_recheck/files/recheckwatchbot.yaml new file mode 100644 index 0000000000..d06c15b65d --- /dev/null +++ b/modules/elastic_recheck/files/recheckwatchbot.yaml @@ -0,0 +1,4 @@ +openstack-qa: + events: + - positive + - negative diff --git a/modules/elastic_recheck/manifests/init.pp b/modules/elastic_recheck/manifests/init.pp new file mode 100644 index 0000000000..29381dde61 --- /dev/null +++ b/modules/elastic_recheck/manifests/init.pp @@ -0,0 +1,131 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Class to install and configure an instance of the elastic-recheck +# service. +# +class elastic_recheck ( + $gerrit_host, + $gerrit_ssh_private_key, + $gerrit_ssh_private_key_contents, + #not used today, will be used when elastic-recheck supports it. + $elasticsearch_url, + $recheck_bot_passwd, + $gerrit_user = 'elasticrecheck', + $recheck_bot_nick = 'openstackrecheck' +) { + group { 'recheck': + ensure => 'present', + } + + user { 'recheck': + ensure => present, + home => '/var/run/elastic-recheck', + shell => '/bin/false', + gid => 'recheck', + require => [ + Group['recheck'], + File['/var/run/elastic-recheck'], + ], + } + + vcsrepo { '/opt/elastic-recheck': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://git.openstack.org/openstack-infra/elastic-recheck', + } + + include pip + exec { 'install_elastic-recheck' : + command => 'python setup.py install', + cwd => '/opt/elastic-recheck', + path => '/bin:/usr/bin', + refreshonly => true, + subscribe => Vcsrepo['/opt/elastic-recheck'], + require => Class['pip'], + } + + file { '/var/run/elastic-recheck': + ensure => directory, + mode => '0755', + owner => 'recheck', + group => 'recheck', + } + + file { '/var/log/elastic-recheck': + ensure => directory, + mode => '0755', + owner => 'recheck', + group => 'recheck', + } + + file { '/etc/elastic-recheck': + ensure => directory, + mode => '0755', + owner => 'recheck', + group => 'recheck', + } + + file { '/etc/elastic-recheck/elastic-recheck.conf': + ensure => present, + mode => '0640', + owner => 'recheck', + group => 'recheck', + content => template('elastic_recheck/elastic-recheck.conf.erb'), + } + + file { '/etc/elastic-recheck/recheckwatchbot.yaml': + ensure => present, + mode => '0640', + owner => 'recheck', + group => 'recheck', + source => 'puppet:///modules/elastic_recheck/recheckwatchbot.yaml', + } + + # TODO(clarkb) put queries.json somewhere else. + file { '/etc/elastic-recheck/queries.json': + ensure => link, + target => '/opt/elastic-recheck/queries.json', + require => Vcsrepo['/opt/elastic-recheck'], + } + + file { $gerrit_ssh_private_key: + ensure => present, + mode => '0600', + owner => 'recheck', + group => 'recheck', + content => $gerrit_ssh_private_key_contents, + } + + file { '/etc/init.d/elastic-recheck': + ensure => present, + mode => '0755', + owner => 'root', + group => 'root', + source => 'puppet:///modules/elastic_recheck/elastic-recheck.init', + } + + service { 'elastic-recheck': + ensure => running, + enable => true, + subscribe => File['/etc/elastic-recheck/elastic-recheck.conf'], + require => [ + File['/etc/init.d/elastic-recheck'], + File['/etc/elastic-recheck/elastic-recheck.conf'], + File['/etc/elastic-recheck/queries.json'], + Exec['install_elastic-recheck'], + ], + } +} diff --git a/modules/elastic_recheck/templates/elastic-recheck.conf.erb b/modules/elastic_recheck/templates/elastic-recheck.conf.erb new file mode 100644 index 0000000000..92c4837caa --- /dev/null +++ b/modules/elastic_recheck/templates/elastic-recheck.conf.erb @@ -0,0 +1,13 @@ +#Must use full paths +[ircbot] +nick=<%= recheck_bot_nick %> +pass=<%= recheck_bot_passwd %> +server=irc.freenode.net +port=6667 +channel_config=/etc/elastic-recheck/recheckwatchbot.yaml + +[gerrit] +user=<%= gerrit_user %> +host=<%= gerrit_host %> +query_file=/etc/elastic-recheck/queries.json +key=<%= gerrit_ssh_private_key %> diff --git a/modules/openstack_project/manifests/logstash.pp b/modules/openstack_project/manifests/logstash.pp index c12f70ba95..509993c332 100644 --- a/modules/openstack_project/manifests/logstash.pp +++ b/modules/openstack_project/manifests/logstash.pp @@ -15,6 +15,13 @@ # Logstash web frontend glue class. # class openstack_project::logstash ( + $gerrit_host, + $gerrit_ssh_private_key, + $gerrit_ssh_private_key_contents, + #not used today, will be used when elastic-recheck supports it. + $elasticsearch_url, + $recheck_bot_passwd, + $recheck_bot_nick = 'openstackrecheck', $elasticsearch_nodes = [], $gearman_workers = [], $discover_nodes = ['elasticsearch.openstack.org:9200'], @@ -109,4 +116,13 @@ class openstack_project::logstash ( ], require => Service['jenkins-log-client'], } + + class { 'elastic_recheck': + gerrit_host => $gerrit_host, + gerrit_ssh_private_key => $gerrit_ssh_private_key, + gerrit_ssh_private_key_contents => $gerrit_ssh_private_key_contents, + elasticsearch_url => $elasticsearch_url, + recheck_bot_passwd => $recheck_bot_passwd, + recheck_bot_nick => $recheck_bot_nick, + } }