From 442e8d258986c51121206ade559d128157ad47b6 Mon Sep 17 00:00:00 2001 From: Mikhail S Medvedev Date: Wed, 28 Jan 2015 20:02:12 -0600 Subject: [PATCH] Split out subunit2sql module The spec: http://specs.openstack.org/openstack-infra/infra-specs/specs/puppet-modules.html Depends-On: I139c15503a9bc9495f1bbbbe77676408e0dc1e47 Change-Id: Ie3f4edacc527b186940d136a3f24463b9320a00e --- modules.env | 1 + .../files/subunit-gearman-worker.py | 278 ------------------ modules/subunit2sql/manifests/init.pp | 102 ------- modules/subunit2sql/manifests/server.pp | 41 --- modules/subunit2sql/manifests/worker.pp | 93 ------ .../templates/jenkins-subunit-worker.init.erb | 158 ---------- .../templates/subunit2sql.conf.erb | 4 - 7 files changed, 1 insertion(+), 676 deletions(-) delete mode 100644 modules/subunit2sql/files/subunit-gearman-worker.py delete mode 100644 modules/subunit2sql/manifests/init.pp delete mode 100644 modules/subunit2sql/manifests/server.pp delete mode 100644 modules/subunit2sql/manifests/worker.pp delete mode 100755 modules/subunit2sql/templates/jenkins-subunit-worker.init.erb delete mode 100644 modules/subunit2sql/templates/subunit2sql.conf.erb diff --git a/modules.env b/modules.env index 790e5ade43..628ba61617 100644 --- a/modules.env +++ b/modules.env @@ -66,6 +66,7 @@ INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-gerritbot" INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-graphite"]="origin/master" INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-statusbot"]="origin/master" INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-storyboard"]="origin/master" +INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-subunit2sql"]="origin/master" INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-iptables"]="origin/master" INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-zuul"]="origin/master" INTEGRATION_MODULES["https://git.openstack.org/openstack-infra/puppet-kibana"]="origin/master" diff --git a/modules/subunit2sql/files/subunit-gearman-worker.py b/modules/subunit2sql/files/subunit-gearman-worker.py deleted file mode 100644 index 48638125da..0000000000 --- a/modules/subunit2sql/files/subunit-gearman-worker.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import argparse -import cStringIO -import daemon -import gear -import gzip -import json -import logging -import os -import Queue -import socket -import subprocess -import threading -import time -import urllib2 -import yaml - -from subunit2sql import read_subunit -from subunit2sql import shell - - -try: - import daemon.pidlockfile as pidfile_mod -except ImportError: - import daemon.pidfile as pidfile_mod - - -def semi_busy_wait(seconds): - # time.sleep() may return early. If it does sleep() again and repeat - # until at least the number of seconds specified has elapsed. - start_time = time.time() - while True: - time.sleep(seconds) - cur_time = time.time() - seconds = seconds - (cur_time - start_time) - if seconds <= 0.0: - return - - -class FilterException(Exception): - pass - - -class SubunitRetriever(threading.Thread): - def __init__(self, gearman_worker, filters, subunitq): - threading.Thread.__init__(self) - self.gearman_worker = gearman_worker - self.filters = filters - self.subunitq = subunitq - - def run(self): - while True: - try: - self._handle_event() - except: - logging.exception("Exception retrieving log event.") - - def _handle_event(self): - job = self.gearman_worker.getJob() - try: - arguments = json.loads(job.arguments.decode('utf-8')) - source_url = arguments['source_url'] - retry = arguments['retry'] - event = arguments['event'] - logging.debug("Handling event: " + json.dumps(event)) - fields = event.get('fields') or event.get('@fields') - if fields.pop('build_status') != 'ABORTED': - # Handle events ignoring aborted builds. These builds are - # discarded by zuul. - subunit_io = self._retrieve_subunit_v2(source_url, retry) - logging.debug("Pushing subunit files.") - out_event = fields.copy() - out_event["subunit"] = subunit_io - self.subunitq.put(out_event) - job.sendWorkComplete() - except Exception as e: - logging.exception("Exception handling log event.") - job.sendWorkException(str(e).encode('utf-8')) - - def _subunit_1_to_2(self, raw_file): - call = subprocess.Popen('subunit-1to2', stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - output, err = call.communicate(raw_file.read()) - if err: - raise Exception(err) - buf = cStringIO.StringIO(output) - return buf - - def _retrieve_subunit_v2(self, source_url, retry): - # TODO (clarkb): This should check the content type instead of file - # extension for determining if gzip was used. - gzipped = False - raw_buf = b'' - try: - gzipped, raw_buf = self._get_subunit_data(source_url, retry) - except urllib2.HTTPError as e: - if e.code == 404: - logging.info("Unable to retrieve %s: HTTP error 404" % - source_url) - else: - logging.exception("Unable to get log data.") - except Exception: - # Silently drop fatal errors when retrieving logs. - # TODO (clarkb): Handle these errors. - # Perhaps simply add a log message to raw_buf? - logging.exception("Unable to get log data.") - if gzipped: - logging.debug("Decompressing gzipped source file.") - raw_strIO = cStringIO.StringIO(raw_buf) - f = gzip.GzipFile(fileobj=raw_strIO) - buf = self._subunit_1_to_2(f) - raw_strIO.close() - f.close() - else: - logging.debug("Decoding source file.") - raw_strIO = cStringIO.StringIO(raw_buf) - buf = self._subunit_1_to_2(raw_strIO) - return buf - - def _get_subunit_data(self, source_url, retry): - gzipped = False - try: - # TODO(clarkb): We really should be using requests instead - # of urllib2. urllib2 will automatically perform a POST - # instead of a GET if we provide urlencoded data to urlopen - # but we need to do a GET. The parameters are currently - # hardcoded so this should be ok for now. - logging.debug("Retrieving: " + source_url + ".gz") - req = urllib2.Request(source_url + ".gz") - req.add_header('Accept-encoding', 'gzip') - r = urllib2.urlopen(req) - except urllib2.URLError: - try: - # Fallback on GETting unzipped data. - logging.debug("Retrieving: " + source_url) - r = urllib2.urlopen(source_url) - except: - logging.exception("Unable to retrieve source file.") - raise - except: - logging.exception("Unable to retrieve source file.") - raise - if ('gzip' in r.info().get('Content-Type', '') or - 'gzip' in r.info().get('Content-Encoding', '')): - gzipped = True - - raw_buf = r.read() - # Hack to read all of Jenkins console logs as they upload - return gzipped, raw_buf - - -class Subunit2SQLProcessor(object): - def __init__(self, subunitq, subunit2sql_conf): - self.subunitq = subunitq - self.config = subunit2sql_conf - # Initialize subunit2sql settings - shell.cli_opts() - shell.parse_args([], [self.config]) - - def handle_subunit_event(self): - # Pull subunit event from queue and separate stream from metadata - subunit = self.subunitq.get() - subunit_v2 = subunit.pop('subunit') - # Set run metadata from gearman - log_url = subunit.pop('log_url', None) - if log_url: - log_dir = os.path.dirname(os.path.dirname(log_url)) - shell.CONF.set_override('artifacts', log_dir) - run_id = subunit.get('build_uuid', None) - if run_id: - shell.CONF.set_override('run_id', run_id) - shell.CONF.set_override('run_meta', subunit) - # Parse subunit stream and store in DB - logging.debug('Converting Subunit V2 stream to SQL') - stream = read_subunit.ReadSubunit(subunit_v2) - shell.process_results(stream.get_results()) - - -class Server(object): - def __init__(self, config, debuglog): - # Config init. - self.config = config - self.gearman_host = self.config['gearman-host'] - self.gearman_port = self.config['gearman-port'] - # Pythong logging output file. - self.debuglog = debuglog - self.retriever = None - self.subunitqueue = Queue.Queue(131072) - self.processor = None - self.filter_factories = [] - - def setup_logging(self): - if self.debuglog: - logging.basicConfig(format='%(asctime)s %(message)s', - filename=self.debuglog, level=logging.DEBUG) - else: - # Prevent leakage into the logstash log stream. - logging.basicConfig(level=logging.CRITICAL) - logging.debug("Log pusher starting.") - - def setup_retriever(self): - hostname = socket.gethostname() - gearman_worker = gear.Worker(hostname + b'-pusher') - gearman_worker.addServer(self.gearman_host, - self.gearman_port) - gearman_worker.registerFunction(b'push-subunit') - self.retriever = SubunitRetriever(gearman_worker, - self.filter_factories, - self.subunitqueue) - - def setup_processor(self): - # Note this processor will not work if the process is run as a - # daemon. You must use the --foreground option. - subunit2sql_config = self.config['config'] - self.processor = Subunit2SQLProcessor(self.subunitqueue, - subunit2sql_config) - - def main(self): - self.setup_retriever() - self.setup_processor() - - self.retriever.daemon = True - self.retriever.start() - - while True: - try: - self.processor.handle_subunit_event() - except: - logging.exception("Exception processing log event.") - raise - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--config", required=True, - help="Path to yaml config file.") - parser.add_argument("-d", "--debuglog", - help="Enable debug log. " - "Specifies file to write log to.") - parser.add_argument("--foreground", action='store_true', - help="Run in the foreground.") - parser.add_argument("-p", "--pidfile", - default="/var/run/jenkins-subunit-pusher/" - "jenkins-subunit-gearman-worker.pid", - help="PID file to lock during daemonization.") - args = parser.parse_args() - - with open(args.config, 'r') as config_stream: - config = yaml.load(config_stream) - server = Server(config, args.debuglog) - - if args.foreground: - server.setup_logging() - server.main() - else: - pidfile = pidfile_mod.TimeoutPIDLockFile(args.pidfile, 10) - with daemon.DaemonContext(pidfile=pidfile): - server.setup_logging() - server.main() - - -if __name__ == '__main__': - main() diff --git a/modules/subunit2sql/manifests/init.pp b/modules/subunit2sql/manifests/init.pp deleted file mode 100644 index f344736e7c..0000000000 --- a/modules/subunit2sql/manifests/init.pp +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# == Class: subunit2sql -# -class subunit2sql ( -) { - include pip - - package {'python-mysqldb': - ensure => present, - } - - package {'python-psycopg2': - ensure => present, - } - - package { 'python-subunit': - ensure => latest, - provider => 'pip', - require => Class['pip'], - } - - package { 'subunit2sql': - ensure => latest, - provider => 'pip', - require => [ - Class['pip'], - Package['python-mysqldb'], - Package['python-psycopg2'] - ], - } - - package { 'testtools': - ensure => latest, - provider => 'pip', - require => Class['pip'], - } - - if ! defined(Package['python-daemon']) { - package { 'python-daemon': - ensure => present, - } - } - - if ! defined(Package['python-zmq']) { - package { 'python-zmq': - ensure => present, - } - } - - if ! defined(Package['python-yaml']) { - package { 'python-yaml': - ensure => present, - } - } - - if ! defined(Package['gear']) { - package { 'gear': - ensure => latest, - provider => 'pip', - require => Class['pip'], - } - } - - if ! defined(Package['statsd']) { - package { 'statsd': - ensure => latest, - provider => 'pip', - require => Class['pip'] - } - } - - file { '/usr/local/bin/subunit-gearman-worker.py': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/subunit2sql/subunit-gearman-worker.py', - require => [ - Package['python-daemon'], - Package['python-zmq'], - Package['python-yaml'], - Package['gear'], - Package['subunit2sql'], - Package['python-subunit'], - Package['testtools'] - ], - } -} diff --git a/modules/subunit2sql/manifests/server.pp b/modules/subunit2sql/manifests/server.pp deleted file mode 100644 index a825468836..0000000000 --- a/modules/subunit2sql/manifests/server.pp +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# == Class: subunit2sql -# -class subunit2sql::server ( - $db_dialect = 'mysql', - $db_user = 'subunit2sql', - $db_pass, - $db_host, - $db_port = '3306', - $db_name = 'subunit2sql', -) { - - file { '/etc/subunit2sql.conf': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - content => template('subunit2sql/subunit2sql.conf.erb'), - } - - exec { 'upgrade_subunit2sql_db': - command => 'subunit2sql-db-manage --config-file /etc/subunit2sql.conf upgrade head', - path => '/usr/local/bin:/usr/bin:/bin/', - subscribe => Package['subunit2sql'], - refreshonly => true, - } -} diff --git a/modules/subunit2sql/manifests/worker.pp b/modules/subunit2sql/manifests/worker.pp deleted file mode 100644 index d03038a5e2..0000000000 --- a/modules/subunit2sql/manifests/worker.pp +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# == Class: subunit_processor::worker -# -define subunit2sql::worker ( - $config_file, - $db_dialect = 'mysql', - $db_user = 'subunit2sql', - $db_pass, - $db_host, - $db_port = '3306', - $db_name = 'subunit2sql' -) { - $suffix = "-${name}" - - if ! defined(File['/etc/logstash']) { - file { '/etc/logstash': - ensure => directory, - owner => 'logstash', - group => 'logstash', - mode => '0644', - } - } - - if ! defined(File['/etc/logstash/subunit2sql.conf']) { - file { '/etc/logstash/subunit2sql.conf': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - content => template('subunit2sql/subunit2sql.conf.erb'), - require => File['/etc/logstash'], - } - } - - file { "/etc/logstash/jenkins-subunit-worker${suffix}.yaml": - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => $config_file, - require => File['/etc/logstash'], - } - - file { "/etc/init.d/jenkins-subunit-worker${suffix}": - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - content => template('subunit2sql/jenkins-subunit-worker.init.erb'), - require => [ - File['/usr/local/bin/subunit-gearman-worker.py'], - File["/etc/logstash/jenkins-subunit-worker${suffix}.yaml"], - ], - } - - service { "jenkins-subunit-worker${suffix}": - enable => true, - hasrestart => true, - subscribe => File["/etc/logstash/jenkins-subunit-worker${suffix}.yaml"], - require => [ - File['/etc/logstash'], - File["/etc/init.d/jenkins-subunit-worker${suffix}"], - ], - } - - include logrotate - logrotate::file { "subunit-worker${suffix}-debug.log": - log => "/var/log/logstash/subunit-worker${suffix}-debug.log", - options => [ - 'compress', - 'copytruncate', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Service["jenkins-subunit-worker${suffix}"], - } -} diff --git a/modules/subunit2sql/templates/jenkins-subunit-worker.init.erb b/modules/subunit2sql/templates/jenkins-subunit-worker.init.erb deleted file mode 100755 index 5425202806..0000000000 --- a/modules/subunit2sql/templates/jenkins-subunit-worker.init.erb +++ /dev/null @@ -1,158 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: jenkins-subunit-worker -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Jenkins Subunit Worker -# Description: Service to push Jenkins subunit files into a SQL DB. -### END INIT INFO - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin -DESC="Jenkins Subunit Worker" -NAME=jenkins-subunit-worker<%= suffix %> -DAEMON=/usr/local/bin/subunit-gearman-worker.py -PIDFILE=/var/run/$NAME/$NAME.pid -DAEMON_ARGS="-c /etc/logstash/jenkins-subunit-worker<%= suffix %>.yaml -d /var/log/logstash/subunit-worker<%= suffix %>-debug.log -p $PIDFILE" -SCRIPTNAME=/etc/init.d/$NAME -USER=logstash - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - - mkdir -p /var/run/$NAME - chown $USER /var/run/$NAME - start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \ - $DAEMON_ARGS \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --signal 9 --pidfile $PIDFILE - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - rm -f /var/run/$NAME/* - return "$RETVAL" -} - -# -# Function that stops the daemon/service -# -#do_graceful_stop() -#{ -# PID=`cat $PIDFILE` -# kill -USR1 $PID -# -# # wait until really stopped -# if [ -n "${PID:-}" ]; then -# i=0 -# while kill -0 "${PID:-}" 2> /dev/null; do -# if [ $i -eq '0' ]; then -# echo -n " ... waiting " -# else -# echo -n "." -# fi -# i=$(($i+1)) -# sleep 1 -# done -# fi -# -# rm -f /var/run/$NAME/* -#} - -# -# Function that sends a SIGHUP to the daemon/service -# -#do_reload() { -# # -# # If the daemon can reload its configuration without -# # restarting (for example, when it is sent a SIGHUP), -# # then implement that here. -# # -# start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name zuul-server -# return 0 -#} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? - ;; -# reload) -# # -# # If do_reload() is not implemented then leave this commented out -# # and leave 'force-reload' as an alias for 'restart'. -# # -# log_daemon_msg "Reloading $DESC" "$NAME" -# do_reload -# log_end_msg $? -# ;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - do_start - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/modules/subunit2sql/templates/subunit2sql.conf.erb b/modules/subunit2sql/templates/subunit2sql.conf.erb deleted file mode 100644 index 70c0166907..0000000000 --- a/modules/subunit2sql/templates/subunit2sql.conf.erb +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] - -[database] -connection = <%= @db_dialect %>://<%= @db_user %>:<%= @db_pass %>@<%= @db_host %>:<%= @db_port %>/<%= @db_name %>