From 837801709a433e7b8e7cf5c74b2195b6e6493a11 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 1 Oct 2013 16:21:55 -0700 Subject: [PATCH] Increase log worker retry timeout length. * modules/openstack_project/files/logstash/log-gearman-worker.py: Increase the log worker retry timeout length to 255 seconds and do an exponential backoff instead of retrying every second when incomplete files are received. This should reduce load on the file server and make console.html indexing more reliable. Change-Id: I42d8f99b3ba2495bb1cac94c4b44e3598f7b9cb6 --- .../files/logstash/log-gearman-worker.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/openstack_project/files/logstash/log-gearman-worker.py b/modules/openstack_project/files/logstash/log-gearman-worker.py index c7539f7872..a0f43d0f56 100644 --- a/modules/openstack_project/files/logstash/log-gearman-worker.py +++ b/modules/openstack_project/files/logstash/log-gearman-worker.py @@ -147,16 +147,18 @@ class LogRetriever(threading.Thread): raw_buf = r.read() # Hack to read all of Jenkins console logs as they upload - # asynchronously. Make one attempt per second for up to 60 seconds to + # asynchronously. After each attempt do an exponential backup before + # the next request for up to 255 seconds total, if we do not # retrieve the entire file. Short circuit when the end of file string # for console logs, '\n\n', is read. if (retry and not gzipped and raw_buf[-8:].decode('utf-8') != '\n\n'): content_len = len(raw_buf) - for i in range(60): - # Try for up to 60 seconds to retrieve the complete log file. + backoff = 1 + while backoff < 129: + # Try for up to 255 seconds to retrieve the complete log file. try: - logging.debug(str(i) + " Retrying fetch of: " + + logging.debug(str(backoff) + " Retrying fetch of: " + source_url + "?level=INFO") logging.debug("Fetching bytes=" + str(content_len) + '-') req = urllib2.Request(source_url + "?level=INFO") @@ -172,7 +174,8 @@ class LogRetriever(threading.Thread): finally: if raw_buf[-8:].decode('utf-8') == '\n\n': break - semi_busy_wait(1) + semi_busy_wait(backoff) + backoff += backoff return gzipped, raw_buf