opendev/system-config
Code Issues Proposed changes

Increase log worker retry timeout length.

Browse Source 
* modules/openstack_project/files/logstash/log-gearman-worker.py:
Increase the log worker retry timeout length to 255 seconds and do an
exponential backoff instead of retrying every second when incomplete
files are received. This should reduce load on the file server and make
console.html indexing more reliable.

Change-Id: I42d8f99b3ba2495bb1cac94c4b44e3598f7b9cb6
This commit is contained in:
Clark Boylan 2013-10-01 16:21:55 -07:00
parent 6fc3a1d583
commit 837801709a
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
modules/openstack_project/files/logstash/log-gearman-worker.py
View File

@ -147,16 +147,18 @@ class LogRetriever(threading.Thread):
raw_buf = r.read()
# Hack to read all of Jenkins console logs as they upload
# asynchronously. Make one attempt per second for up to 60 seconds to
# asynchronously. After each attempt do an exponential backup before
# the next request for up to 255 seconds total, if we do not
# retrieve the entire file. Short circuit when the end of file string
# for console logs, '\n</pre>\n', is read.
if (retry and not gzipped and
raw_buf[-8:].decode('utf-8') != '\n</pre>\n'):
content_len = len(raw_buf)
for i in range(60):
# Try for up to 60 seconds to retrieve the complete log file.
backoff = 1
while backoff < 129:
# Try for up to 255 seconds to retrieve the complete log file.
try:
logging.debug(str(i) + " Retrying fetch of: " +
logging.debug(str(backoff) + " Retrying fetch of: " +
source_url + "?level=INFO")
logging.debug("Fetching bytes=" + str(content_len) + '-')
req = urllib2.Request(source_url + "?level=INFO")
@ -172,7 +174,8 @@ class LogRetriever(threading.Thread):
finally:
if raw_buf[-8:].decode('utf-8') == '\n</pre>\n':
break
semi_busy_wait(1)
semi_busy_wait(backoff)
backoff += backoff
return gzipped, raw_buf