opendev/system-config
Code Issues Proposed changes

Make nodepool git repo caching more robust

Browse Source 
Previously, nodepool's prep scripts would not abort when
encountering git cloning errors. Now we retry once on failure and
then abort if it fails a second time. Also the script now makes sure
the clone actually results in a functional copy of each repository
by attempting to list branches and reset to HEAD.

Change-Id: I84565e4e90b34c6a9b8c86754efdde137cda08b2
This commit is contained in:
Jeremy Stanley 2014-02-21 15:52:32 +00:00
parent eccd20e38c
commit 4c9ca0597b
1 changed files with 40 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
modules/openstack_project/files/nodepool/scripts/cache_git_repos.py
View File

@ -16,7 +16,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path
import re
import shutil
import urllib2
from common import run_local
@ -26,6 +28,36 @@ URL = ('http://git.openstack.org/cgit/openstack-infra/config/plain/'
PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
def clone_repo(project):
remote = 'git://git.openstack.org/%s.git' % project
# Clear out any existing target directory first, in case of a retry.
try:
shutil.rmtree(os.path.join('/opt/git', project))
except OSError:
pass
# Try to clone the requested git repository.
(status, out) = run_local(['git', 'clone', remote, project],
status=True, cwd='/opt/git')
# If it claims to have worked, make sure we can list branches.
if status == 0:
(status, moreout) = run_local(['git', 'branch', '-a'], status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# If that worked, try resetting to HEAD to make sure it's there.
if status == 0:
(status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# Status of 0 imples all the above worked, 1 means something failed.
return (status, out)
def main():
# TODO(jeblair): use gerrit rest api when available
data = urllib2.urlopen(URL).read()
@ -34,9 +66,14 @@ def main():
# YAML module which is not in the stdlib.
m = PROJECT_RE.match(line)
if m:
project = 'git://git.openstack.org/%s' % m.group(1)
print run_local(['git', 'clone', project, m.group(1)],
cwd='/opt/git')
(status, out) = clone_repo(m.group(1))
print out
if status != 0:
print 'Retrying to clone %s' % m.group(1)
(status, out) = clone_repo(m.group(1))
print out
if status != 0:
raise Exception('Failed to clone %s' % m.group(1))
if __name__ == '__main__':