From 979591af872f92e73458e8c7484b4f4f5bd61859 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Wed, 14 Jun 2017 10:06:51 -0500 Subject: [PATCH] KubeADM-AIO: Make init more stable and support multiple PVC backends This PS makes the init of the KubeADM environment more stable by ensuring all kube-system pods are up before trying to launch tiller. the NFS PVC provider is now made optional but enabled by default, so that Ceph (and other more complex providers) can be supported in future. Finaly the scripts to bring up k8s/helm are refined to not bring in the clients if already present with the correct versions on the host. Change-Id: I4d7ceb6196e8fd5e3350ec99f09fbe6bb5fe01f6 --- tools/gate/funcs/helm.sh | 13 ++++-- tools/gate/funcs/kube.sh | 14 ++++-- tools/kubeadm-aio/Dockerfile | 1 + .../assets/usr/bin/wait-for-kube-nodes | 42 +++++++++++++++++ .../assets/usr/bin/wait-for-kube-pods | 46 +++++++++++++++++++ tools/kubeadm-aio/kubeadm-aio-launcher.sh | 10 +++- 6 files changed, 114 insertions(+), 12 deletions(-) create mode 100755 tools/kubeadm-aio/assets/usr/bin/wait-for-kube-nodes create mode 100755 tools/kubeadm-aio/assets/usr/bin/wait-for-kube-pods diff --git a/tools/gate/funcs/helm.sh b/tools/gate/funcs/helm.sh index 109dc69470..7da90e9385 100755 --- a/tools/gate/funcs/helm.sh +++ b/tools/gate/funcs/helm.sh @@ -14,7 +14,6 @@ set -e function helm_install { - TMP_DIR=$(mktemp -d) if [ "x$HOST_OS" == "xubuntu" ]; then sudo apt-get update -y sudo apt-get install -y --no-install-recommends -qq \ @@ -35,10 +34,14 @@ function helm_install { fi # install helm - curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} - sudo mv ${TMP_DIR}/helm /usr/local/bin/helm - - rm -rf ${TMP_DIR} + if CURRENT_HELM_LOC=$(type -p helm); then + CURRENT_HELM_VERSION=$(${CURRENT_HELM_LOC} version --client --short | awk '{ print $NF }' | awk -F '+' '{ print $1 }') + fi + [ "x$HELM_VERSION" == "x$CURRENT_HELM_VERSION" ] || ( \ + TMP_DIR=$(mktemp -d) + curl -sSL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} + sudo mv ${TMP_DIR}/helm /usr/local/bin/helm + rm -rf ${TMP_DIR} ) } function helm_serve { diff --git a/tools/gate/funcs/kube.sh b/tools/gate/funcs/kube.sh index 0c11b81dca..75a445421d 100755 --- a/tools/gate/funcs/kube.sh +++ b/tools/gate/funcs/kube.sh @@ -72,7 +72,6 @@ function kube_wait_for_nodes { } function kubeadm_aio_reqs_install { - TMP_DIR=$(mktemp -d) if [ "x$HOST_OS" == "xubuntu" ]; then sudo apt-get update -y sudo apt-get install -y --no-install-recommends -qq \ @@ -108,11 +107,16 @@ function kubeadm_aio_reqs_install { sudo systemctl restart docker fi - curl -sSL https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl - chmod +x ${TMP_DIR}/kubectl - sudo mv ${TMP_DIR}/kubectl /usr/local/bin/kubectl + if CURRENT_KUBECTL_LOC=$(type -p kubectl); then + CURRENT_KUBECTL_VERSION=$(${CURRENT_KUBECTL_LOC} version --client --short | awk '{ print $NF }' | awk -F '+' '{ print $1 }') + fi + [ "x$KUBE_VERSION" == "x$CURRENT_KUBECTL_VERSION" ] || ( \ + TMP_DIR=$(mktemp -d) + curl -sSL https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl + chmod +x ${TMP_DIR}/kubectl + sudo mv ${TMP_DIR}/kubectl /usr/local/bin/kubectl + rm -rf ${TMP_DIR} ) - rm -rf ${TMP_DIR} } function kubeadm_aio_build { diff --git a/tools/kubeadm-aio/Dockerfile b/tools/kubeadm-aio/Dockerfile index 23a6694090..897a3eb13d 100644 --- a/tools/kubeadm-aio/Dockerfile +++ b/tools/kubeadm-aio/Dockerfile @@ -60,6 +60,7 @@ RUN set -x \ make \ git \ vim \ + jq \ # Install nfs utils for development PVC provisioner nfs-common \ # Tweak Systemd units and targets for running in a container diff --git a/tools/kubeadm-aio/assets/usr/bin/wait-for-kube-nodes b/tools/kubeadm-aio/assets/usr/bin/wait-for-kube-nodes new file mode 100755 index 0000000000..30b12b4a99 --- /dev/null +++ b/tools/kubeadm-aio/assets/usr/bin/wait-for-kube-nodes @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -e + +# Default wait timeout is 180 seconds +: ${KUBECONFIG:="/etc/kubernetes/admin.conf"} +export KUBECONFIG=${KUBECONFIG} + +end=$(date +%s) +if [ x$2 != "x" ]; then + end=$((end + $2)) +else + end=$((end + 180)) +fi +while true; do + NUMBER_OF_NODES=$(kubectl get nodes --no-headers -o name | wc -l) + NUMBER_OF_NODES_EXPECTED=$(($(cat /etc/nodepool/sub_nodes_private | wc -l) + 1)) + [ $NUMBER_OF_NODES -eq $NUMBER_OF_NODES_EXPECTED ] && \ + NODES_ONLINE="True" || NODES_ONLINE="False" + while read SUB_NODE; do + echo $SUB_NODE | grep -q ^Ready && NODES_READY="True" || NODES_READY="False" + done < <(kubectl get nodes --no-headers | awk '{ print $2 }') + [ $NODES_ONLINE == "True" -a $NODES_READY == "True" ] && \ + break || true + sleep 5 + now=$(date +%s) + [ $now -gt $end ] && echo "Nodes Failed to be ready in time." && \ + kubectl get nodes -o wide && exit -1 +done diff --git a/tools/kubeadm-aio/assets/usr/bin/wait-for-kube-pods b/tools/kubeadm-aio/assets/usr/bin/wait-for-kube-pods new file mode 100755 index 0000000000..de9ccebde0 --- /dev/null +++ b/tools/kubeadm-aio/assets/usr/bin/wait-for-kube-pods @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -e + +# From Kolla-Kubernetes, orginal authors Kevin Fox & Serguei Bezverkhi +# Default wait timeout is 180 seconds +: ${KUBECONFIG:="/etc/kubernetes/admin.conf"} +export KUBECONFIG=${KUBECONFIG} + +end=$(date +%s) +if [ x$2 != "x" ]; then + end=$((end + $2)) +else + end=$((end + 180)) +fi +while true; do + kubectl get pods --namespace=$1 -o json | jq -r \ + '.items[].status.phase' | grep Pending > /dev/null && \ + PENDING=True || PENDING=False + query='.items[]|select(.status.phase=="Running")' + query="$query|.status.containerStatuses[].ready" + kubectl get pods --namespace=$1 -o json | jq -r "$query" | \ + grep false > /dev/null && READY="False" || READY="True" + kubectl get jobs -o json --namespace=$1 | jq -r \ + '.items[] | .spec.completions == .status.succeeded' | \ + grep false > /dev/null && JOBR="False" || JOBR="True" + [ $PENDING == "False" -a $READY == "True" -a $JOBR == "True" ] && \ + break || true + sleep 1 + now=$(date +%s) + [ $now -gt $end ] && echo containers failed to start. && \ + kubectl get pods --namespace $1 -o wide && exit -1 +done diff --git a/tools/kubeadm-aio/kubeadm-aio-launcher.sh b/tools/kubeadm-aio/kubeadm-aio-launcher.sh index ae82ab2e9a..2f42fb768c 100755 --- a/tools/kubeadm-aio/kubeadm-aio-launcher.sh +++ b/tools/kubeadm-aio/kubeadm-aio-launcher.sh @@ -89,11 +89,17 @@ while true; do done set -x +# Waiting for kube-system pods to be ready before continuing +sudo docker exec kubeadm-aio wait-for-kube-pods kube-system + # Initialize Helm helm init # Initialize Environment for Development sudo docker exec kubeadm-aio openstack-helm-dev-prep -# Deploy NFS provisioner into enviromment -sudo docker exec kubeadm-aio openstack-helm-nfs-prep +: ${PVC_BACKEND:="nfs"} +if [ "$PVC_BACKEND" == "nfs" ]; then + # Deploy NFS provisioner into enviromment + sudo docker exec kubeadm-aio openstack-helm-nfs-prep +fi