From 4749e620b6d3e04a5647d8a7f8a6b984b714af2e Mon Sep 17 00:00:00 2001 From: Roger Luethi Date: Sun, 8 Mar 2015 14:12:59 +0100 Subject: [PATCH] labs: repeat test from snapshot With this changeset, the cluster build can be tested starting from a specific snapshot. This is mostly useful for testing client scripts without having to go through the whole cluster build every time. For instance, to rebuild the cluster from snapshot neuntron_configured: ./tools/repeat-test.sh -b -t neutron_configured -s "controller compute" This will restore the appropriate snapshot on each node, boot nodes controller and compute, and continue the cluster build from the named snapshot onward. Note: this procedure restores, boots, and continues in short order which uncovers new shortcomings (races) in scripts that assume a service is available when they should check and wait for the service. If you just want to restore all nodes to the appropriate snapshots: ./tools/restore-cluster.sh -t Change-Id: I0b7888d02de6eb50760d421d6d897f516f8c7718 --- labs/lib/osbash/functions-host.sh | 9 +- labs/osbash.sh | 20 +++- labs/tools/repeat-test.sh | 91 ++++++++++++--- labs/tools/restore-cluster.sh | 182 +++++++++++++++++++++--------- 4 files changed, 230 insertions(+), 72 deletions(-) diff --git a/labs/lib/osbash/functions-host.sh b/labs/lib/osbash/functions-host.sh index d7e6d905..ebf8fcc3 100644 --- a/labs/lib/osbash/functions-host.sh +++ b/labs/lib/osbash/functions-host.sh @@ -397,7 +397,14 @@ function autostart_from_config { # Skip empty lines and lines that are commented out continue elif [ "$field_1" == "cmd" ]; then - command_from_config $field_2 + if [ -n "${JUMP_SNAPSHOT:-""}" ]; then + if [[ $field_2 =~ ^snapshot.*${JUMP_SNAPSHOT} ]]; then + echo >&2 "Skipped forward to snapshot $JUMP_SNAPSHOT." + unset JUMP_SNAPSHOT + fi + else + command_from_config $field_2 + fi else # Syntax error echo -n >&2 "ERROR in $config_file: '$field_1" diff --git a/labs/osbash.sh b/labs/osbash.sh index cad79e9f..3e82e5ed 100755 --- a/labs/osbash.sh +++ b/labs/osbash.sh @@ -22,13 +22,17 @@ source "$OSBASH_LIB_DIR/virtualbox-install_base.sh" source "$LIB_DIR/osbash/lib-color.sh" function usage { - echo "Usage: $0 {-b|-w} [-g GUI] [--no-color] [-n] {TARGET}" + echo "Usage: $0 {-b|-w} [-g GUI] [--no-color] [-n] [-t SNAP] {TARGET}" # Don't advertise export until it is working properly # echo " $0 [-e EXPORT] [-n] NODE [NODE..]" echo echo "-h|--help Help" echo "-n Print configuration status and exit" echo "-b Build basedisk (if necessary) and node VMs (if any)" + + # Don't use -t directly, have tools/repeat-test.sh call it + #echo "-t SNAP Jump to snapshot SNAP and continue build" + echo "-w Create Windows batch files" echo "-g GUI GUI type during build" #echo "-e EXPORT Export node VMs" @@ -38,6 +42,10 @@ function usage { echo " cluster : build OpenStack cluster [all nodes]" echo " (and basedisk if necessary)" echo "GUI gui, sdl, or headless (GUI type for VirtualBox)" + + # Don't use -t SNAP directly, have tools/repeat-test.sh call it + #echo "SNAP Name of snapshot from which build continues" + #echo "EXPORT ova (OVA package file) or dir (VM clone directory)" exit } @@ -67,11 +75,16 @@ function print_config { # GUI is the VirtualBox default echo -e "${CInfo:-}GUI type:${CData:-} ${VM_UI:-gui}${CReset:-}" + + if [ -n "${JUMP_SNAPSHOT:-}" ]; then + echo -e "${CInfo:-}Continuing from snapshot:" \ + "${CData:-}${JUMP_SNAPSHOT}${CReset:-}" + fi fi } -while getopts :be:g:-:hnw opt; do +while getopts :be:g:-:hnt:w opt; do case $opt in e) if [ "$OPTARG" = ova ]; then @@ -116,6 +129,9 @@ while getopts :be:g:-:hnw opt; do n) INFO_ONLY=1 ;; + t) + JUMP_SNAPSHOT=$OPTARG + ;; w) source "$LIB_DIR/wbatch/batch_for_windows" ;; diff --git a/labs/tools/repeat-test.sh b/labs/tools/repeat-test.sh index be7d9d29..43e81932 100755 --- a/labs/tools/repeat-test.sh +++ b/labs/tools/repeat-test.sh @@ -3,7 +3,11 @@ set -o errexit -o nounset TOP_DIR=$(cd "$(dirname "$0")/.." && pwd) source "$TOP_DIR/config/paths" source "$CONFIG_DIR/deploy.osbash" +source "$CONFIG_DIR/provider.virtualbox" source "$OSBASH_LIB_DIR/functions-host.sh" +source "$OSBASH_LIB_DIR/virtualbox-functions.sh" + +OSBASH=exec_cmd LOG_NAME=test.log RESULTS_ROOT=$LOG_DIR/test-results @@ -14,22 +18,59 @@ TEST_SCRIPT=$TOP_DIR/scripts/test/launch_instance.sh VERBOSE=${VERBOSE:=1} function usage { - echo "Usage: $0 {rebuild|restore}" - echo " rebuild: rebuild cluster for each test (osbash.sh -b cluster)" - echo " restore: restore cluster for each test (cluster-restore.sh)" - exit 1 + echo "Usage: $0 {-b|-c|-t } [-s '']" + echo "" + echo "-h Help" + echo "-c Restore node VMs to current snapshot for each test" + echo "-t SNAP Restore cluster to target snapshot for each test" + echo "-s NODES Start each named node VM after restoring the cluster" + echo "-b Rebuild cluster for each test, from scratch or snapshot" + echo " (osbash.sh -b cluster [...])" } -if [ $# = 0 ]; then - usage -elif [ "$1" = "rebuild" ]; then - INIT=rebuild -elif [ "$1" = "restore" ]; then - unset INIT -else +while getopts :bchs:t: opt; do + case $opt in + b) + REBUILD=yes + ;; + c) + CURRENT=yes + ;; + h) + usage + exit 0 + ;; + s) + START_VMS=$OPTARG + ;; + t) + TARGET_SNAPSHOT=$OPTARG + if ! "$TOP_DIR/tools/restore-cluster.sh" -l | + grep -q "Name: $TARGET_SNAPSHOT "; then + echo >&2 "No snapshot named $TARGET_SNAPSHOT found." + exit 1 + fi + ;; + :) + echo "Error: -$OPTARG needs argument" + ;; + ?) + echo "Error: invalid option -$OPTARG" + echo + usage + exit 1 + ;; + esac +done + +if [ -z "${REBUILD:-}" -a -z "${CURRENT:-}" -a -z "${TARGET_SNAPSHOT:-}" ]; then usage + exit 1 fi +# Remove processed options from arguments +shift $(( OPTIND - 1 )); + mkdir -p "$RESULTS_ROOT" while [ : ]; do @@ -41,18 +82,32 @@ while [ : ]; do ( cd "$TOP_DIR" - if [ "${INIT:=""}" = "rebuild" ]; then - echo "Building cluster." - "$TOP_DIR/osbash.sh" -b cluster - else - echo "Restoring cluster." - "$TOP_DIR/tools/restore-cluster.sh" "$CONTROLLER_SNAPSHOT" + if [ -n "${TARGET_SNAPSHOT:-}" ]; then + "$TOP_DIR/tools/restore-cluster.sh" -t "$TARGET_SNAPSHOT" + if [ -n "${START_VMS:-}" ]; then + # Start VMs as requested by user + for vm_name in $START_VMS; do + echo >&2 "$0: booting node $vm_name." + vbox_boot "$vm_name" + # Sleeping for 10 s fixes some problems, but it might be + # better to fix client scripts to wait for the services they + # need instead of just failing. + done + fi + fi + + if [ -n "${REBUILD:-}" ]; then + if [ -n "${TARGET_SNAPSHOT:-}" ]; then + "$TOP_DIR/osbash.sh" -t "$TARGET_SNAPSHOT" -b cluster + else + "$TOP_DIR/osbash.sh" -b cluster + fi fi echo "Running test. Log file: $dir/$LOG_NAME" rc=0 TEST_ONCE=$TOP_DIR/tools/test-once.sh - if [ "$VERBOSE" -eq 1 ]; then + if [ "${VERBOSE:-}" -eq 1 ]; then "$TEST_ONCE" "$TEST_SCRIPT" 2>&1 | tee "$dir/$LOG_NAME" || rc=$? else "$TEST_ONCE" "$TEST_SCRIPT" > "$dir/$LOG_NAME" 2>&1 || rc=$? diff --git a/labs/tools/restore-cluster.sh b/labs/tools/restore-cluster.sh index d4309cc9..22ee1e12 100755 --- a/labs/tools/restore-cluster.sh +++ b/labs/tools/restore-cluster.sh @@ -3,73 +3,153 @@ set -o errexit -o nounset TOP_DIR=$(cd "$(dirname "$0")/.." && pwd) source "$TOP_DIR/config/paths" source "$CONFIG_DIR/deploy.osbash" +source "$CONFIG_DIR/provider.virtualbox" source "$OSBASH_LIB_DIR/functions-host.sh" +source "$OSBASH_LIB_DIR/virtualbox-functions.sh" -CONTROLLER_VM=controller -NETWORK_VM=network -COMPUTE_VM=compute +OSBASH=exec_cmd + +VM_LIST="controller network compute" function usage { # Setting to empty string selects latest (current snapshot) - echo "Usage: $0 {current|} {list-snapshots}" - echo " current: restore to currently active snapshot" - echo " list-snapshots: to list the snapshots of the vms" + echo "Usage: $0 {-l|-c|-t } [-s]" + echo "" + echo "-h Help" + echo "-l List snapshots for node VMs" + echo "-c Restore cluster node VMs to current snapshot" + echo "-t SNAP Restore cluster to target snapshot" + echo "-s Start each node VMs after restoring it" exit } -function cluster_restore { - vboxmanage controlvm $CONTROLLER_VM poweroff >/dev/null 2>&1 || rc=$? - sleep 1 - if [ -n "$CONTROLLER_SNAPSHOT" ]; then - echo "Restoring $CONTROLLER_SNAPSHOT." - vboxmanage snapshot $CONTROLLER_VM restore "$CONTROLLER_SNAPSHOT" - else - echo "Restoring current snapshot." - vboxmanage snapshot $CONTROLLER_VM restorecurrent - fi - - vboxmanage controlvm $COMPUTE_VM poweroff >/dev/null 2>&1 || rc=$? - sleep 1 - vboxmanage snapshot $COMPUTE_VM restorecurrent - - vboxmanage controlvm $NETWORK_VM poweroff >/dev/null 2>&1 || rc=$? - sleep 1 - vboxmanage snapshot $NETWORK_VM restorecurrent -} - -function cluster_start { - vboxmanage startvm $CONTROLLER_VM -t headless - vboxmanage startvm $COMPUTE_VM -t headless - vboxmanage startvm $NETWORK_VM -t headless -} - function list_snapshots { - - for node in $CONTROLLER_VM $COMPUTE_VM $NETWORK_VM; do - echo -e "\n$node node's Snapshot" - vboxmanage snapshot $node list + for vm_name in $VM_LIST; do + echo -e "Snapshot list for $vm_name node:" + vboxmanage snapshot "$vm_name" list echo - echo - sleep 1 done - exit 0 } -# Call the main brains -if [ $# -eq 0 ]; then +while getopts :chlst: opt; do + case $opt in + c) + CURRENT=yes + ;; + h) + usage + ;; + l) + list_snapshots + ;; + s) + START=yes + ;; + t) + TARGET_SNAPSHOT=$OPTARG + ;; + :) + echo "Error: -$OPTARG needs argument" + ;; + ?) + echo "Error: invalid option -$OPTARG" + echo + usage + ;; + esac +done + +# Remove processed options from arguments +shift $(( OPTIND - 1 )); + +if [ $# -ne 0 ]; then + usage +elif [ -z "${TARGET_SNAPSHOT:-}" -a -z "${CURRENT:-""}" ]; then + echo + echo "Error: no target snapshot given." + echo + usage +elif [ -n "{$TARGET_SNAPSHOT:-}" -a -n "${CURRENT:-""}" ]; then + echo + echo "Error: conflicting options: target snapshot name and -c." + echo usage -elif [ "$1" = "list-snapshots" ]; then - list_snapshots -elif [ "$1" = "current" ]; then - CONTROLLER_SNAPSHOT="" -else - CONTROLLER_SNAPSHOT=$1 fi +# Find target_snapshot in scripts_cfg and set global *_SNAPSHOT variables +# to the correct snapshot name for each node (to allow building from there) +function set_snapshot_vars { + local target_snapshot=$1 -echo "Restoring cluster snapshots." -cluster_restore + local found=0 + local scripts_cfg="$TOP_DIR/config/scripts.ubuntu_cluster" + + while read -r line; do + if [[ $line =~ ^cmd\ snapshot.*-n\ ([^ ]*)\ (.*) ]]; then + # Node name (e.g. controller) + node=${BASH_REMATCH[1]} + + # Snapshot name (e.g. keystone_installed) + snapshot=${BASH_REMATCH[2]} + + # Global variable name (e.g. CONTROLLER_SNAPSHOT) + # Can't use ${node,,} (OS X bash version is only 3.2) + var_name=$(echo "$node"|tr "a-z" "A-Z")_SNAPSHOT + + if [ "$snapshot" = "$target_snapshot" ]; then + # Can't use associative arrays (OS X bash version is only 3.2) + eval "${var_name}=$snapshot" + + found=1 + elif [ $found -eq 0 ]; then + eval "${var_name}=$snapshot" + fi + fi + done < "$scripts_cfg" + + if [ $found -eq 0 ]; then + echo "ERROR: snapshot '$target_snapshot' not found" + exit 1 + fi +} + +function restore_current_snapshot { + local vm_name=$1 + vboxmanage snapshot "$vm_name" restorecurrent +} + +function restore_named_snapshot { + local vm_name=$1 + local snapshot=$2 + vboxmanage snapshot "$vm_name" restore "$snapshot" +} + +if [ -n "$TARGET_SNAPSHOT" ]; then + set_snapshot_vars "$TARGET_SNAPSHOT" +fi + +for vm_name in $VM_LIST; do + vm_power_off "$vm_name" + vm_wait_for_shutdown "$vm_name" + + if [ "${CURRENT:-""}" = "yes" ]; then + restore_current_snapshot "$vm_name" + if [ "${START:-""}" = "yes" ]; then + vbox_boot "$vm_name" + fi + else + # Global variable name (e.g. CONTROLLER_SNAPSHOT) + # (use tr due to OS X bash limitation) + var_name=$(echo "$vm_name"|tr "a-z" "A-Z")_SNAPSHOT + if [ -z "${!var_name:=""}" ]; then + vm_delete "$vm_name" + else + restore_named_snapshot "$vm_name" "${!var_name}" + if [ "${START:-""}" = "yes" ]; then + vbox_boot "$vm_name" + fi + fi + fi +done -echo "Starting VMs." -cluster_start >/dev/null