labs: repeat test from snapshot
With this changeset, the cluster build can be tested starting from a specific snapshot. This is mostly useful for testing client scripts without having to go through the whole cluster build every time. For instance, to rebuild the cluster from snapshot neuntron_configured: ./tools/repeat-test.sh -b -t neutron_configured -s "controller compute" This will restore the appropriate snapshot on each node, boot nodes controller and compute, and continue the cluster build from the named snapshot onward. Note: this procedure restores, boots, and continues in short order which uncovers new shortcomings (races) in scripts that assume a service is available when they should check and wait for the service. If you just want to restore all nodes to the appropriate snapshots: ./tools/restore-cluster.sh -t <snapshot> Change-Id: I0b7888d02de6eb50760d421d6d897f516f8c7718
This commit is contained in:
Roger Luethi
parent
f8df591df5
commit
4749e620b6
@ -397,7 +397,14 @@ function autostart_from_config {
|
||||
# Skip empty lines and lines that are commented out
|
||||
continue
|
||||
elif [ "$field_1" == "cmd" ]; then
|
||||
command_from_config $field_2
|
||||
if [ -n "${JUMP_SNAPSHOT:-""}" ]; then
|
||||
if [[ $field_2 =~ ^snapshot.*${JUMP_SNAPSHOT} ]]; then
|
||||
echo >&2 "Skipped forward to snapshot $JUMP_SNAPSHOT."
|
||||
unset JUMP_SNAPSHOT
|
||||
fi
|
||||
else
|
||||
command_from_config $field_2
|
||||
fi
|
||||
else
|
||||
# Syntax error
|
||||
echo -n >&2 "ERROR in $config_file: '$field_1"
|
||||
|
||||
@ -22,13 +22,17 @@ source "$OSBASH_LIB_DIR/virtualbox-install_base.sh"
|
||||
source "$LIB_DIR/osbash/lib-color.sh"
|
||||
|
||||
function usage {
|
||||
echo "Usage: $0 {-b|-w} [-g GUI] [--no-color] [-n] {TARGET}"
|
||||
echo "Usage: $0 {-b|-w} [-g GUI] [--no-color] [-n] [-t SNAP] {TARGET}"
|
||||
# Don't advertise export until it is working properly
|
||||
# echo " $0 [-e EXPORT] [-n] NODE [NODE..]"
|
||||
echo
|
||||
echo "-h|--help Help"
|
||||
echo "-n Print configuration status and exit"
|
||||
echo "-b Build basedisk (if necessary) and node VMs (if any)"
|
||||
|
||||
# Don't use -t directly, have tools/repeat-test.sh call it
|
||||
#echo "-t SNAP Jump to snapshot SNAP and continue build"
|
||||
|
||||
echo "-w Create Windows batch files"
|
||||
echo "-g GUI GUI type during build"
|
||||
#echo "-e EXPORT Export node VMs"
|
||||
@ -38,6 +42,10 @@ function usage {
|
||||
echo " cluster : build OpenStack cluster [all nodes]"
|
||||
echo " (and basedisk if necessary)"
|
||||
echo "GUI gui, sdl, or headless (GUI type for VirtualBox)"
|
||||
|
||||
# Don't use -t SNAP directly, have tools/repeat-test.sh call it
|
||||
#echo "SNAP Name of snapshot from which build continues"
|
||||
|
||||
#echo "EXPORT ova (OVA package file) or dir (VM clone directory)"
|
||||
exit
|
||||
}
|
||||
@ -67,11 +75,16 @@ function print_config {
|
||||
|
||||
# GUI is the VirtualBox default
|
||||
echo -e "${CInfo:-}GUI type:${CData:-} ${VM_UI:-gui}${CReset:-}"
|
||||
|
||||
if [ -n "${JUMP_SNAPSHOT:-}" ]; then
|
||||
echo -e "${CInfo:-}Continuing from snapshot:" \
|
||||
"${CData:-}${JUMP_SNAPSHOT}${CReset:-}"
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
while getopts :be:g:-:hnw opt; do
|
||||
while getopts :be:g:-:hnt:w opt; do
|
||||
case $opt in
|
||||
e)
|
||||
if [ "$OPTARG" = ova ]; then
|
||||
@ -116,6 +129,9 @@ while getopts :be:g:-:hnw opt; do
|
||||
n)
|
||||
INFO_ONLY=1
|
||||
;;
|
||||
t)
|
||||
JUMP_SNAPSHOT=$OPTARG
|
||||
;;
|
||||
w)
|
||||
source "$LIB_DIR/wbatch/batch_for_windows"
|
||||
;;
|
||||
|
||||
@ -3,7 +3,11 @@ set -o errexit -o nounset
|
||||
TOP_DIR=$(cd "$(dirname "$0")/.." && pwd)
|
||||
source "$TOP_DIR/config/paths"
|
||||
source "$CONFIG_DIR/deploy.osbash"
|
||||
source "$CONFIG_DIR/provider.virtualbox"
|
||||
source "$OSBASH_LIB_DIR/functions-host.sh"
|
||||
source "$OSBASH_LIB_DIR/virtualbox-functions.sh"
|
||||
|
||||
OSBASH=exec_cmd
|
||||
|
||||
LOG_NAME=test.log
|
||||
RESULTS_ROOT=$LOG_DIR/test-results
|
||||
@ -14,22 +18,59 @@ TEST_SCRIPT=$TOP_DIR/scripts/test/launch_instance.sh
|
||||
VERBOSE=${VERBOSE:=1}
|
||||
|
||||
function usage {
|
||||
echo "Usage: $0 {rebuild|restore}"
|
||||
echo " rebuild: rebuild cluster for each test (osbash.sh -b cluster)"
|
||||
echo " restore: restore cluster for each test (cluster-restore.sh)"
|
||||
exit 1
|
||||
echo "Usage: $0 {-b|-c|-t <SNAP>} [-s '<NODES>']"
|
||||
echo ""
|
||||
echo "-h Help"
|
||||
echo "-c Restore node VMs to current snapshot for each test"
|
||||
echo "-t SNAP Restore cluster to target snapshot for each test"
|
||||
echo "-s NODES Start each named node VM after restoring the cluster"
|
||||
echo "-b Rebuild cluster for each test, from scratch or snapshot"
|
||||
echo " (osbash.sh -b cluster [...])"
|
||||
}
|
||||
|
||||
if [ $# = 0 ]; then
|
||||
usage
|
||||
elif [ "$1" = "rebuild" ]; then
|
||||
INIT=rebuild
|
||||
elif [ "$1" = "restore" ]; then
|
||||
unset INIT
|
||||
else
|
||||
while getopts :bchs:t: opt; do
|
||||
case $opt in
|
||||
b)
|
||||
REBUILD=yes
|
||||
;;
|
||||
c)
|
||||
CURRENT=yes
|
||||
;;
|
||||
h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
s)
|
||||
START_VMS=$OPTARG
|
||||
;;
|
||||
t)
|
||||
TARGET_SNAPSHOT=$OPTARG
|
||||
if ! "$TOP_DIR/tools/restore-cluster.sh" -l |
|
||||
grep -q "Name: $TARGET_SNAPSHOT "; then
|
||||
echo >&2 "No snapshot named $TARGET_SNAPSHOT found."
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
:)
|
||||
echo "Error: -$OPTARG needs argument"
|
||||
;;
|
||||
?)
|
||||
echo "Error: invalid option -$OPTARG"
|
||||
echo
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${REBUILD:-}" -a -z "${CURRENT:-}" -a -z "${TARGET_SNAPSHOT:-}" ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove processed options from arguments
|
||||
shift $(( OPTIND - 1 ));
|
||||
|
||||
mkdir -p "$RESULTS_ROOT"
|
||||
|
||||
while [ : ]; do
|
||||
@ -41,18 +82,32 @@ while [ : ]; do
|
||||
(
|
||||
cd "$TOP_DIR"
|
||||
|
||||
if [ "${INIT:=""}" = "rebuild" ]; then
|
||||
echo "Building cluster."
|
||||
"$TOP_DIR/osbash.sh" -b cluster
|
||||
else
|
||||
echo "Restoring cluster."
|
||||
"$TOP_DIR/tools/restore-cluster.sh" "$CONTROLLER_SNAPSHOT"
|
||||
if [ -n "${TARGET_SNAPSHOT:-}" ]; then
|
||||
"$TOP_DIR/tools/restore-cluster.sh" -t "$TARGET_SNAPSHOT"
|
||||
if [ -n "${START_VMS:-}" ]; then
|
||||
# Start VMs as requested by user
|
||||
for vm_name in $START_VMS; do
|
||||
echo >&2 "$0: booting node $vm_name."
|
||||
vbox_boot "$vm_name"
|
||||
# Sleeping for 10 s fixes some problems, but it might be
|
||||
# better to fix client scripts to wait for the services they
|
||||
# need instead of just failing.
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "${REBUILD:-}" ]; then
|
||||
if [ -n "${TARGET_SNAPSHOT:-}" ]; then
|
||||
"$TOP_DIR/osbash.sh" -t "$TARGET_SNAPSHOT" -b cluster
|
||||
else
|
||||
"$TOP_DIR/osbash.sh" -b cluster
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Running test. Log file: $dir/$LOG_NAME"
|
||||
rc=0
|
||||
TEST_ONCE=$TOP_DIR/tools/test-once.sh
|
||||
if [ "$VERBOSE" -eq 1 ]; then
|
||||
if [ "${VERBOSE:-}" -eq 1 ]; then
|
||||
"$TEST_ONCE" "$TEST_SCRIPT" 2>&1 | tee "$dir/$LOG_NAME" || rc=$?
|
||||
else
|
||||
"$TEST_ONCE" "$TEST_SCRIPT" > "$dir/$LOG_NAME" 2>&1 || rc=$?
|
||||
|
||||
@ -3,73 +3,153 @@ set -o errexit -o nounset
|
||||
TOP_DIR=$(cd "$(dirname "$0")/.." && pwd)
|
||||
source "$TOP_DIR/config/paths"
|
||||
source "$CONFIG_DIR/deploy.osbash"
|
||||
source "$CONFIG_DIR/provider.virtualbox"
|
||||
source "$OSBASH_LIB_DIR/functions-host.sh"
|
||||
source "$OSBASH_LIB_DIR/virtualbox-functions.sh"
|
||||
|
||||
CONTROLLER_VM=controller
|
||||
NETWORK_VM=network
|
||||
COMPUTE_VM=compute
|
||||
OSBASH=exec_cmd
|
||||
|
||||
VM_LIST="controller network compute"
|
||||
|
||||
function usage {
|
||||
# Setting to empty string selects latest (current snapshot)
|
||||
echo "Usage: $0 {current|<controller_snapshot_name>} {list-snapshots}"
|
||||
echo " current: restore to currently active snapshot"
|
||||
echo " list-snapshots: to list the snapshots of the vms"
|
||||
echo "Usage: $0 {-l|-c|-t <SNAP>} [-s]"
|
||||
echo ""
|
||||
echo "-h Help"
|
||||
echo "-l List snapshots for node VMs"
|
||||
echo "-c Restore cluster node VMs to current snapshot"
|
||||
echo "-t SNAP Restore cluster to target snapshot"
|
||||
echo "-s Start each node VMs after restoring it"
|
||||
exit
|
||||
}
|
||||
|
||||
function cluster_restore {
|
||||
vboxmanage controlvm $CONTROLLER_VM poweroff >/dev/null 2>&1 || rc=$?
|
||||
sleep 1
|
||||
if [ -n "$CONTROLLER_SNAPSHOT" ]; then
|
||||
echo "Restoring $CONTROLLER_SNAPSHOT."
|
||||
vboxmanage snapshot $CONTROLLER_VM restore "$CONTROLLER_SNAPSHOT"
|
||||
else
|
||||
echo "Restoring current snapshot."
|
||||
vboxmanage snapshot $CONTROLLER_VM restorecurrent
|
||||
fi
|
||||
|
||||
vboxmanage controlvm $COMPUTE_VM poweroff >/dev/null 2>&1 || rc=$?
|
||||
sleep 1
|
||||
vboxmanage snapshot $COMPUTE_VM restorecurrent
|
||||
|
||||
vboxmanage controlvm $NETWORK_VM poweroff >/dev/null 2>&1 || rc=$?
|
||||
sleep 1
|
||||
vboxmanage snapshot $NETWORK_VM restorecurrent
|
||||
}
|
||||
|
||||
function cluster_start {
|
||||
vboxmanage startvm $CONTROLLER_VM -t headless
|
||||
vboxmanage startvm $COMPUTE_VM -t headless
|
||||
vboxmanage startvm $NETWORK_VM -t headless
|
||||
}
|
||||
|
||||
function list_snapshots {
|
||||
|
||||
for node in $CONTROLLER_VM $COMPUTE_VM $NETWORK_VM; do
|
||||
echo -e "\n$node node's Snapshot"
|
||||
vboxmanage snapshot $node list
|
||||
for vm_name in $VM_LIST; do
|
||||
echo -e "Snapshot list for $vm_name node:"
|
||||
vboxmanage snapshot "$vm_name" list
|
||||
echo
|
||||
echo
|
||||
sleep 1
|
||||
done
|
||||
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Call the main brains
|
||||
if [ $# -eq 0 ]; then
|
||||
while getopts :chlst: opt; do
|
||||
case $opt in
|
||||
c)
|
||||
CURRENT=yes
|
||||
;;
|
||||
h)
|
||||
usage
|
||||
;;
|
||||
l)
|
||||
list_snapshots
|
||||
;;
|
||||
s)
|
||||
START=yes
|
||||
;;
|
||||
t)
|
||||
TARGET_SNAPSHOT=$OPTARG
|
||||
;;
|
||||
:)
|
||||
echo "Error: -$OPTARG needs argument"
|
||||
;;
|
||||
?)
|
||||
echo "Error: invalid option -$OPTARG"
|
||||
echo
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Remove processed options from arguments
|
||||
shift $(( OPTIND - 1 ));
|
||||
|
||||
if [ $# -ne 0 ]; then
|
||||
usage
|
||||
elif [ -z "${TARGET_SNAPSHOT:-}" -a -z "${CURRENT:-""}" ]; then
|
||||
echo
|
||||
echo "Error: no target snapshot given."
|
||||
echo
|
||||
usage
|
||||
elif [ -n "{$TARGET_SNAPSHOT:-}" -a -n "${CURRENT:-""}" ]; then
|
||||
echo
|
||||
echo "Error: conflicting options: target snapshot name and -c."
|
||||
echo
|
||||
usage
|
||||
elif [ "$1" = "list-snapshots" ]; then
|
||||
list_snapshots
|
||||
elif [ "$1" = "current" ]; then
|
||||
CONTROLLER_SNAPSHOT=""
|
||||
else
|
||||
CONTROLLER_SNAPSHOT=$1
|
||||
fi
|
||||
|
||||
# Find target_snapshot in scripts_cfg and set global *_SNAPSHOT variables
|
||||
# to the correct snapshot name for each node (to allow building from there)
|
||||
function set_snapshot_vars {
|
||||
local target_snapshot=$1
|
||||
|
||||
echo "Restoring cluster snapshots."
|
||||
cluster_restore
|
||||
local found=0
|
||||
local scripts_cfg="$TOP_DIR/config/scripts.ubuntu_cluster"
|
||||
|
||||
while read -r line; do
|
||||
if [[ $line =~ ^cmd\ snapshot.*-n\ ([^ ]*)\ (.*) ]]; then
|
||||
# Node name (e.g. controller)
|
||||
node=${BASH_REMATCH[1]}
|
||||
|
||||
# Snapshot name (e.g. keystone_installed)
|
||||
snapshot=${BASH_REMATCH[2]}
|
||||
|
||||
# Global variable name (e.g. CONTROLLER_SNAPSHOT)
|
||||
# Can't use ${node,,} (OS X bash version is only 3.2)
|
||||
var_name=$(echo "$node"|tr "a-z" "A-Z")_SNAPSHOT
|
||||
|
||||
if [ "$snapshot" = "$target_snapshot" ]; then
|
||||
# Can't use associative arrays (OS X bash version is only 3.2)
|
||||
eval "${var_name}=$snapshot"
|
||||
|
||||
found=1
|
||||
elif [ $found -eq 0 ]; then
|
||||
eval "${var_name}=$snapshot"
|
||||
fi
|
||||
fi
|
||||
done < "$scripts_cfg"
|
||||
|
||||
if [ $found -eq 0 ]; then
|
||||
echo "ERROR: snapshot '$target_snapshot' not found"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function restore_current_snapshot {
|
||||
local vm_name=$1
|
||||
vboxmanage snapshot "$vm_name" restorecurrent
|
||||
}
|
||||
|
||||
function restore_named_snapshot {
|
||||
local vm_name=$1
|
||||
local snapshot=$2
|
||||
vboxmanage snapshot "$vm_name" restore "$snapshot"
|
||||
}
|
||||
|
||||
if [ -n "$TARGET_SNAPSHOT" ]; then
|
||||
set_snapshot_vars "$TARGET_SNAPSHOT"
|
||||
fi
|
||||
|
||||
for vm_name in $VM_LIST; do
|
||||
vm_power_off "$vm_name"
|
||||
vm_wait_for_shutdown "$vm_name"
|
||||
|
||||
if [ "${CURRENT:-""}" = "yes" ]; then
|
||||
restore_current_snapshot "$vm_name"
|
||||
if [ "${START:-""}" = "yes" ]; then
|
||||
vbox_boot "$vm_name"
|
||||
fi
|
||||
else
|
||||
# Global variable name (e.g. CONTROLLER_SNAPSHOT)
|
||||
# (use tr due to OS X bash limitation)
|
||||
var_name=$(echo "$vm_name"|tr "a-z" "A-Z")_SNAPSHOT
|
||||
if [ -z "${!var_name:=""}" ]; then
|
||||
vm_delete "$vm_name"
|
||||
else
|
||||
restore_named_snapshot "$vm_name" "${!var_name}"
|
||||
if [ "${START:-""}" = "yes" ]; then
|
||||
vbox_boot "$vm_name"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Starting VMs."
|
||||
cluster_start >/dev/null
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user