8a6a88b6b8
Add script launch_instance.sh for basic instance VM tests. The script tries to deal with a number of failures that have turned up in testing (e.g., services failing to start, instance not launching). The changeset includes three scripts in a new tools directory. 1) To run a test once, use test-once.sh: $ ./tools/test-once.sh scripts/test/launch_instance.sh 2) To restore (and boot) the cluster to an earlier snapshot, use restore-cluster.sh. The argument selects the snapshot used for the controller node VM. To select the most recently used snapshot: $ ./tools/restore-cluster.sh current To select the controller snapshot, "controller_node_installed": $ ./tools/restore-cluster.sh controller_node_installed 3) To run the same test repeatedly, use repeat-test.sh. The test script name is hard-coded (launch_instance.sh). The argument determines whether the cluster is rebuilt for each test or if a snapshot of the cluster is restored. The controller snapshot is hardcoded (controller_node_installed); this particular snapshot is of interest because it does not seem to result in a reliable cluster. Log files are stored in log/test-results. Repeat-test.sh also saves log files from each node's /var/log/upstart to help with analyzing failures. $ ./tools/repeat-test.sh restore After running a number of tests, you can get some simple stats using a command like this: $ grep -h SUM log/test-results/*/test.log|LC_ALL=C sort|uniq -c Co-Author: Pranav Salunke <dguitarbite@gmail.com> Change-Id: I20b7273683b281bf7822ef66e311b955b8c5ec8a
71 lines
1.6 KiB
Bash
Executable File
71 lines
1.6 KiB
Bash
Executable File
#!/bin/bash
|
|
set -o errexit -o nounset
|
|
TOP_DIR=$(cd "$(dirname "$0")/.." && pwd)
|
|
source "$TOP_DIR/config/paths"
|
|
source "$CONFIG_DIR/deploy.osbash"
|
|
source "$OSBASH_LIB_DIR/functions.host"
|
|
|
|
LOG_NAME=test.log
|
|
RESULTS_ROOT=$LOG_DIR/test-results
|
|
|
|
CONTROLLER_SNAPSHOT="controller_node_installed"
|
|
TEST_SCRIPT=$TOP_DIR/scripts/test/launch_instance.sh
|
|
|
|
VERBOSE=${VERBOSE:=1}
|
|
|
|
function usage {
|
|
echo "Usage: $0 {rebuild|restore}"
|
|
echo " rebuild: rebuild cluster for each test (osbash.sh -b cluster)"
|
|
echo " restore: restore cluster for each test (cluster-restore.sh)"
|
|
exit 1
|
|
}
|
|
|
|
if [ $# = 0 ]; then
|
|
usage
|
|
elif [ "$1" = "rebuild" ]; then
|
|
INIT=rebuild
|
|
elif [ "$1" = "restore" ]; then
|
|
unset INIT
|
|
else
|
|
usage
|
|
fi
|
|
|
|
mkdir -p "$RESULTS_ROOT"
|
|
|
|
while [ : ]; do
|
|
dir_name=$(get_next_prefix "$RESULTS_ROOT" "")
|
|
echo "Starting test $dir_name."
|
|
dir=$RESULTS_ROOT/$dir_name
|
|
mkdir -p "$dir"
|
|
|
|
(
|
|
cd "$TOP_DIR"
|
|
|
|
if [ "${INIT:=""}" = "rebuild" ]; then
|
|
echo "Building cluster."
|
|
"$TOP_DIR/osbash.sh" -b cluster
|
|
else
|
|
echo "Restoring cluster."
|
|
"$TOP_DIR/tools/restore-cluster.sh" "$CONTROLLER_SNAPSHOT"
|
|
fi
|
|
|
|
echo "Running test. Log file: $dir/$LOG_NAME"
|
|
rc=0
|
|
TEST_ONCE=$TOP_DIR/tools/test-once.sh
|
|
if [ "$VERBOSE" -eq 1 ]; then
|
|
"$TEST_ONCE" "$TEST_SCRIPT" 2>&1 | tee "$dir/$LOG_NAME" || rc=$?
|
|
else
|
|
"$TEST_ONCE" "$TEST_SCRIPT" > "$dir/$LOG_NAME" 2>&1 || rc=$?
|
|
fi
|
|
|
|
if [ $rc -eq 0 ]; then
|
|
echo "Test done."
|
|
else
|
|
echo "Failed to run test. Aborting."
|
|
exit 1
|
|
fi
|
|
)
|
|
|
|
"$TOP_DIR/tools/get_upstart_logs.sh" "$dir"
|
|
done
|