diff --git a/modules/jenkins/files/slave_scripts/run-tox.sh b/modules/jenkins/files/slave_scripts/run-tox.sh
index 7337fb3a6a..f969686edf 100755
--- a/modules/jenkins/files/slave_scripts/run-tox.sh
+++ b/modules/jenkins/files/slave_scripts/run-tox.sh
@@ -90,4 +90,21 @@ else
     echo
 fi
 
+if [ -d ".testrepository" ]
+then
+    source .tox/$venv/bin/activate
+    foundcount=$(testr list-tests | sed -e '1d' | wc -l)
+    rancount=$(testr last | sed -ne 's/Ran \([0-9]\+\).*tests in.*/\1/p')
+    if [ "$foundcount" -ne "$rancount" ]
+    then
+        echo
+        echo "The number of tests found did not match the number of tests"
+        echo "that were run. This indicates a fatal error occured while"
+        echo "running the tests."
+        echo "Tests found: $foundcount Tests ran: $rancount"
+        echo
+        exit 1
+    fi
+fi
+
 exit $result