From 3cf797d7a59a9f5fc05bd22af32d21c23409e60d Mon Sep 17 00:00:00 2001
From: Neels Hofmeyr <neels@hofmeyr.de>
Date: Thu, 5 Apr 2018 16:56:38 +0200
Subject: add compare-results.sh, call from start-testsuite.sh

Compare current test results to the expected results, and exit in error on
discrepancies.

Add compare-result.sh: (trivially) grep junit xml output to determine which
tests passed and which didn't, and compare against an expected-result.log,
another junit file from a previous run. Summarize and determine success.

Include an "xfail" feature: tests that are expected to fail are marked as
"xfail", unexpected failures as "FAIL".

In various subdirs, copy the current jenkins jobs' junit xml outputs as
expected-results.log, so that we will start getting useful output in both
jenkins runs and manual local runs.

In start-testsuite.sh, after running the tests, invoke the results comparison.

Due to the single-line parsing nature, the script so far does not distinguish
between error and failure. I doubt that we actually need to do that though.

Related: OS#3136
Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d
---
 start-testsuite.sh | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

(limited to 'start-testsuite.sh')

diff --git a/start-testsuite.sh b/start-testsuite.sh
index 8affdbae..5b187fa1 100755
--- a/start-testsuite.sh
+++ b/start-testsuite.sh
@@ -10,7 +10,9 @@ if [ $# -lt 1 ]; then
 fi
 
 SUITE=$1
-CFG=$(basename "$SUITE").cfg
+SUITE_DIR="$(dirname "$SUITE")"
+SUITE_NAME="$(basename "$SUITE")"
+CFG="$SUITE_NAME.cfg"
 if [ $# -gt 1 ]; then
 	CFG=$2
 fi
@@ -19,4 +21,26 @@ if [ $# -gt 2 ]; then
 	TEST=$3
 fi
 
-LD_LIBRARY_PATH=$(dirname "$SUITE"):/usr/lib/titan:/usr/ttcn3/lib ttcn3_start $SUITE $CFG $TEST
+LD_LIBRARY_PATH="$SUITE_DIR:/usr/lib/titan:/usr/ttcn3/lib" ttcn3_start $SUITE $CFG $TEST
+
+expected="$SUITE_DIR/expected-results.log"
+if [ ! -f "$expected" ]; then
+  echo "No expected results found, not comparing outcome. ($expected)"
+  exit 0
+fi
+
+# find the most recent junit output log here
+last_log="$(ls -1tr junit*.log | tail -n 1)"
+if [ ! -f "$last_log" ]; then
+  echo "No junit log found."
+  exit 1
+fi
+
+compare="$SUITE_DIR/../compare-results.sh"
+if [ ! -x "$compare" ]; then
+  echo "ERROR: cannot find $compare"
+  exit 1
+fi
+
+set -e
+"$compare" "$expected" "$last_log" $OSMO_TTCN3_COMPARE_ARGS
-- 
cgit v1.2.3