From 3cf797d7a59a9f5fc05bd22af32d21c23409e60d Mon Sep 17 00:00:00 2001 From: Neels Hofmeyr Date: Thu, 5 Apr 2018 16:56:38 +0200 Subject: add compare-results.sh, call from start-testsuite.sh Compare current test results to the expected results, and exit in error on discrepancies. Add compare-result.sh: (trivially) grep junit xml output to determine which tests passed and which didn't, and compare against an expected-result.log, another junit file from a previous run. Summarize and determine success. Include an "xfail" feature: tests that are expected to fail are marked as "xfail", unexpected failures as "FAIL". In various subdirs, copy the current jenkins jobs' junit xml outputs as expected-results.log, so that we will start getting useful output in both jenkins runs and manual local runs. In start-testsuite.sh, after running the tests, invoke the results comparison. Due to the single-line parsing nature, the script so far does not distinguish between error and failure. I doubt that we actually need to do that though. Related: OS#3136 Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d --- .gitignore | 1 + bsc/expected-results.log | 95 +++++++++++++++++++ compare-results.sh | 198 ++++++++++++++++++++++++++++++++++++++++ ggsn_tests/expected-results.log | 14 +++ hlr/expected-results.log | 23 +++++ mgw/expected-results.log | 36 ++++++++ msc/expected-results.log | 110 ++++++++++++++++++++++ sgsn/expected-results.log | 54 +++++++++++ sip/expected-results.log | 7 ++ start-testsuite.sh | 28 +++++- sysinfo/expected-results.log | 69 ++++++++++++++ 11 files changed, 633 insertions(+), 2 deletions(-) create mode 100644 bsc/expected-results.log create mode 100755 compare-results.sh create mode 100644 ggsn_tests/expected-results.log create mode 100644 hlr/expected-results.log create mode 100644 mgw/expected-results.log create mode 100644 msc/expected-results.log create mode 100644 sgsn/expected-results.log create mode 100644 sip/expected-results.log create mode 100644 sysinfo/expected-results.log diff --git a/.gitignore b/.gitignore index 8282f5ff..519dc488 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ deps/*/ *.o *.log +!expected-results.log *.so compile */.gitignore diff --git a/bsc/expected-results.log b/bsc/expected-results.log new file mode 100644 index 00000000..18a9a20e --- /dev/null +++ b/bsc/expected-results.log @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Timeout expecting { msg_disc := { msg_group := RSL_MDISC_CCHAN (6), transparent := false }, msg_type := RSL_MT_PAGING_CMD (21), ies := { { iei := ?, body := { chan_nr := { u := { ch0 := RSL_CHAN_NR_PCH_AGCH (18) }, tn := ? } } }, { iei := ?, body := { paging_group := ? } }, { iei := ?, body := { ms_identity := { len := ?, payload := ? } } }, * } } + BSC_Tests.ttcn:2203 BSC_Tests control part + BSC_Tests.ttcn:1131 TC_paging_imsi_nochan_lai testcase + + + + + + + + + + + + + + Received PAGING after A-RESET + BSC_Tests.ttcn:2212 BSC_Tests control part + BSC_Tests.ttcn:1306 TC_paging_imsi_a_reset testcase + + + + + + + + + + + + + + + + + Timeout of T_guard + BSC_Tests.ttcn:2230 BSC_Tests control part + BSC_Tests.ttcn:2056 TC_bssmap_clear_does_not_cause_bssmap_reset testcase + + + + Timeout of T_guard + BSC_Tests.ttcn:2231 BSC_Tests control part + BSC_Tests.ttcn:2115 TC_ms_rel_ind_does_not_cause_bssmap_reset testcase + + + diff --git a/compare-results.sh b/compare-results.sh new file mode 100755 index 00000000..cb80a0dd --- /dev/null +++ b/compare-results.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +expected_file="$1" +results_file="$2" + +usage() { + echo " +Usage: + + $(basename "$0") expected_results.junit-log current_results.junit-log [--allow-* [...]] + +Return 0 if the expected results match the current results exactly. + + --allow-skip Allow runnning less tests than are listed in the expected file. + Default is to return failure on any skipped tests. + --allow-new Allow more test results than found in the expected file. + Default is to return failure on any unknown tests. + --allow-xpass If a test was expected to fail but passed, return success. + Default is to return failure on any mismatch. +" +} + +if [ ! -f "$expected_file" ]; then + usage + echo "Expected file not found: '$expected_file'" + exit 1 +fi + +if [ ! -f "$results_file" ]; then + usage + echo "Current results file not found: '$results_file'" + exit 1 +fi + +shift +shift + +allow_xpass=0 +allow_skip=0 +allow_new=0 + +while test -n "$1"; do + arg="$1" + if [ "x$arg" = "x--allow-xpass" ]; then + allow_xpass=1 + elif [ "x$arg" = "x--allow-skip" ]; then + allow_skip=1 + elif [ "x$arg" = "x--allow-new" ]; then + allow_new=1 + else + usage + echo "Unknown argument: '$arg'" + exit 1 + fi + shift +done + +echo "Comparing expected results $expected_file against results in $results_file +--------------------" + +parse_testcase() { + line="$1" + suite_name="$(echo "$line" | sed 's,.*classname='"'"'\([^'"'"']*\)'"'"'.*,\1,')" + test_name="$(echo "$line" | sed 's,.*\$')" ]; then + test_result="pass" + else + test_result="FAIL" + fi +} + +pass=0 +xfail=0 +more_failures=0 +more_successes=0 +skipped=0 +new=0 + +while read line; do + parse_testcase "$line" + exp_suite_name="$suite_name" + exp_test_name="$test_name" + exp_test_result="$test_result" + matched="0" + + while read line; do + parse_testcase "$line" + if [ "x$exp_suite_name" != "x$suite_name" ]; then + continue + fi + if [ "x$exp_test_name" != "x$test_name" ]; then + continue + fi + + if [ "x$exp_test_result" = "x$test_result" ]; then + if [ "x$exp_test_result" = "xFAIL" ]; then + exp_test_result="xfail" + (( xfail += 1 )) + else + (( pass += 1 )) + fi + echo "$exp_test_result $suite_name.$test_name" + else + if [ "x$exp_test_result" = "xFAIL" ]; then + exp_test_result="xfail" + fi + echo "$exp_test_result->$test_result $suite_name.$test_name" + if [ "x$test_result" = "xFAIL" ]; then + (( more_failures += 1 )) + else + (( more_successes += 1 )) + fi + fi + matched="1" + break + done <<< "$(grep " + + + + + + + + + + + + + diff --git a/hlr/expected-results.log b/hlr/expected-results.log new file mode 100644 index 00000000..be8000c0 --- /dev/null +++ b/hlr/expected-results.log @@ -0,0 +1,23 @@ + + + + Unexpected SAI ERROR Cause + HLR_Tests.ttcn:558 HLR_Tests control part + HLR_Tests.ttcn:364 TC_gsup_sai_err_invalid_imsi testcase + + + + + + + + + Timeout waiting for ISD.req + HLR_Tests.ttcn:564 HLR_Tests control part + HLR_Tests.ttcn:485 TC_vty_msisdn_isd testcase + + + + + + diff --git a/mgw/expected-results.log b/mgw/expected-results.log new file mode 100644 index 00000000..c67d5d91 --- /dev/null +++ b/mgw/expected-results.log @@ -0,0 +1,36 @@ + + + + no verdict + + + + + + + + + + + + MGCP_Test.ttcn:887 MGCP_Test control part + MGCP_Test.ttcn:436 TC_crcx_illegal_double_lco testcase + + + + + + + + + + + + + + + + + + + diff --git a/msc/expected-results.log b/msc/expected-results.log new file mode 100644 index 00000000..3945a5ff --- /dev/null +++ b/msc/expected-results.log @@ -0,0 +1,110 @@ + + + + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1829 MSC_Tests control part + MSC_Tests.ttcn:502 TC_lu_imsi_reject testcase + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1830 MSC_Tests control part + MSC_Tests.ttcn:531 TC_lu_imsi_timeout_gsup testcase + + + + + + + + + + + + + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1843 MSC_Tests control part + MSC_Tests.ttcn:903 TC_emerg_call_imei_reject testcase + + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1845 MSC_Tests control part + MSC_Tests.ttcn:940 TC_cm_serv_req_vgcs_reject testcase + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1846 MSC_Tests control part + MSC_Tests.ttcn:961 TC_cm_serv_req_vbs_reject testcase + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1847 MSC_Tests control part + MSC_Tests.ttcn:982 TC_cm_serv_req_lcs_reject testcase + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1848 MSC_Tests control part + MSC_Tests.ttcn:1003 TC_cm_reest_req_reject testcase + + + + + + + Timeout waiting for ClearCommand or SCCP Release + MSC_Tests.ttcn:1852 MSC_Tests control part + MSC_Tests.ttcn:1125 TC_cl3_rnd_payload testcase + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1853 MSC_Tests control part + MSC_Tests.ttcn:1142 TC_establish_and_nothing testcase + + + + Timeout waiting for ClearCommand/Release + MSC_Tests.ttcn:1854 MSC_Tests control part + MSC_Tests.ttcn:1166 TC_mo_setup_and_nothing testcase + + + + + Timeout waiting for channel release + MSC_Tests.ttcn:1856 MSC_Tests control part + MSC_Tests.ttcn:1283 TC_mo_crcx_ran_reject testcase + + + + Timeout waiting for channel release + MSC_Tests.ttcn:1857 MSC_Tests control part + MSC_Tests.ttcn:1380 TC_mt_crcx_ran_reject testcase + + + + + + + + + + + + + + + + + + diff --git a/sgsn/expected-results.log b/sgsn/expected-results.log new file mode 100644 index 00000000..502dde8d --- /dev/null +++ b/sgsn/expected-results.log @@ -0,0 +1,54 @@ + + + + + + + Tguard timeout + SGSN_Tests.ttcn:1157 SGSN_Tests control part + SGSN_Tests.ttcn:480 TC_attach_auth_sai_timeout testcase + + + + Tguard timeout + SGSN_Tests.ttcn:1158 SGSN_Tests control part + SGSN_Tests.ttcn:501 TC_attach_auth_sai_reject testcase + + + + Tguard timeout + SGSN_Tests.ttcn:1159 SGSN_Tests control part + SGSN_Tests.ttcn:529 TC_attach_gsup_lu_timeout testcase + + + + Tguard timeout + SGSN_Tests.ttcn:1160 SGSN_Tests control part + SGSN_Tests.ttcn:558 TC_attach_gsup_lu_reject testcase + + + + + + + + + Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 77, which has already terminated. + + + Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 83, which has already terminated. + + + + + + + + + + Tguard timeout + SGSN_Tests.ttcn:1175 SGSN_Tests control part + SGSN_Tests.ttcn:1148 TC_attach_pdp_act_user_deact_mt testcase + + + diff --git a/sip/expected-results.log b/sip/expected-results.log new file mode 100644 index 00000000..54123a55 --- /dev/null +++ b/sip/expected-results.log @@ -0,0 +1,7 @@ + + + + + + + diff --git a/start-testsuite.sh b/start-testsuite.sh index 8affdbae..5b187fa1 100755 --- a/start-testsuite.sh +++ b/start-testsuite.sh @@ -10,7 +10,9 @@ if [ $# -lt 1 ]; then fi SUITE=$1 -CFG=$(basename "$SUITE").cfg +SUITE_DIR="$(dirname "$SUITE")" +SUITE_NAME="$(basename "$SUITE")" +CFG="$SUITE_NAME.cfg" if [ $# -gt 1 ]; then CFG=$2 fi @@ -19,4 +21,26 @@ if [ $# -gt 2 ]; then TEST=$3 fi -LD_LIBRARY_PATH=$(dirname "$SUITE"):/usr/lib/titan:/usr/ttcn3/lib ttcn3_start $SUITE $CFG $TEST +LD_LIBRARY_PATH="$SUITE_DIR:/usr/lib/titan:/usr/ttcn3/lib" ttcn3_start $SUITE $CFG $TEST + +expected="$SUITE_DIR/expected-results.log" +if [ ! -f "$expected" ]; then + echo "No expected results found, not comparing outcome. ($expected)" + exit 0 +fi + +# find the most recent junit output log here +last_log="$(ls -1tr junit*.log | tail -n 1)" +if [ ! -f "$last_log" ]; then + echo "No junit log found." + exit 1 +fi + +compare="$SUITE_DIR/../compare-results.sh" +if [ ! -x "$compare" ]; then + echo "ERROR: cannot find $compare" + exit 1 +fi + +set -e +"$compare" "$expected" "$last_log" $OSMO_TTCN3_COMPARE_ARGS diff --git a/sysinfo/expected-results.log b/sysinfo/expected-results.log new file mode 100644 index 00000000..cea6f54b --- /dev/null +++ b/sysinfo/expected-results.log @@ -0,0 +1,69 @@ + + + + + + No SI messages for TC=0! + + + + + No SI messages for TC=0! + + + + + No SI messages for TC=0! + + + + + Test.ttcn:162: Dynamic test case error: testcase.stop + + + + Test.ttcn:162: Dynamic test case error: testcase.stop + + + Test.ttcn:162: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + + Test.ttcn:478: Dynamic test case error: testcase.stop + + -- cgit v1.2.3