summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.gitignore1
-rw-r--r--bsc/expected-results.log95
-rwxr-xr-xcompare-results.sh198
-rw-r--r--ggsn_tests/expected-results.log14
-rw-r--r--hlr/expected-results.log23
-rw-r--r--mgw/expected-results.log36
-rw-r--r--msc/expected-results.log110
-rw-r--r--sgsn/expected-results.log54
-rw-r--r--sip/expected-results.log7
-rwxr-xr-xstart-testsuite.sh28
-rw-r--r--sysinfo/expected-results.log69
11 files changed, 633 insertions, 2 deletions
diff --git a/.gitignore b/.gitignore
index 8282f5f..519dc48 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
deps/*/
*.o
*.log
+!expected-results.log
*.so
compile
*/.gitignore
diff --git a/bsc/expected-results.log b/bsc/expected-results.log
new file mode 100644
index 0000000..18a9a20
--- /dev/null
+++ b/bsc/expected-results.log
@@ -0,0 +1,95 @@
+<?xml version="1.0"?>
+<testsuite name='BSC_Tests' tests='68' failures='4' errors='2' skipped='0' inconc='0' time='932.00'>
+ <testcase classname='BSC_Tests' name='TC_ctrl_msc_connection_status' time='2.267915'/>
+ <testcase classname='BSC_Tests' name='TC_ctrl_msc0_connection_status' time='2.225696'/>
+ <testcase classname='BSC_Tests' name='TC_ctrl' time='4.242800'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_noreply' time='2.229171'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_counter' time='2.226685'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_ack_noest' time='5.225675'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_ack_est_ind_noreply' time='24.248343'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_ack_est_ind_refused' time='4.231838'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_nack' time='2.728319'/>
+ <testcase classname='BSC_Tests' name='TC_chan_exhaustion' time='2.239509'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_rll_rel_ind' time='8.276291'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_conn_fail' time='8.290978'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_hard_clear' time='7.431206'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_hard_rlsd' time='4.451855'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_a_reset' time='4.437509'/>
+ <testcase classname='BSC_Tests' name='TC_outbound_connect' time='2.230366'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_cic_only' time='2.432230'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_csd' time='2.435962'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_ctm' time='2.431946'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_sign' time='3.139859'>
+ <error type='DTE'></error>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_0' time='5.362074'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_1' time='5.360999'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_1_codec_missing' time='3.371507'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_3' time='5.373823'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_4' time='5.385205'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_fr' time='5.370247'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_hr' time='5.370154'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_efr' time='5.373357'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_amr_f' time='5.369318'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_amr_h' time='5.362888'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inact_lchan' time='2.224803'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sapi1' time='4.226237'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sapi3' time='4.231547'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sacch' time='4.246674'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan' time='8.753319'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_nochan' time='8.759194'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_any' time='8.750134'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_sdcch' time='8.754159'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_tch_f' time='8.758687'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_tch_hf' time='9.289936'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_cgi' time='8.956272'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac_ci' time='8.956649'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_ci' time='8.969455'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lai' time='8.763181'>
+ <failure type='fail-verdict'>Timeout expecting { msg_disc := { msg_group := RSL_MDISC_CCHAN (6), transparent := false }, msg_type := RSL_MT_PAGING_CMD (21), ies := { { iei := ?, body := { chan_nr := { u := { ch0 := RSL_CHAN_NR_PCH_AGCH (18) }, tn := ? } } }, { iei := ?, body := { paging_group := ? } }, { iei := ?, body := { ms_identity := { len := ?, payload := ? } } }, * } }
+ BSC_Tests.ttcn:2203 BSC_Tests control part
+ BSC_Tests.ttcn:1131 TC_paging_imsi_nochan_lai testcase
+ </failure>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac' time='8.752368'>
+ <error type='DTE'></error>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_all' time='8.751176'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_plmn_lac_rnc' time='8.563604'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_rnc' time='8.551772'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac_rnc' time='8.568040'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lacs' time='8.762731'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lacs_empty' time='8.548303'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_cgi_unknown_cid' time='8.557896'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_a_reset' time='8.759365'>
+ <failure type='fail-verdict'>Received PAGING after A-RESET
+ BSC_Tests.ttcn:2212 BSC_Tests control part
+ BSC_Tests.ttcn:1306 TC_paging_imsi_a_reset testcase
+ </failure>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_load' time='12.961436'/>
+ <testcase classname='BSC_Tests' name='TC_paging_counter' time='20.788631'/>
+ <testcase classname='BSC_Tests' name='TC_rsl_drop_counter' time='2.227479'/>
+ <testcase classname='BSC_Tests' name='TC_rsl_unknown_unit_id' time='0.010705'/>
+ <testcase classname='BSC_Tests' name='TC_oml_unknown_unit_id' time='0.010646'/>
+ <testcase classname='BSC_Tests' name='TC_classmark' time='3.185732'/>
+ <testcase classname='BSC_Tests' name='TC_unsol_ass_fail' time='8.149850'/>
+ <testcase classname='BSC_Tests' name='TC_unsol_ass_compl' time='8.155615'/>
+ <testcase classname='BSC_Tests' name='TC_unsol_ho_fail' time='8.147592'/>
+ <testcase classname='BSC_Tests' name='TC_err_82_short_msg' time='8.161174'/>
+ <testcase classname='BSC_Tests' name='TC_err_84_unknown_msg' time='3.174456'/>
+ <testcase classname='BSC_Tests' name='TC_ho_int' time='7.423814'/>
+ <testcase classname='BSC_Tests' name='TC_bssap_rlsd_does_not_cause_bssmap_reset' time='14.310727'/>
+ <testcase classname='BSC_Tests' name='TC_bssmap_clear_does_not_cause_bssmap_reset' time='45.459198'>
+ <failure type='fail-verdict'>Timeout of T_guard
+ BSC_Tests.ttcn:2230 BSC_Tests control part
+ BSC_Tests.ttcn:2056 TC_bssmap_clear_does_not_cause_bssmap_reset testcase
+ </failure>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_ms_rel_ind_does_not_cause_bssmap_reset' time='45.320202'>
+ <failure type='fail-verdict'>Timeout of T_guard
+ BSC_Tests.ttcn:2231 BSC_Tests control part
+ BSC_Tests.ttcn:2115 TC_ms_rel_ind_does_not_cause_bssmap_reset testcase
+ </failure>
+ </testcase>
+</testsuite>
diff --git a/compare-results.sh b/compare-results.sh
new file mode 100755
index 0000000..cb80a0d
--- /dev/null
+++ b/compare-results.sh
@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+expected_file="$1"
+results_file="$2"
+
+usage() {
+ echo "
+Usage:
+
+ $(basename "$0") expected_results.junit-log current_results.junit-log [--allow-* [...]]
+
+Return 0 if the expected results match the current results exactly.
+
+ --allow-skip Allow runnning less tests than are listed in the expected file.
+ Default is to return failure on any skipped tests.
+ --allow-new Allow more test results than found in the expected file.
+ Default is to return failure on any unknown tests.
+ --allow-xpass If a test was expected to fail but passed, return success.
+ Default is to return failure on any mismatch.
+"
+}
+
+if [ ! -f "$expected_file" ]; then
+ usage
+ echo "Expected file not found: '$expected_file'"
+ exit 1
+fi
+
+if [ ! -f "$results_file" ]; then
+ usage
+ echo "Current results file not found: '$results_file'"
+ exit 1
+fi
+
+shift
+shift
+
+allow_xpass=0
+allow_skip=0
+allow_new=0
+
+while test -n "$1"; do
+ arg="$1"
+ if [ "x$arg" = "x--allow-xpass" ]; then
+ allow_xpass=1
+ elif [ "x$arg" = "x--allow-skip" ]; then
+ allow_skip=1
+ elif [ "x$arg" = "x--allow-new" ]; then
+ allow_new=1
+ else
+ usage
+ echo "Unknown argument: '$arg'"
+ exit 1
+ fi
+ shift
+done
+
+echo "Comparing expected results $expected_file against results in $results_file
+--------------------"
+
+parse_testcase() {
+ line="$1"
+ suite_name="$(echo "$line" | sed 's,.*classname='"'"'\([^'"'"']*\)'"'"'.*,\1,')"
+ test_name="$(echo "$line" | sed 's,.*\<name='"'"'\([^'"'"']*\)'"'"'.*,\1,')"
+ if [ -n "$(echo "$line" | grep '/>$')" ]; then
+ test_result="pass"
+ else
+ test_result="FAIL"
+ fi
+}
+
+pass=0
+xfail=0
+more_failures=0
+more_successes=0
+skipped=0
+new=0
+
+while read line; do
+ parse_testcase "$line"
+ exp_suite_name="$suite_name"
+ exp_test_name="$test_name"
+ exp_test_result="$test_result"
+ matched="0"
+
+ while read line; do
+ parse_testcase "$line"
+ if [ "x$exp_suite_name" != "x$suite_name" ]; then
+ continue
+ fi
+ if [ "x$exp_test_name" != "x$test_name" ]; then
+ continue
+ fi
+
+ if [ "x$exp_test_result" = "x$test_result" ]; then
+ if [ "x$exp_test_result" = "xFAIL" ]; then
+ exp_test_result="xfail"
+ (( xfail += 1 ))
+ else
+ (( pass += 1 ))
+ fi
+ echo "$exp_test_result $suite_name.$test_name"
+ else
+ if [ "x$exp_test_result" = "xFAIL" ]; then
+ exp_test_result="xfail"
+ fi
+ echo "$exp_test_result->$test_result $suite_name.$test_name"
+ if [ "x$test_result" = "xFAIL" ]; then
+ (( more_failures += 1 ))
+ else
+ (( more_successes += 1 ))
+ fi
+ fi
+ matched="1"
+ break
+ done <<< "$(grep "<testcase" "$results_file")"
+
+ if [ "x$matched" = "x0" ]; then
+ echo "skipped $exp_suite_name.$exp_test_name"
+ (( skipped += 1 ))
+ fi
+
+done <<< "$(grep "<testcase" "$expected_file")"
+
+# Also catch all new tests that aren't covered in the expected results
+while read line; do
+ parse_testcase "$line"
+ got_suite_name="$suite_name"
+ got_test_name="$test_name"
+ got_test_result="$test_result"
+ matched="0"
+
+ while read line; do
+ parse_testcase "$line"
+ if [ "x$got_suite_name" != "x$suite_name" ]; then
+ continue
+ fi
+ if [ "x$got_test_name" != "x$test_name" ]; then
+ continue
+ fi
+
+ matched="1"
+ break
+ done <<< "$(grep "<testcase" "$expected_file")"
+
+ if [ "x$matched" = "x0" ]; then
+ echo "NEW-$got_test_result $got_suite_name.$got_test_name"
+ (( new += 1 ))
+ fi
+
+done <<< "$(grep "<testcase" "$results_file")"
+
+echo "--------------------"
+overall_verdict=0
+
+if [ "x$pass" != x0 ]; then
+ echo "$pass pass"
+fi
+
+if [ "x$xfail" != x0 ]; then
+ echo "$xfail xfail"
+fi
+
+if [ "x$skipped" != x0 ]; then
+ echo "$skipped skipped"
+ if [ "x$allow_skip" = x0 ]; then
+ overall_verdict=4
+ echo " Failing due to skipped tests. If tests were removed, update the expected results!"
+ fi
+fi
+
+if [ "x$new" != x0 ]; then
+ echo "$new new"
+ if [ "x$allow_new" = x0 ]; then
+ overall_verdict=3
+ echo " Failing due to new tests. Update the expected results!"
+ fi
+fi
+
+if [ "x$more_successes" != x0 ]; then
+ echo "$more_successes pass unexpectedly"
+ if [ "x$allow_xpass" = x0 ]; then
+ overall_verdict=2
+ echo " Update the expected results!"
+ fi
+fi
+
+if [ "x$more_failures" != x0 ]; then
+ echo "$more_failures FAIL"
+ overall_verdict=1
+fi
+
+echo
+if [ "x$overall_verdict" != x0 ]; then
+ echo "FAILURE"
+else
+ echo "ok"
+fi
+exit $overall_verdict
diff --git a/ggsn_tests/expected-results.log b/ggsn_tests/expected-results.log
new file mode 100644
index 0000000..bff953a
--- /dev/null
+++ b/ggsn_tests/expected-results.log
@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+<testsuite name='GGSN_Tests' tests='11' failures='0' errors='0' skipped='0' inconc='0' time='205.00'>
+ <testcase classname='GGSN_Tests' name='TC_pdp4_act_deact' time='0.006364'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp4_act_deact_ipcp' time='0.003752'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp4_act_deact_pcodns' time='0.003510'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp4_act_deact_gtpu_access' time='6.011141'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp4_clients_interact' time='0.006517'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp6_act_deact' time='0.003312'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp6_act_deact_pcodns' time='0.003403'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp6_act_deact_icmp6' time='0.005114'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp6_act_deact_gtpu_access' time='9.018065'/>
+ <testcase classname='GGSN_Tests' name='TC_pdp6_clients_interact' time='6.014695'/>
+ <testcase classname='GGSN_Tests' name='TC_echo_req_resp' time='0.001674'/>
+</testsuite>
diff --git a/hlr/expected-results.log b/hlr/expected-results.log
new file mode 100644
index 0000000..be8000c
--- /dev/null
+++ b/hlr/expected-results.log
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='10' failures='2' errors='0' skipped='0' inconc='0' time='118.00'>
+ <testcase classname='HLR_Tests' name='TC_gsup_sai_err_invalid_imsi' time='0.008962'>
+ <failure type='fail-verdict'>Unexpected SAI ERROR Cause
+ HLR_Tests.ttcn:558 HLR_Tests control part
+ HLR_Tests.ttcn:364 TC_gsup_sai_err_invalid_imsi testcase
+ </failure>
+ </testcase>
+ <testcase classname='HLR_Tests' name='TC_gsup_sai' time='0.431623'/>
+ <testcase classname='HLR_Tests' name='TC_gsup_ul_unknown_imsi' time='0.009124'/>
+ <testcase classname='HLR_Tests' name='TC_gsup_sai_err_unknown_imsi' time='0.008056'/>
+ <testcase classname='HLR_Tests' name='TC_gsup_ul' time='0.402349'/>
+ <testcase classname='HLR_Tests' name='TC_vty' time='0.151754'/>
+ <testcase classname='HLR_Tests' name='TC_vty_msisdn_isd' time='5.076033'>
+ <failure type='fail-verdict'>Timeout waiting for ISD.req
+ HLR_Tests.ttcn:564 HLR_Tests control part
+ HLR_Tests.ttcn:485 TC_vty_msisdn_isd testcase
+ </failure>
+ </testcase>
+ <testcase classname='HLR_Tests' name='TC_gsup_purge_cs' time='0.459958'/>
+ <testcase classname='HLR_Tests' name='TC_gsup_purge_ps' time='0.467724'/>
+ <testcase classname='HLR_Tests' name='TC_gsup_purge_unknown' time='0.008034'/>
+</testsuite>
diff --git a/mgw/expected-results.log b/mgw/expected-results.log
new file mode 100644
index 0000000..c67d5d9
--- /dev/null
+++ b/mgw/expected-results.log
@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='26' failures='1' errors='0' skipped='1' inconc='0' time='177.00'>
+ <testcase classname='MGCP_Test' name='TC_selftest' time='0.002990'>
+ <skipped>no verdict</skipped>
+ </testcase>
+ <testcase classname='MGCP_Test' name='TC_crcx' time='0.008097'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_noprefix' time='0.007103'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_unsupp_mode' time='0.006485'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_early_bidir_mode' time='0.023678'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_unsupp_param' time='0.005827'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_missing_callid' time='0.006864'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_missing_mode' time='0.006328'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_unsupp_packet_intv' time='0.005716'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_illegal_double_lco' time='0.014705'>
+ <failure type='fail-verdict'>
+ MGCP_Test.ttcn:887 MGCP_Test control part
+ MGCP_Test.ttcn:436 TC_crcx_illegal_double_lco testcase
+ </failure>
+ </testcase>
+ <testcase classname='MGCP_Test' name='TC_crcx_sdp' time='0.008481'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_wildcarded' time='0.006528'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_wildcarded_exhaust' time='0.033504'/>
+ <testcase classname='MGCP_Test' name='TC_mdcx_without_crcx' time='0.023214'/>
+ <testcase classname='MGCP_Test' name='TC_dlcx_without_crcx' time='0.006506'/>
+ <testcase classname='MGCP_Test' name='TC_mdcx_wildcarded' time='0.006675'/>
+ <testcase classname='MGCP_Test' name='TC_dlcx_wildcarded' time='0.022936'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid_connid' time='0.006310'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid' time='0.006509'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep' time='0.006822'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid_inval' time='0.007519'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_ep_callid_connid_inval' time='0.006912'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_and_dlcx_retrans' time='0.006681'/>
+ <testcase classname='MGCP_Test' name='TC_crcx_dlcx_30ep' time='0.032986'/>
+ <testcase classname='MGCP_Test' name='TC_rtpem_selftest' time='5.514538'/>
+ <testcase classname='MGCP_Test' name='TC_two_crcx_and_rtp' time='1.111828'/>
+</testsuite>
diff --git a/msc/expected-results.log b/msc/expected-results.log
new file mode 100644
index 0000000..3945a5f
--- /dev/null
+++ b/msc/expected-results.log
@@ -0,0 +1,110 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='45' failures='12' errors='1' skipped='0' inconc='0' time='497.00'>
+ <testcase classname='MSC_Tests' name='TC_cr_before_reset' time='4.009127'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_noauth_tmsi' time='1.035697'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_noauth_notmsi' time='1.045502'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_reject' time='6.041282'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1829 MSC_Tests control part
+ MSC_Tests.ttcn:502 TC_lu_imsi_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_timeout_gsup' time='11.044559'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1830 MSC_Tests control part
+ MSC_Tests.ttcn:531 TC_lu_imsi_timeout_gsup testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi' time='1.036704'/>
+ <testcase classname='MSC_Tests' name='TC_cmserv_imsi_unknown' time='1.021692'/>
+ <testcase classname='MSC_Tests' name='TC_lu_and_mo_call' time='4.291881'/>
+ <testcase classname='MSC_Tests' name='TC_lu_auth_sai_timeout' time='6.032964'/>
+ <testcase classname='MSC_Tests' name='TC_lu_auth_sai_err' time='1.028503'/>
+ <testcase classname='MSC_Tests' name='TC_lu_clear_request' time='2.029726'/>
+ <testcase classname='MSC_Tests' name='TC_lu_disconnect' time='2.028908'/>
+ <testcase classname='MSC_Tests' name='TC_lu_by_imei' time='6.030599'/>
+ <testcase classname='MSC_Tests' name='TC_lu_by_tmsi_noauth_unknown' time='1.246818'/>
+ <testcase classname='MSC_Tests' name='TC_imsi_detach_by_imsi' time='1.238427'/>
+ <testcase classname='MSC_Tests' name='TC_imsi_detach_by_tmsi' time='1.228194'/>
+ <testcase classname='MSC_Tests' name='TC_imsi_detach_by_imei' time='1.233577'/>
+ <testcase classname='MSC_Tests' name='TC_emerg_call_imei_reject' time='6.025962'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1843 MSC_Tests control part
+ MSC_Tests.ttcn:903 TC_emerg_call_imei_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_emerg_call_imsi' time='4.295068'/>
+ <testcase classname='MSC_Tests' name='TC_cm_serv_req_vgcs_reject' time='6.251082'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1845 MSC_Tests control part
+ MSC_Tests.ttcn:940 TC_cm_serv_req_vgcs_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_cm_serv_req_vbs_reject' time='6.249901'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1846 MSC_Tests control part
+ MSC_Tests.ttcn:961 TC_cm_serv_req_vbs_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_cm_serv_req_lcs_reject' time='6.244740'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1847 MSC_Tests control part
+ MSC_Tests.ttcn:982 TC_cm_serv_req_lcs_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_cm_reest_req_reject' time='6.250134'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1848 MSC_Tests control part
+ MSC_Tests.ttcn:1003 TC_cm_reest_req_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_auth_2G_fail' time='1.042594'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_13_13' time='1.052763'/>
+ <testcase classname='MSC_Tests' name='TC_cl3_no_payload' time='1.021188'/>
+ <testcase classname='MSC_Tests' name='TC_cl3_rnd_payload' time='5.996049'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand or SCCP Release
+ MSC_Tests.ttcn:1852 MSC_Tests control part
+ MSC_Tests.ttcn:1125 TC_cl3_rnd_payload testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_establish_and_nothing' time='6.263255'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1853 MSC_Tests control part
+ MSC_Tests.ttcn:1142 TC_establish_and_nothing testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mo_setup_and_nothing' time='31.481429'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1854 MSC_Tests control part
+ MSC_Tests.ttcn:1166 TC_mo_setup_and_nothing testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mo_crcx_ran_timeout' time='5.267724'/>
+ <testcase classname='MSC_Tests' name='TC_mo_crcx_ran_reject' time='31.279836'>
+ <failure type='fail-verdict'>Timeout waiting for channel release
+ MSC_Tests.ttcn:1856 MSC_Tests control part
+ MSC_Tests.ttcn:1283 TC_mo_crcx_ran_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mt_crcx_ran_reject' time='31.062826'>
+ <failure type='fail-verdict'>Timeout waiting for channel release
+ MSC_Tests.ttcn:1857 MSC_Tests control part
+ MSC_Tests.ttcn:1380 TC_mt_crcx_ran_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mo_setup_and_dtmf_dup' time='2.468132'/>
+ <testcase classname='MSC_Tests' name='TC_gsup_cancel' time='1.259044'>
+ <error type='DTE'></error>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_1_13' time='1.049565'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_13' time='1.054199'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1' time='1.034815'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1_no_cm' time='1.038335'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_13_2' time='1.042416'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_013_2' time='1.050732'/>
+ <testcase classname='MSC_Tests' name='TC_mo_release_timeout' time='25.264303'/>
+ <testcase classname='MSC_Tests' name='TC_lu_and_mt_call_no_dlcx_resp' time='10.273413'/>
+ <testcase classname='MSC_Tests' name='TC_reset_two' time='2.999986'/>
+ <testcase classname='MSC_Tests' name='TC_lu_and_mt_call' time='4.284560'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1_log_msc_debug' time='1.040196'/>
+</testsuite>
diff --git a/sgsn/expected-results.log b/sgsn/expected-results.log
new file mode 100644
index 0000000..502dde8
--- /dev/null
+++ b/sgsn/expected-results.log
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='22' failures='5' errors='2' skipped='0' inconc='0' time='491.00'>
+ <testcase classname='SGSN_Tests' name='TC_attach' time='7.164349'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_mnc3' time='7.007076'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_auth_id_timeout' time='35.991976'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_auth_sai_timeout' time='36.007949'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1157 SGSN_Tests control part
+ SGSN_Tests.ttcn:480 TC_attach_auth_sai_timeout testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_auth_sai_reject' time='36.020705'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1158 SGSN_Tests control part
+ SGSN_Tests.ttcn:501 TC_attach_auth_sai_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_gsup_lu_timeout' time='37.025962'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1159 SGSN_Tests control part
+ SGSN_Tests.ttcn:529 TC_attach_gsup_lu_timeout testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_gsup_lu_reject' time='37.024621'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1160 SGSN_Tests control part
+ SGSN_Tests.ttcn:558 TC_attach_gsup_lu_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_combined' time='7.007181'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_accept_all' time='7.995753'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_closed' time='8.017589'/>
+ <testcase classname='SGSN_Tests' name='TC_rau_unknown' time='6.998580'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_rau' time='7.008848'/>
+ <testcase classname='SGSN_Tests' name='TC_detach_unknown_nopoweroff' time='6.993647'>
+ <error type='DTE'>Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 77, which has already terminated.</error>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_detach_unknown_poweroff' time='6.982069'>
+ <error type='DTE'>Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 83, which has already terminated.</error>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_detach_nopoweroff' time='7.010464'/>
+ <testcase classname='SGSN_Tests' name='TC_detach_poweroff' time='12.011997'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act' time='6.050101'/>
+ <testcase classname='SGSN_Tests' name='TC_pdp_act_unattached' time='5.999444'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user' time='6.053814'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_ggsn_reject' time='6.038840'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user_deact_mo' time='6.041516'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user_deact_mt' time='36.021021'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1175 SGSN_Tests control part
+ SGSN_Tests.ttcn:1148 TC_attach_pdp_act_user_deact_mt testcase
+ </failure>
+ </testcase>
+</testsuite>
diff --git a/sip/expected-results.log b/sip/expected-results.log
new file mode 100644
index 0000000..54123a5
--- /dev/null
+++ b/sip/expected-results.log
@@ -0,0 +1,7 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='4' failures='0' errors='0' skipped='0' inconc='0' time='106.00'>
+ <testcase classname='SIP_Tests' name='TC_mt_success_rel_gsm' time='5.032948'/>
+ <testcase classname='SIP_Tests' name='TC_mt_success_rel_sip' time='5.032518'/>
+ <testcase classname='SIP_Tests' name='TC_mo_success_rel_gsm' time='5.032548'/>
+ <testcase classname='SIP_Tests' name='TC_mo_success_rel_sip' time='5.059944'/>
+</testsuite>
diff --git a/start-testsuite.sh b/start-testsuite.sh
index 8affdba..5b187fa 100755
--- a/start-testsuite.sh
+++ b/start-testsuite.sh
@@ -10,7 +10,9 @@ if [ $# -lt 1 ]; then
fi
SUITE=$1
-CFG=$(basename "$SUITE").cfg
+SUITE_DIR="$(dirname "$SUITE")"
+SUITE_NAME="$(basename "$SUITE")"
+CFG="$SUITE_NAME.cfg"
if [ $# -gt 1 ]; then
CFG=$2
fi
@@ -19,4 +21,26 @@ if [ $# -gt 2 ]; then
TEST=$3
fi
-LD_LIBRARY_PATH=$(dirname "$SUITE"):/usr/lib/titan:/usr/ttcn3/lib ttcn3_start $SUITE $CFG $TEST
+LD_LIBRARY_PATH="$SUITE_DIR:/usr/lib/titan:/usr/ttcn3/lib" ttcn3_start $SUITE $CFG $TEST
+
+expected="$SUITE_DIR/expected-results.log"
+if [ ! -f "$expected" ]; then
+ echo "No expected results found, not comparing outcome. ($expected)"
+ exit 0
+fi
+
+# find the most recent junit output log here
+last_log="$(ls -1tr junit*.log | tail -n 1)"
+if [ ! -f "$last_log" ]; then
+ echo "No junit log found."
+ exit 1
+fi
+
+compare="$SUITE_DIR/../compare-results.sh"
+if [ ! -x "$compare" ]; then
+ echo "ERROR: cannot find $compare"
+ exit 1
+fi
+
+set -e
+"$compare" "$expected" "$last_log" $OSMO_TTCN3_COMPARE_ARGS
diff --git a/sysinfo/expected-results.log b/sysinfo/expected-results.log
new file mode 100644
index 0000000..cea6f54
--- /dev/null
+++ b/sysinfo/expected-results.log
@@ -0,0 +1,69 @@
+<?xml version="1.0"?>
+<testsuite name='osmo-ttcn3-hacks/sysinfo' tests='22' failures='3' errors='16' skipped='0' inconc='0' time='200.00'>
+ <testcase classname='Test' name='TC_si1' time='0.001430'/>
+ <testcase classname='Test' name='TC_telnet' time='0.004703'/>
+ <testcase classname='Test' name='TC_si_default' time='8.009958'>
+ <failure type='fail-verdict'>No SI messages for TC=0!
+
+ </failure>
+ </testcase>
+ <testcase classname='Test' name='TC_si_sched_2bis' time='10.034880'>
+ <failure type='fail-verdict'>No SI messages for TC=0!
+
+ </failure>
+ </testcase>
+ <testcase classname='Test' name='TC_si_sched_2ter' time='10.035191'>
+ <failure type='fail-verdict'>No SI messages for TC=0!
+
+ </failure>
+ </testcase>
+ <testcase classname='Test' name='TC_si_sched_2ter_2bis' time='9.020499'>
+ <error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_si_sched_2quater' time='2.027051'/>
+ <testcase classname='Test' name='TC_si_sched_13' time='9.024905'>
+ <error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_si_sched_13_2bis_2ter_2quater' time='9.024221'>
+ <error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_neci' time='11.005209'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_cell_resel_hyst' time='11.020545'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_rxlev_acc_min' time='11.016459'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_cellid' time='11.014834'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_lac' time='11.014391'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_rach_tx_int' time='11.016352'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_rach_max_tx' time='11.014782'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_attach' time='11.005593'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_dtx_ul' time='10.013151'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_emerg_allowed' time='11.014455'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_bs_pa_mfrms' time='11.005825'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_bs_ag_blks_res' time='11.014585'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+ <testcase classname='Test' name='TC_radio_link_timeout' time='11.025433'>
+ <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+ </testcase>
+</testsuite>