path: root/sgsn
diff options
authorNeels Hofmeyr <>2018-04-05 16:56:38 +0200
committerNeels Hofmeyr <>2018-04-06 01:07:15 +0200
commit3cf797d7a59a9f5fc05bd22af32d21c23409e60d (patch)
treed3dd5b5741ba5e576814b8879319e8d85a076f7b /sgsn
parentdef4ad4bcefbe065bc3446b20fb8b32769158425 (diff)
add, call from
Compare current test results to the expected results, and exit in error on discrepancies. Add (trivially) grep junit xml output to determine which tests passed and which didn't, and compare against an expected-result.log, another junit file from a previous run. Summarize and determine success. Include an "xfail" feature: tests that are expected to fail are marked as "xfail", unexpected failures as "FAIL". In various subdirs, copy the current jenkins jobs' junit xml outputs as expected-results.log, so that we will start getting useful output in both jenkins runs and manual local runs. In, after running the tests, invoke the results comparison. Due to the single-line parsing nature, the script so far does not distinguish between error and failure. I doubt that we actually need to do that though. Related: OS#3136 Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d
Diffstat (limited to 'sgsn')
1 files changed, 54 insertions, 0 deletions
diff --git a/sgsn/expected-results.log b/sgsn/expected-results.log
new file mode 100644
index 0000000..502dde8
--- /dev/null
+++ b/sgsn/expected-results.log
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='22' failures='5' errors='2' skipped='0' inconc='0' time='491.00'>
+ <testcase classname='SGSN_Tests' name='TC_attach' time='7.164349'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_mnc3' time='7.007076'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_auth_id_timeout' time='35.991976'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_auth_sai_timeout' time='36.007949'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1157 SGSN_Tests control part
+ SGSN_Tests.ttcn:480 TC_attach_auth_sai_timeout testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_auth_sai_reject' time='36.020705'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1158 SGSN_Tests control part
+ SGSN_Tests.ttcn:501 TC_attach_auth_sai_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_gsup_lu_timeout' time='37.025962'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1159 SGSN_Tests control part
+ SGSN_Tests.ttcn:529 TC_attach_gsup_lu_timeout testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_gsup_lu_reject' time='37.024621'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1160 SGSN_Tests control part
+ SGSN_Tests.ttcn:558 TC_attach_gsup_lu_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_attach_combined' time='7.007181'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_accept_all' time='7.995753'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_closed' time='8.017589'/>
+ <testcase classname='SGSN_Tests' name='TC_rau_unknown' time='6.998580'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_rau' time='7.008848'/>
+ <testcase classname='SGSN_Tests' name='TC_detach_unknown_nopoweroff' time='6.993647'>
+ <error type='DTE'>Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 77, which has already terminated.</error>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_detach_unknown_poweroff' time='6.982069'>
+ <error type='DTE'>Dynamic test case error: Error message was received from MC: The connect operation refers to test component with component reference 83, which has already terminated.</error>
+ </testcase>
+ <testcase classname='SGSN_Tests' name='TC_detach_nopoweroff' time='7.010464'/>
+ <testcase classname='SGSN_Tests' name='TC_detach_poweroff' time='12.011997'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act' time='6.050101'/>
+ <testcase classname='SGSN_Tests' name='TC_pdp_act_unattached' time='5.999444'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user' time='6.053814'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_ggsn_reject' time='6.038840'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user_deact_mo' time='6.041516'/>
+ <testcase classname='SGSN_Tests' name='TC_attach_pdp_act_user_deact_mt' time='36.021021'>
+ <failure type='fail-verdict'>Tguard timeout
+ SGSN_Tests.ttcn:1175 SGSN_Tests control part
+ SGSN_Tests.ttcn:1148 TC_attach_pdp_act_user_deact_mt testcase
+ </failure>
+ </testcase>