From 3cf797d7a59a9f5fc05bd22af32d21c23409e60d Mon Sep 17 00:00:00 2001
From: Neels Hofmeyr <neels@hofmeyr.de>
Date: Thu, 5 Apr 2018 16:56:38 +0200
Subject: add compare-results.sh, call from start-testsuite.sh

Compare current test results to the expected results, and exit in error on
discrepancies.

Add compare-result.sh: (trivially) grep junit xml output to determine which
tests passed and which didn't, and compare against an expected-result.log,
another junit file from a previous run. Summarize and determine success.

Include an "xfail" feature: tests that are expected to fail are marked as
"xfail", unexpected failures as "FAIL".

In various subdirs, copy the current jenkins jobs' junit xml outputs as
expected-results.log, so that we will start getting useful output in both
jenkins runs and manual local runs.

In start-testsuite.sh, after running the tests, invoke the results comparison.

Due to the single-line parsing nature, the script so far does not distinguish
between error and failure. I doubt that we actually need to do that though.

Related: OS#3136
Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d
---
 sysinfo/expected-results.log | 69 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 sysinfo/expected-results.log

(limited to 'sysinfo')

diff --git a/sysinfo/expected-results.log b/sysinfo/expected-results.log
new file mode 100644
index 00000000..cea6f54b
--- /dev/null
+++ b/sysinfo/expected-results.log
@@ -0,0 +1,69 @@
+<?xml version="1.0"?>
+<testsuite name='osmo-ttcn3-hacks/sysinfo' tests='22' failures='3' errors='16' skipped='0' inconc='0' time='200.00'>
+  <testcase classname='Test' name='TC_si1' time='0.001430'/>
+  <testcase classname='Test' name='TC_telnet' time='0.004703'/>
+  <testcase classname='Test' name='TC_si_default' time='8.009958'>
+    <failure type='fail-verdict'>No SI messages for TC=0!
+
+    </failure>
+  </testcase>
+  <testcase classname='Test' name='TC_si_sched_2bis' time='10.034880'>
+    <failure type='fail-verdict'>No SI messages for TC=0!
+
+    </failure>
+  </testcase>
+  <testcase classname='Test' name='TC_si_sched_2ter' time='10.035191'>
+    <failure type='fail-verdict'>No SI messages for TC=0!
+
+    </failure>
+  </testcase>
+  <testcase classname='Test' name='TC_si_sched_2ter_2bis' time='9.020499'>
+    <error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_si_sched_2quater' time='2.027051'/>
+  <testcase classname='Test' name='TC_si_sched_13' time='9.024905'>
+    <error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_si_sched_13_2bis_2ter_2quater' time='9.024221'>
+    <error type='DTE'>Test.ttcn:162: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_neci' time='11.005209'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_cell_resel_hyst' time='11.020545'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_rxlev_acc_min' time='11.016459'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_cellid' time='11.014834'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_lac' time='11.014391'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_rach_tx_int' time='11.016352'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_rach_max_tx' time='11.014782'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_attach' time='11.005593'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_dtx_ul' time='10.013151'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_emerg_allowed' time='11.014455'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_bs_pa_mfrms' time='11.005825'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_bs_ag_blks_res' time='11.014585'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+  <testcase classname='Test' name='TC_radio_link_timeout' time='11.025433'>
+    <error type='DTE'>Test.ttcn:478: Dynamic test case error: testcase.stop</error>
+  </testcase>
+</testsuite>
-- 
cgit v1.2.3