aboutsummaryrefslogtreecommitdiffstats
path: root/selftest
diff options
context:
space:
mode:
authorPau Espin Pedrol <pespin@sysmocom.de>2020-06-15 14:27:50 +0200
committerpespin <pespin@sysmocom.de>2020-06-15 15:42:58 +0000
commite3d1b61175d3f177ff15679954785d5d72b7ad8e (patch)
tree1be6d6a8ccd70f8df952945f23b4e6451a2bce8a /selftest
parentd4dc2ad3a2767bcebf4eb2bba2cefee0c26f6d6e (diff)
Add per-test KPI support
tests can now use 'tenv.test().set_kpis(some_dict)' to set any kind of data as KPIs, which will be presented in the junit report. The representation of KPIs in the xml file doesn't follow the junit format, mainly because it has no support for per-test properties. Change-Id: I00e976f65a202e82d440bf33708f06c8ce2643e2
Diffstat (limited to 'selftest')
-rw-r--r--selftest/report_test/expected_junit_output.xml34
-rw-r--r--selftest/report_test/report_test.ok2
-rwxr-xr-xselftest/report_test/report_test.py15
3 files changed, 48 insertions, 3 deletions
diff --git a/selftest/report_test/expected_junit_output.xml b/selftest/report_test/expected_junit_output.xml
index 5de0edf..9f6185c 100644
--- a/selftest/report_test/expected_junit_output.xml
+++ b/selftest/report_test/expected_junit_output.xml
@@ -1,4 +1,4 @@
-<testsuites errors="2" failures="1" name="trial" tests="10" time="102">
+<testsuites errors="2" failures="2" name="trial" tests="12" time="122">
<testsuite disabled="0" errors="0" failures="0" hostname="localhost" id="0" name="suiteA" skipped="0" tests="2">
<testcase classname="suiteA" name="suiteA-0" time="30">
<system-out>test log file not available</system-out>
@@ -59,4 +59,34 @@
<property name="ref:orange" value="abcd"></property>
</properties>
</testsuite>
-</testsuites>
+ <testsuite disabled="0" errors="0" failures="1" hostname="localhost" id="4" name="suiteE" skipped="0" tests="2">
+ <testcase classname="suiteE" name="suiteE-0" time="12">
+ <failure type="fake_fail_type">fake_fail_message</failure>
+ <system-err>system stderr fake content</system-err>
+ <kpis>
+ <kpi_node name="ueA">
+ <property name="kpiA" value="30"></property>
+ <property name="kpiB" value="foobar"></property>
+ <kpi_node name="yet-another-level">
+ <property name="foo" value="bar"></property>
+ </kpi_node>
+ </kpi_node>
+ <kpi_node name="enbD">
+ <property name="foobar-boolean" value="True"></property>
+ </kpi_node>
+ <property name="somekpi" value="someval"></property>
+ </kpis>
+ <system-out>test log file not available</system-out>
+ </testcase>
+ <testcase classname="suiteE" name="suiteE-1" time="10">
+ <kpis>
+ <property name="abcd" value="abcdval"></property>
+ </kpis>
+ <system-out>test log file not available</system-out>
+ </testcase>
+ <properties>
+ <property name="ref:foobar/potato" value="1234"></property>
+ <property name="ref:orange" value="abcd"></property>
+ </properties>
+ </testsuite>
+</testsuites> \ No newline at end of file
diff --git a/selftest/report_test/report_test.ok b/selftest/report_test/report_test.ok
index 87092ea..442d343 100644
--- a/selftest/report_test/report_test.ok
+++ b/selftest/report_test/report_test.ok
@@ -15,3 +15,5 @@ tst suiteC: DBG: {combining='config'}
tst {combining_scenarios='config'}: DBG: {definition_conf={}} [suiteC↪{combining_scenarios='config'}]
tst suiteD: DBG: {combining='config'}
tst {combining_scenarios='config'}: DBG: {definition_conf={}} [suiteD↪{combining_scenarios='config'}]
+tst suiteE: DBG: {combining='config'}
+tst {combining_scenarios='config'}: DBG: {definition_conf={}} [suiteE↪{combining_scenarios='config'}]
diff --git a/selftest/report_test/report_test.py b/selftest/report_test/report_test.py
index 57e3a89..5888bf3 100755
--- a/selftest/report_test/report_test.py
+++ b/selftest/report_test/report_test.py
@@ -39,11 +39,13 @@ class FakeSuiteDefinition(log.Origin):
self.suite_dir = util.Dir(example_trial_dir).new_child('suitedef' + name)
-def fake_run_test(test_obj, status, duration, sysout=None):
+def fake_run_test(test_obj, status, duration, sysout=None, kpis=None):
test_obj.status = status
test_obj.duration = duration
if sysout is not None:
test_obj.set_report_stdout(sysout)
+ if kpis is not None:
+ test_obj.set_kpis(kpis)
if status == test.Test.FAIL:
test_obj.fail_type = 'fake_fail_type'
test_obj.fail_message = 'fake_fail_message'
@@ -92,6 +94,14 @@ fake_run_test(s.tests[0], test.Test.FAIL, 12)
fake_run_test(s.tests[1], test.Test.PASS, 10)
fake_run_suite(s, 20)
+# Test adding KPIs
+s_def = FakeSuiteDefinition('suiteE', 2)
+s = suite.SuiteRun(trial, s_def.name(), s_def)
+trial.suites.append(s)
+fake_run_test(s.tests[0], test.Test.FAIL, 12, kpis={'ueA': {'kpiA': 30, 'kpiB': 'foobar', 'yet-another-level': {'foo': 'bar'}}, 'enbD': {'foobar-boolean': True }, 'somekpi': 'someval'})
+fake_run_test(s.tests[1], test.Test.PASS, 10, kpis={'abcd': 'abcdval'})
+fake_run_suite(s, 20)
+
element = report.trial_to_junit(trial)
def indent(elem, level=0):
@@ -126,6 +136,9 @@ if hasattr(et, 'canonicalize'):
with open(exp_path, 'r') as f:
exp = f.read().rstrip()
udiff(exp, got, exp_path)
+ # Uncomment to update exp_path:
+ #with open(exp_path, 'w') as f:
+ # f.write(got)
#deleting generated tmp trial dir:
shutil.rmtree(example_trial_dir, ignore_errors=True)