aboutsummaryrefslogtreecommitdiffstats
path: root/selftest
diff options
context:
space:
mode:
authorPau Espin Pedrol <pespin@sysmocom.de>2017-05-15 18:24:35 +0200
committerPau Espin Pedrol <pespin@sysmocom.de>2017-05-19 14:22:39 +0200
commit0ffb41440661631fa1d520c152be4cf8ebd4c46b (patch)
tree4b9863b5907e43974a5387750eba3d4ead24287e /selftest
parent023fd2c748b97d295872dfbfc3aafe119de74684 (diff)
Add JUnit XML reports; refactor test reporting
* Add Junit output file support * Differentiate between an expected failure test and an error in the test, as described in JUnit. * In case of an error/exception during test, record and attach it to the Test object and continue running the tests, and show it at the end during the trial report. Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5
Diffstat (limited to 'selftest')
-rw-r--r--selftest/suite_test.ok25
-rwxr-xr-xselftest/suite_test.py26
-rwxr-xr-xselftest/suite_test/test_suite/test_fail.py6
-rwxr-xr-xselftest/suite_test/test_suite/test_fail_raise.py6
-rw-r--r--selftest/trial_test.ok2
5 files changed, 56 insertions, 9 deletions
diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok
index fda77dc..30c6915 100644
--- a/selftest/suite_test.ok
+++ b/selftest/suite_test.ok
@@ -59,15 +59,30 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] PASS [test_suite↪hello_world.py]
-pass: all 1 tests passed.
+tst test_suite: PASS
+pass: all 6 tests passed (5 skipped).
- a test with an error
tst test_suite: Suite run start [suite.py:[LINENR]]
tst test_error.py:[LINENR] START [test_suite↪test_error.py] [suite.py:[LINENR]]
tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]]
-tst test_error.py:[LINENR]: FAIL [test_suite↪test_error.py:[LINENR]] [suite.py:[LINENR]]
-tst test_error.py:[LINENR]: ERR: AssertionError: [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False]
-FAIL: 1 of 1 tests failed:
- test_error.py
+tst test_error.py:[LINENR]: ERR: AssertionError: [test_error.py:[LINENR]: assert False]
+tst test_error.py:[LINENR] FAIL (AssertionError) [test_suite↪test_error.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
+
+- a test with a failure
+tst test_suite: Suite run start [suite.py:[LINENR]]
+tst test_fail.py:[LINENR] START [test_suite↪test_fail.py] [suite.py:[LINENR]]
+tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]]
+tst test_fail.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
+
+- a test with a raised failure
+tst test_suite: Suite run start [suite.py:[LINENR]]
+tst test_fail_raise.py:[LINENR] START [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
+tst test_fail_raise.py:[LINENR]: I am 'test_suite' / 'test_fail_raise.py:[LINENR]' [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]]
+tst test_fail_raise.py:[LINENR]: ERR: Failure: ('EpicFail', 'This failure is expected') [test_fail_raise.py:[LINENR]: raise Failure('EpicFail', 'This failure is expected')]
+tst test_fail_raise.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
- graceful exit.
diff --git a/selftest/suite_test.py b/selftest/suite_test.py
index 315c683..16342c5 100755
--- a/selftest/suite_test.py
+++ b/selftest/suite_test.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import os
import _prep
-from osmo_gsm_tester import log, suite, config
+from osmo_gsm_tester import log, suite, config, report
config.ENV_CONF = './suite_test'
@@ -22,13 +22,33 @@ print(config.tostr(s_def.conf))
print('- run hello world test')
s = suite.SuiteRun(None, 'test_suite', s_def)
results = s.run_tests('hello_world.py')
-print(str(results))
+print(report.suite_to_text(s))
log.style_change(src=True)
#log.style_change(trace=True)
print('\n- a test with an error')
results = s.run_tests('test_error.py')
-print(str(results))
+output = report.suite_to_text(s)
+assert 'FAIL: [test_suite] 1 failed ' in output
+assert 'FAIL: [test_error.py]' in output
+assert "type:'AssertionError' message: AssertionError()" in output
+assert 'assert False' in output
+
+print('\n- a test with a failure')
+results = s.run_tests('test_fail.py')
+output = report.suite_to_text(s)
+assert 'FAIL: [test_suite] 1 failed ' in output
+assert 'FAIL: [test_fail.py]' in output
+assert "type:'EpicFail' message: This failure is expected" in output
+assert "test.set_fail('EpicFail', 'This failure is expected')" in output
+
+print('\n- a test with a raised failure')
+results = s.run_tests('test_fail_raise.py')
+output = report.suite_to_text(s)
+assert 'FAIL: [test_suite] 1 failed ' in output
+assert 'FAIL: [test_fail_raise.py]' in output
+assert "type:'EpicFail' message: This failure is expected" in output
+assert "raise Failure('EpicFail', 'This failure is expected')" in output
print('\n- graceful exit.')
# vim: expandtab tabstop=4 shiftwidth=4
diff --git a/selftest/suite_test/test_suite/test_fail.py b/selftest/suite_test/test_suite/test_fail.py
new file mode 100755
index 0000000..6880c81
--- /dev/null
+++ b/selftest/suite_test/test_suite/test_fail.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+print('I am %r / %r' % (suite.name(), test.name()))
+
+test.set_fail('EpicFail', 'This failure is expected')
diff --git a/selftest/suite_test/test_suite/test_fail_raise.py b/selftest/suite_test/test_suite/test_fail_raise.py
new file mode 100755
index 0000000..a7b0b61
--- /dev/null
+++ b/selftest/suite_test/test_suite/test_fail_raise.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+print('I am %r / %r' % (suite.name(), test.name()))
+
+raise Failure('EpicFail', 'This failure is expected')
diff --git a/selftest/trial_test.ok b/selftest/trial_test.ok
index 6ad39a9..7434a10 100644
--- a/selftest/trial_test.ok
+++ b/selftest/trial_test.ok
@@ -4,7 +4,7 @@
[TMP]/third
- fetch trial dirs in order
first
-['taken']
+['last_run', 'run.[TIMESTAMP]', 'taken']
second
third
- no more trial dirs left