summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNeels Hofmeyr <neels@hofmeyr.de>2018-04-05 15:44:17 +0200
committerNeels Hofmeyr <neels@hofmeyr.de>2018-04-05 15:46:33 +0200
commit94d11b4d300213424e19bcc3ac4b882a1ca09e7b (patch)
treea760aab193c95524e5b4fe9f0139e02cdb12a244
parentee2e866b416dc295d37a1f283feac64d7e809119 (diff)
summary logger: add expected-verdicts functionalityneels/summarylogger
-rw-r--r--libsummarylogger/SummaryLogger.cc96
-rw-r--r--libsummarylogger/SummaryLogger.hh6
2 files changed, 100 insertions, 2 deletions
diff --git a/libsummarylogger/SummaryLogger.cc b/libsummarylogger/SummaryLogger.cc
index 9cadb55..ca30689 100644
--- a/libsummarylogger/SummaryLogger.cc
+++ b/libsummarylogger/SummaryLogger.cc
@@ -30,7 +30,7 @@ TestCase::TestCase()
void TestCase::reset()
{
verdict = Unbound;
- expected_verdict = Skipped;
+ expected_verdict = NULL;
tc_name = "";
module_name = "";
}
@@ -174,7 +174,23 @@ void SummaryLogger::log(const TitanLoggerApi::TitanLogEvent& event,
}
void TestCase::writeTestCase(FILE* file_stream_) const{
- fprintf(file_stream_, "%s.%s: %s\n", module_name.data(), tc_name.data(), verdict);
+
+ if (!expected_verdict || expected_verdict == verdict) {
+ const char *verdict_str = verdict;
+ if (expected_verdict == Fail)
+ verdict_str = XFail;
+ /* We don't know what verdict to expect. Just write out results. */
+ fprintf(file_stream_, "%-5s", verdict_str);
+ } else {
+ /* There is an expected verdict read in from an expected-verdicts.txt.
+ * Clearly indicate verdict changes. */
+ const char *expected_verdict_str = expected_verdict;
+ if (expected_verdict == Fail)
+ expected_verdict_str = XFail;
+
+ fprintf(file_stream_, "%s->%s", expected_verdict_str, verdict);
+ }
+ fprintf(file_stream_, " %s.%s\n", module_name.data(), tc_name.data());
fflush(file_stream_);
}
@@ -189,6 +205,7 @@ void TestSuite::addTestCase(const TestCase& testcase) {
}
void TestSuite::write(FILE* file_stream_) {
+ readExpectedVerdicts();
for (TestCases::const_iterator it = testcases.begin(); it != testcases.end(); ++it) {
(*it)->writeTestCase(file_stream_);
}
@@ -224,3 +241,78 @@ void TestCase::setTCVerdict(const TitanLoggerApi::TitanLogEvent& event){
break;
}
}
+
+void TestSuite::readExpectedVerdicts(const char *path)
+{
+ char *line = NULL;
+ char *saveptr = NULL;
+ size_t len = 0;
+ ssize_t read;
+ FILE *f;
+ if (!path)
+ path = "expected-verdicts.txt";
+ f = fopen(path, "r");
+ if (!f)
+ return;
+
+ while ((read = getline(&line, &len, f)) != -1) {
+ char *tokens[3] = {};
+ int i;
+
+ /* Expecting a line like written above:
+ * pass MSC_Tests.TC_foo
+ * [0] [1] [2]
+ */
+ for (i = 0; i < 3; i++) {
+ tokens[i] = strtok_r(i ? NULL : line, " .\n", &saveptr);
+ if (!tokens[i]) {
+ i = -1;
+ break;
+ }
+ }
+
+ /* This line didn't have enough tokens. */
+ if (i != 3)
+ continue;
+
+ evaluateExpectedVerdict(tokens[1], tokens[2], tokens[0]);
+ }
+
+ fclose(f);
+
+}
+
+void TestSuite::evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+ const char *verdict_name)
+{
+ for (TestCases::const_iterator it = testcases.begin(); it != testcases.end(); ++it) {
+ if ((*it)->evaluateExpectedVerdict(suite_name, test_name, verdict_name))
+ return;
+ }
+}
+
+bool TestCase::evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+ const char *verdict_name)
+{
+ if (strcmp(suite_name, this->module_name.data()))
+ return false;
+ if (strcmp(test_name, this->tc_name.data()))
+ return false;
+
+ if (!strcmp(verdict_name, Pass))
+ this->expected_verdict = Pass;
+ else if (!strcmp(verdict_name, Inconc))
+ this->expected_verdict = Inconc;
+ else if (!strcmp(verdict_name, Fail)
+ || !strcmp(verdict_name, XFail))
+ this->expected_verdict = Fail;
+ else if (!strcmp(verdict_name, Error))
+ this->expected_verdict = Error;
+ else if (!strcmp(verdict_name, Unbound))
+ this->expected_verdict = Unbound;
+ else if (!strcmp(verdict_name, Skipped))
+ this->expected_verdict = Skipped;
+ else
+ return false;
+ return true;
+}
diff --git a/libsummarylogger/SummaryLogger.hh b/libsummarylogger/SummaryLogger.hh
index a98533a..dadeaea 100644
--- a/libsummarylogger/SummaryLogger.hh
+++ b/libsummarylogger/SummaryLogger.hh
@@ -29,6 +29,7 @@ struct TestCase {
static constexpr const char* Pass = "pass";
static constexpr const char* Inconc = "INCONCLUSIVE";
static constexpr const char* Fail = "FAIL";
+ static constexpr const char* XFail = "xfail";
static constexpr const char* Error = "ERROR";
static constexpr const char* Unbound = "UNBOUND";
static constexpr const char* Skipped = "skipped";
@@ -43,6 +44,8 @@ struct TestCase {
void writeTestCase(FILE* file_stream_) const;
void setTCVerdict(const TitanLoggerApi::TitanLogEvent& event);
void reset();
+ bool evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+ const char *verdict_name);
};
@@ -62,6 +65,9 @@ struct TestSuite {
void addTestCase(const TestCase& element);
void write(FILE* file_stream_);
+ void readExpectedVerdicts(const char *path=NULL);
+ void evaluateExpectedVerdict(const char *suite_name, const char *test_name,
+ const char *verdict_name);
};
class SummaryLogger: public ILoggerPlugin