summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Hans Peter Freyther <holger@moiji-mobile.com>2019-02-23 09:58:59 +0000
committerHolger Hans Peter Freyther <holger@moiji-mobile.com>2019-03-05 03:07:41 +0000
commitb4ad8d72431df2336b9c7fbc6eb9597daa80a35c (patch)
tree470080808a7c6c49f8510f5bc5d927f9f2920209
parent93a89424d0cd4838fe4d6c17c623efe62284d2ed (diff)
nitb_netreg_mass: Add code to declare a run successful/failure
-rw-r--r--src/osmo_gsm_tester/ms_driver.py12
-rw-r--r--src/osmo_ms_driver/location_update_test.py27
-rw-r--r--suites/nitb_netreg_mass/register_default_mass.py39
3 files changed, 71 insertions, 7 deletions
diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py
index e8e543c..de2f967 100644
--- a/src/osmo_gsm_tester/ms_driver.py
+++ b/src/osmo_gsm_tester/ms_driver.py
@@ -146,6 +146,18 @@ class MsDriver(log.Origin):
"""
self._test_case.print_stats()
+ def get_stats(self):
+ """
+ Returns a statistical summary of the test.
+ """
+ return self._test_case.get_stats()
+
+ def get_result_values(self):
+ """
+ Returns the raw result values of the test run in any order.
+ """
+ return self._test_case.get_result_values()
+
def cleanup(self):
"""
Cleans up the driver (e.g. AF_UNIX files).
diff --git a/src/osmo_ms_driver/location_update_test.py b/src/osmo_ms_driver/location_update_test.py
index 8500607..f14bd28 100644
--- a/src/osmo_ms_driver/location_update_test.py
+++ b/src/osmo_ms_driver/location_update_test.py
@@ -23,9 +23,11 @@ from .test_support import imsi_ki_gen, Results
from datetime import timedelta
+import collections
import time
class LUResult(Results):
+ """Representation of a Location Updating Result."""
def __init__(self, name):
super().__init__(name)
@@ -44,6 +46,10 @@ class LUResult(Results):
def lu_delay(self):
return self.lu_time() - self.start_time()
+
+LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed",
+ "min_latency", "max_latency"])
+
class MassUpdateLocationTest(log.Origin):
"""
A test to launch a configurable amount of MS and make them
@@ -207,9 +213,24 @@ class MassUpdateLocationTest(log.Origin):
max_value = result.lu_delay()
return min_value, max_value
+ def get_result_values(self):
+ """
+ Returns the raw result values of the test run in any order.
+ """
+ return self._results.values()
+
+ def get_stats(self):
+ """
+ Returns a statistical summary of the test.
+ """
+ attempted = self._number_of_ms
+ completed = attempted - self._outstanding
+ min_latency, max_latency = self.find_min_max(filter(lambda x: x.has_lu_time(), self._results.values()))
+ return LUStats(attempted, completed, min_latency, max_latency)
+
def print_stats(self):
- all_completed = self.all_completed()
- min_value, max_value = self.find_min_max(filter(lambda x: x.has_lu_time(), self._results.values()))
+ stats = self.get_stats()
+ all_completed = stats.num_attempted == stats.num_completed
self.log("Tests done", all_completed=all_completed,
- min=min_value, max=max_value)
+ min=stats.min_latency, max=stats.max_latency)
diff --git a/suites/nitb_netreg_mass/register_default_mass.py b/suites/nitb_netreg_mass/register_default_mass.py
index ede2b74..8259359 100644
--- a/suites/nitb_netreg_mass/register_default_mass.py
+++ b/suites/nitb_netreg_mass/register_default_mass.py
@@ -1,16 +1,17 @@
#!/usr/bin/env python3
"""
-Run a network registration with a 'massive' amount of MS
+Runs a network registration with a 'massive' amount of MS
using the ms_driver infrastructure.
"""
from osmo_gsm_tester.testenv import *
+from datetime import timedelta
-print('use resources...')
+print('Claiming resources for the test')
nitb = suite.nitb()
bts = suite.bts()
ms_driver = suite.ms_driver()
-print('start nitb and bts...')
+print('Launching a simple network')
nitb.bts_add(bts)
nitb.start()
bts.start()
@@ -23,5 +24,35 @@ for ms in ms_driver.ms_subscribers():
# Run the base test.
ms_driver.run_test()
-# Print stats
+# Print the stats of the run.
ms_driver.print_stats()
+
+# Evaluate if this run was successful or not. Our initial acceptance criteria
+# is quite basic but it should allow us to scale to a larger number of MS and
+# reasons (e.g. have a full BCCH).
+#
+# 99% of LUs should complete
+# 99% of successful LUs should complete within 10s.
+stats = ms_driver.get_stats()
+if len(mobiles) > 0 and stats.num_completed < 1:
+ raise Exception("No run completed.")
+completion_ratio = stats.num_attempted / stats.num_completed
+
+# Verify that 99% of LUs completed.
+if completion_ratio < 0.99:
+ raise Exception("Completion ratio of %f%% lower than threshold." % (completion_ratio * 100.0))
+
+# Check how many results are below our threshold.
+acceptable_delay = timedelta(seconds=20)
+results = ms_driver.get_result_values()
+quick_enough = 0
+for result in results:
+ if not result.has_lu_time():
+ continue
+ if timedelta(seconds=result.lu_delay()) >= acceptable_delay:
+ continue
+ quick_enough = quick_enough + 1
+
+latency_ratio = quick_enough / len(results)
+if latency_ratio < 0.99:
+ raise Exception("Latency ratio of %f%% lower than threshold." % (latency_ratio * 100.0))