summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Hans Peter Freyther <holger@moiji-mobile.com>2019-02-27 13:00:33 +0000
committerHolger Freyther <holger@freyther.de>2019-03-05 03:53:10 +0000
commit54b4fa928ea371cc2b1a0e249223719dff5031cb (patch)
treea9c202941153a90c0c6752b1639900540552f1c0
parentb4ad8d72431df2336b9c7fbc6eb9597daa80a35c (diff)
process: Make killing processes non-sequential
-rw-r--r--selftest/suite_test.ok56
-rw-r--r--src/osmo_gsm_tester/process.py76
2 files changed, 129 insertions, 3 deletions
diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok
index 908f24f..85e94b2 100644
--- a/selftest/suite_test.ok
+++ b/selftest/suite_test.ok
@@ -96,6 +96,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes.
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes.
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL
---------------------------------------------------------------------
trial test_suite PASS
---------------------------------------------------------------------
@@ -119,6 +127,14 @@ trial test_suite test_error.py
tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]]
tst test_error.py:[LINENR]: ERR: AssertionError: test_error.py:[LINENR]: assert False [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False]
tst test_error.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_error.py:[LINENR]] [test.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
---------------------------------------------------------------------
trial test_suite FAIL
---------------------------------------------------------------------
@@ -142,6 +158,14 @@ trial test_suite test_fail.py
tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]]
tst test_fail.py:[LINENR]: ERR: EpicFail: This failure is expected [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]]
tst test_fail.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail.py:[LINENR]] [test.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
---------------------------------------------------------------------
trial test_suite FAIL
---------------------------------------------------------------------
@@ -164,6 +188,14 @@ trial test_suite test_fail_raise.py
----------------------------------------------
tst test_fail_raise.py:[LINENR]: ERR: ExpectedFail: This failure is expected [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]: raise ExpectedFail('This failure is expected')]
tst test_fail_raise.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail_raise.py:[LINENR]] [test.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
---------------------------------------------------------------------
trial test_suite FAIL
---------------------------------------------------------------------
@@ -251,6 +283,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
---------------------------------------------------------------------
trial test_suite PASS
---------------------------------------------------------------------
@@ -338,6 +378,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
---------------------------------------------------------------------
trial test_suite PASS
---------------------------------------------------------------------
@@ -471,6 +519,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]]
+--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]]
---------------------------------------------------------------------
trial test_suite PASS
---------------------------------------------------------------------
diff --git a/src/osmo_gsm_tester/process.py b/src/osmo_gsm_tester/process.py
index 3050f83..a104e10 100644
--- a/src/osmo_gsm_tester/process.py
+++ b/src/osmo_gsm_tester/process.py
@@ -47,10 +47,75 @@ class TerminationStrategy(log.Origin, metaclass=ABCMeta):
class ParallelTerminationStrategy(TerminationStrategy):
"""Processes will be terminated in parallel."""
- def terminate_all(self):
- # TODO(zecke): Actually make this non-sequential.
+ def _prune_dead_processes(self, poll_first):
+ """Removes all dead processes from the list."""
+ # Remove all processes that terminated!
+ self._processes = list(filter(lambda proc: proc.is_running(poll_first), self._processes))
+
+ def _build_process_map(self):
+ """Builds a mapping from pid to process."""
+ self._process_map = {}
for process in self._processes:
- process.terminate()
+ pid = process.pid()
+ if pid is None:
+ continue
+ self._process_map[pid] = process
+
+ def _poll_once(self):
+ """Polls for to be collected children once."""
+ pid, result = os.waitpid(0, os.WNOHANG)
+ # Did some other process die?
+ if pid == 0:
+ return False
+ proc = self._process_map.get(pid)
+ if proc is None:
+ self.dbg("Unknown process with pid(%d) died." % pid)
+ return False
+ # Update the process state and forget about it
+ self.log("PID %d died..." % pid)
+ proc.result = result
+ proc.cleanup()
+ self._processes.remove(proc)
+ del self._process_map[pid]
+ return True
+
+ def _poll_for_termination(self, time_to_wait_for_term=5):
+ """Waits for the termination of processes until timeout|all ended."""
+
+ wait_step = 0.001
+ waited_time = 0
+ while len(self._processes) > 0:
+ # Collect processes until there are none to be collected.
+ while True:
+ try:
+ if not self._poll_once():
+ break
+ except ChildProcessError:
+ break
+
+ # All processes died and we can return before sleeping
+ if len(self._processes) == 0:
+ break
+ waited_time += wait_step
+ # make wait_step approach 1.0
+ wait_step = (1. + 5. * wait_step) / 6.
+ if waited_time >= time_to_wait_for_term:
+ break
+ time.sleep(wait_step)
+
+ def terminate_all(self):
+ self.dbg("Scheduled to terminate %d processes." % len(self._processes))
+ self._prune_dead_processes(True)
+ self._build_process_map()
+
+ # Iterate through all signals.
+ for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGKILL]:
+ self.dbg("Starting to kill with %s" % sig.name)
+ for process in self._processes:
+ process.kill(sig)
+ if sig == signal.SIGKILL:
+ continue
+ self._poll_for_termination()
class Process(log.Origin):
@@ -145,6 +210,11 @@ class Process(log.Origin):
def send_signal(self, sig):
os.kill(self.process_obj.pid, sig)
+ def pid(self):
+ if self.process_obj is None:
+ return None
+ return self.process_obj.pid
+
def kill(self, sig):
"""Kills the process with the given signal and remembers it."""
self.log('Terminating (%s)' % sig.name)