From 54b4fa928ea371cc2b1a0e249223719dff5031cb Mon Sep 17 00:00:00 2001 From: Holger Hans Peter Freyther Date: Wed, 27 Feb 2019 13:00:33 +0000 Subject: process: Make killing processes non-sequential Change-Id: Icf1ac6774ea11880542012fd6c6ac73302bb74f5 --- selftest/suite_test.ok | 56 +++++++++++++++++++++++++++++++ src/osmo_gsm_tester/process.py | 76 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 3 deletions(-) diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok index 908f24f..85e94b2 100644 --- a/selftest/suite_test.ok +++ b/selftest/suite_test.ok @@ -96,6 +96,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- @@ -119,6 +127,14 @@ trial test_suite test_error.py tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]] tst test_error.py:[LINENR]: ERR: AssertionError: test_error.py:[LINENR]: assert False [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False] tst test_error.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_error.py:[LINENR]] [test.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- @@ -142,6 +158,14 @@ trial test_suite test_fail.py tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]] tst test_fail.py:[LINENR]: ERR: EpicFail: This failure is expected [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]] tst test_fail.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail.py:[LINENR]] [test.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- @@ -164,6 +188,14 @@ trial test_suite test_fail_raise.py ---------------------------------------------- tst test_fail_raise.py:[LINENR]: ERR: ExpectedFail: This failure is expected [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]: raise ExpectedFail('This failure is expected')] tst test_fail_raise.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail_raise.py:[LINENR]] [test.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- @@ -251,6 +283,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- @@ -338,6 +378,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- @@ -471,6 +519,14 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Scheduled to terminate 0 processes. [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGTERM [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGINT [process.py:[LINENR]] +--- ParallelTerminationStrategy: DBG: Starting to kill with SIGKILL [process.py:[LINENR]] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- diff --git a/src/osmo_gsm_tester/process.py b/src/osmo_gsm_tester/process.py index 3050f83..a104e10 100644 --- a/src/osmo_gsm_tester/process.py +++ b/src/osmo_gsm_tester/process.py @@ -47,10 +47,75 @@ class TerminationStrategy(log.Origin, metaclass=ABCMeta): class ParallelTerminationStrategy(TerminationStrategy): """Processes will be terminated in parallel.""" - def terminate_all(self): - # TODO(zecke): Actually make this non-sequential. + def _prune_dead_processes(self, poll_first): + """Removes all dead processes from the list.""" + # Remove all processes that terminated! + self._processes = list(filter(lambda proc: proc.is_running(poll_first), self._processes)) + + def _build_process_map(self): + """Builds a mapping from pid to process.""" + self._process_map = {} for process in self._processes: - process.terminate() + pid = process.pid() + if pid is None: + continue + self._process_map[pid] = process + + def _poll_once(self): + """Polls for to be collected children once.""" + pid, result = os.waitpid(0, os.WNOHANG) + # Did some other process die? + if pid == 0: + return False + proc = self._process_map.get(pid) + if proc is None: + self.dbg("Unknown process with pid(%d) died." % pid) + return False + # Update the process state and forget about it + self.log("PID %d died..." % pid) + proc.result = result + proc.cleanup() + self._processes.remove(proc) + del self._process_map[pid] + return True + + def _poll_for_termination(self, time_to_wait_for_term=5): + """Waits for the termination of processes until timeout|all ended.""" + + wait_step = 0.001 + waited_time = 0 + while len(self._processes) > 0: + # Collect processes until there are none to be collected. + while True: + try: + if not self._poll_once(): + break + except ChildProcessError: + break + + # All processes died and we can return before sleeping + if len(self._processes) == 0: + break + waited_time += wait_step + # make wait_step approach 1.0 + wait_step = (1. + 5. * wait_step) / 6. + if waited_time >= time_to_wait_for_term: + break + time.sleep(wait_step) + + def terminate_all(self): + self.dbg("Scheduled to terminate %d processes." % len(self._processes)) + self._prune_dead_processes(True) + self._build_process_map() + + # Iterate through all signals. + for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGKILL]: + self.dbg("Starting to kill with %s" % sig.name) + for process in self._processes: + process.kill(sig) + if sig == signal.SIGKILL: + continue + self._poll_for_termination() class Process(log.Origin): @@ -145,6 +210,11 @@ class Process(log.Origin): def send_signal(self, sig): os.kill(self.process_obj.pid, sig) + def pid(self): + if self.process_obj is None: + return None + return self.process_obj.pid + def kill(self, sig): """Kills the process with the given signal and remembers it.""" self.log('Terminating (%s)' % sig.name) -- cgit v1.2.3