From b1526b9979ae1b04819cc6130a7fb36e97c82811 Mon Sep 17 00:00:00 2001 From: Pau Espin Pedrol Date: Tue, 22 May 2018 20:32:30 +0200 Subject: Add option to expect bts/pcu failures and respawn its processes Some tests may want to reproduce some scenarios in which it is expected that a BTS process is stopped, for instance if the BSC link is dropped. Provide a keepalive parameter to start() for bts and pcu objects to inform suite that failures are expected and that it should keep them alive in case that ocurrs by respawning the BTS process. Change-Id: Ia2a7539f9fad457125ac9b60a52a52999e885ba8 --- src/osmo_gsm_tester/bts.py | 5 +++-- src/osmo_gsm_tester/bts_nanobts.py | 2 +- src/osmo_gsm_tester/bts_osmo.py | 4 ++-- src/osmo_gsm_tester/bts_osmotrx.py | 18 +++++++++--------- src/osmo_gsm_tester/bts_sysmo.py | 8 ++++---- src/osmo_gsm_tester/pcu.py | 4 ++-- src/osmo_gsm_tester/pcu_osmo.py | 8 ++++---- src/osmo_gsm_tester/pcu_sysmo.py | 8 ++++---- src/osmo_gsm_tester/process.py | 7 +++++++ src/osmo_gsm_tester/suite.py | 31 +++++++++++++++++++++---------- 10 files changed, 57 insertions(+), 38 deletions(-) (limited to 'src/osmo_gsm_tester') diff --git a/src/osmo_gsm_tester/bts.py b/src/osmo_gsm_tester/bts.py index f59cff3..ca33eb4 100644 --- a/src/osmo_gsm_tester/bts.py +++ b/src/osmo_gsm_tester/bts.py @@ -139,8 +139,9 @@ class Bts(log.Origin, metaclass=ABCMeta): # PUBLIC (test API included) ################### @abstractmethod - def start(self): - 'Starts BTS proccess and sets self.proc_bts with an object of Process interface' + def start(self, keepalive=False): + '''Starts BTS. If keepalive is set, it will expect internal issues and + respawn related processes when detected''' pass @abstractmethod diff --git a/src/osmo_gsm_tester/bts_nanobts.py b/src/osmo_gsm_tester/bts_nanobts.py index d631f1c..29a8ac5 100644 --- a/src/osmo_gsm_tester/bts_nanobts.py +++ b/src/osmo_gsm_tester/bts_nanobts.py @@ -80,7 +80,7 @@ class NanoBts(bts.Bts): # PUBLIC (test API included) ################### - def start(self): + def start(self, keepalive=False): if self.conf.get('ipa_unit_id') is None: raise log.Error('No attribute %s provided in conf!' % attr) self.run_dir = util.Dir(self.suite_run.get_test_run_dir().new_dir(self.name())) diff --git a/src/osmo_gsm_tester/bts_osmo.py b/src/osmo_gsm_tester/bts_osmo.py index 21ae135..b9b7fef 100644 --- a/src/osmo_gsm_tester/bts_osmo.py +++ b/src/osmo_gsm_tester/bts_osmo.py @@ -57,7 +57,7 @@ class OsmoBts(bts.Bts, metaclass=ABCMeta): # PUBLIC (test API included) ################### @abstractmethod - def start(self): + def start(self, keepalive=False): # coming from bts.Bts, we forward the implementation to children. pass @@ -108,6 +108,6 @@ class OsmoBtsMainUnit(OsmoBts, metaclass=ABCMeta): # PUBLIC (test API included) ################### @abstractmethod - def start(self): + def start(self, keepalive=False): # coming from bts.Bts, we forward the implementation to children. pass diff --git a/src/osmo_gsm_tester/bts_osmotrx.py b/src/osmo_gsm_tester/bts_osmotrx.py index 9f76194..b9310f8 100644 --- a/src/osmo_gsm_tester/bts_osmotrx.py +++ b/src/osmo_gsm_tester/bts_osmotrx.py @@ -51,7 +51,7 @@ class OsmoBtsTrx(bts_osmo.OsmoBtsMainUnit): def launch_trx_enabled(self): return util.str2bool(self.conf.get('launch_trx')) - def launch_process(self, binary_name, *args): + def launch_process(self, keepalive, binary_name, *args): binary = os.path.abspath(self.inst.child('bin', binary_name)) run_dir = self.run_dir.new_dir(binary_name) if not os.path.isfile(binary): @@ -59,7 +59,7 @@ class OsmoBtsTrx(bts_osmo.OsmoBtsMainUnit): proc = process.Process(binary_name, run_dir, (binary,) + args, env=self.env) - self.suite_run.remember_to_stop(proc) + self.suite_run.remember_to_stop(proc, keepalive) proc.launch() return proc @@ -99,7 +99,7 @@ class OsmoBtsTrx(bts_osmo.OsmoBtsMainUnit): ################### # PUBLIC (test API included) ################### - def start(self): + def start(self, keepalive=False): if self.bsc is None: raise RuntimeError('BTS needs to be added to a BSC or NITB before it can be started') self.suite_run.poll() @@ -110,7 +110,7 @@ class OsmoBtsTrx(bts_osmo.OsmoBtsMainUnit): if self.launch_trx_enabled(): self.trx = OsmoTrx(self.suite_run, self.conf, self.trx_remote_ip(), self.remote_addr()) - self.trx.start() + self.trx.start(keepalive) self.log('Waiting for osmo-trx to start up...') MainLoop.wait(self, self.trx.trx_ready) @@ -120,7 +120,7 @@ class OsmoBtsTrx(bts_osmo.OsmoBtsMainUnit): raise RuntimeError('No lib/ in %r' % self.inst) self.env = { 'LD_LIBRARY_PATH': util.prepend_library_path(lib) } - self.proc_bts = self.launch_process(OsmoBtsTrx.BIN_BTS_TRX, '-r', '1', + self.proc_bts = self.launch_process(keepalive, OsmoBtsTrx.BIN_BTS_TRX, '-r', '1', '-c', os.path.abspath(self.config_file), '-i', self.bsc.addr()) self.suite_run.poll() @@ -163,17 +163,17 @@ class OsmoTrx(log.Origin): self.dbg(r) f.write(r) - def start(self): + def start(self, keepalive=False): self.run_dir = util.Dir(self.suite_run.get_test_run_dir().new_dir(self.name())) self.configure() self.inst = util.Dir(os.path.abspath(self.suite_run.trial.get_inst('osmo-trx'))) lib = self.inst.child('lib') self.env = { 'LD_LIBRARY_PATH': util.prepend_library_path(lib) } - self.proc_trx = self.launch_process(OsmoTrx.BIN_TRX, '-x', + self.proc_trx = self.launch_process(keepalive, OsmoTrx.BIN_TRX, '-x', '-j', self.listen_ip, '-i', self.bts_ip, '-C', os.path.abspath(self.config_file)) - def launch_process(self, binary_name, *args): + def launch_process(self, keepalive, binary_name, *args): binary = os.path.abspath(self.inst.child('bin', binary_name)) run_dir = self.run_dir.new_dir(binary_name) if not os.path.isfile(binary): @@ -181,7 +181,7 @@ class OsmoTrx(log.Origin): proc = process.Process(binary_name, run_dir, (binary,) + args, env=self.env) - self.suite_run.remember_to_stop(proc) + self.suite_run.remember_to_stop(proc, keepalive) proc.launch() return proc diff --git a/src/osmo_gsm_tester/bts_sysmo.py b/src/osmo_gsm_tester/bts_sysmo.py index d0f6ff3..65c9279 100644 --- a/src/osmo_gsm_tester/bts_sysmo.py +++ b/src/osmo_gsm_tester/bts_sysmo.py @@ -54,9 +54,9 @@ class SysmoBts(bts_osmo.OsmoBts): log.ctx(proc) raise log.Error('Exited in error') - def launch_remote(self, name, popen_args, remote_cwd=None): + def launch_remote(self, name, popen_args, remote_cwd=None, keepalive=False): proc = self._process_remote(name, popen_args, remote_cwd) - self.suite_run.remember_to_stop(proc) + self.suite_run.remember_to_stop(proc, keepalive) proc.launch() return proc @@ -110,7 +110,7 @@ class SysmoBts(bts_osmo.OsmoBts): ################### # PUBLIC (test API included) ################### - def start(self): + def start(self, keepalive=False): if self.bsc is None: raise RuntimeError('BTS needs to be added to a BSC or NITB before it can be started') log.log('Starting sysmoBTS to connect to', self.bsc) @@ -151,6 +151,6 @@ class SysmoBts(bts_osmo.OsmoBts): if self._direct_pcu_enabled(): args += ('-M',) - self.proc_bts = self.launch_remote('osmo-bts-sysmo', args, remote_cwd=remote_run_dir) + self.proc_bts = self.launch_remote('osmo-bts-sysmo', args, remote_cwd=remote_run_dir, keepalive=keepalive) # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/src/osmo_gsm_tester/pcu.py b/src/osmo_gsm_tester/pcu.py index 97d0b92..0260296 100644 --- a/src/osmo_gsm_tester/pcu.py +++ b/src/osmo_gsm_tester/pcu.py @@ -42,7 +42,7 @@ class Pcu(log.Origin, metaclass=ABCMeta): ################### @abstractmethod - def start(self): + def start(self, keepalive=False): """Start the PCU. Must be implemented by subclass.""" pass @@ -54,7 +54,7 @@ class PcuDummy(Pcu): def __init__(self, suite_run, bts, conf): super().__init__(suite_run, bts, conf, 'PcuDummy') - def start(self): + def start(self, keepalive=False): pass # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/src/osmo_gsm_tester/pcu_osmo.py b/src/osmo_gsm_tester/pcu_osmo.py index 6ab97de..50ae134 100644 --- a/src/osmo_gsm_tester/pcu_osmo.py +++ b/src/osmo_gsm_tester/pcu_osmo.py @@ -34,7 +34,7 @@ class OsmoPcu(pcu.Pcu): self.conf = conf self.env = {} - def start(self): + def start(self, keepalive=False): self.run_dir = util.Dir(self.suite_run.get_test_run_dir().new_dir(self.name())) self.configure() @@ -44,12 +44,12 @@ class OsmoPcu(pcu.Pcu): raise RuntimeError('No lib/ in %r' % self.inst) self.env = { 'LD_LIBRARY_PATH': util.prepend_library_path(lib) } - self.launch_process(OsmoPcu.BIN_PCU, '-r', '1', + self.launch_process(keepalive, OsmoPcu.BIN_PCU, '-r', '1', '-c', os.path.abspath(self.config_file), '-i', self.bts.bsc.addr()) self.suite_run.poll() - def launch_process(self, binary_name, *args): + def launch_process(self, keepalive, binary_name, *args): binary = os.path.abspath(self.inst.child('bin', binary_name)) run_dir = self.run_dir.new_dir(binary_name) if not os.path.isfile(binary): @@ -57,7 +57,7 @@ class OsmoPcu(pcu.Pcu): proc = process.Process(binary_name, run_dir, (binary,) + args, env=self.env) - self.suite_run.remember_to_stop(proc) + self.suite_run.remember_to_stop(proc, keepalive) proc.launch() return proc diff --git a/src/osmo_gsm_tester/pcu_sysmo.py b/src/osmo_gsm_tester/pcu_sysmo.py index 675de50..b97852a 100644 --- a/src/osmo_gsm_tester/pcu_sysmo.py +++ b/src/osmo_gsm_tester/pcu_sysmo.py @@ -43,7 +43,7 @@ class OsmoPcuSysmo(log.Origin): self.remote_env = {} self.remote_user = 'root' - def start(self): + def start(self, keepalive=False): self.run_dir = util.Dir(self.suite_run.get_test_run_dir().new_dir(self.name())) self.configure() @@ -75,7 +75,7 @@ class OsmoPcuSysmo(log.Origin): ('LD_LIBRARY_PATH=%s' % remote_lib, remote_binary, '-c', remote_config_file, '-r', '1', '-i', self.sysmobts.bsc.addr()), - remote_cwd=remote_run_dir) + remote_cwd=remote_run_dir, keepalive=keepalive) def _process_remote(self, name, popen_args, remote_cwd=None): run_dir = self.run_dir.new_dir(name) @@ -90,9 +90,9 @@ class OsmoPcuSysmo(log.Origin): log.ctx(proc) raise log.Error('Exited in error') - def launch_remote(self, name, popen_args, remote_cwd=None): + def launch_remote(self, name, popen_args, remote_cwd=None, keepalive=False): proc = self._process_remote(name, popen_args, remote_cwd) - self.suite_run.remember_to_stop(proc) + self.suite_run.remember_to_stop(proc, keepalive) proc.launch() def run_local(self, name, popen_args): diff --git a/src/osmo_gsm_tester/process.py b/src/osmo_gsm_tester/process.py index 477a096..c13ded0 100644 --- a/src/osmo_gsm_tester/process.py +++ b/src/osmo_gsm_tester/process.py @@ -79,6 +79,13 @@ class Process(log.Origin): self.set_name(self.name_str, pid=self.process_obj.pid) self.log('Launched') + def respawn(self): + self.dbg('respawn') + assert not self.is_running() + self.result = None + self.killed = None + self.launch() + def _poll_termination(self, time_to_wait_for_term=5): wait_step = 0.001 waited_time = 0 diff --git a/src/osmo_gsm_tester/suite.py b/src/osmo_gsm_tester/suite.py index 76cd248..618a39b 100644 --- a/src/osmo_gsm_tester/suite.py +++ b/src/osmo_gsm_tester/suite.py @@ -230,19 +230,27 @@ class SuiteRun(log.Origin): skipped += 1 return (passed, skipped, failed) - def remember_to_stop(self, process): + def remember_to_stop(self, process, respawn=False): + '''Ask suite to monitor and manage lifecycle of the Process object. If a + process managed by suite finishes before cleanup time, the current test + will be marked as FAIL and end immediatelly. If respwan=True, then suite + will respawn() the process instead.''' if self._processes is None: self._processes = [] - self._processes.insert(0, process) + self._processes.insert(0, (process, respawn)) def stop_processes(self): while self._processes: - self._processes.pop().terminate() + process, respawn = self._processes.pop() + process.terminate() def stop_process(self, process): 'Remove process from monitored list and stop it' - self._processes.remove(process) - process.terminate() + for proc_respawn in self._processes: + proc, respawn = proc_respawn + if proc == process: + self._processes.remove(proc_respawn) + proc.terminate() def free_resources(self): if self.reserved_resources is None: @@ -351,12 +359,15 @@ class SuiteRun(log.Origin): def poll(self): if self._processes: - for process in self._processes: + for process, respawn in self._processes: if process.terminated(): - process.log_stdout_tail() - process.log_stderr_tail() - log.ctx(process) - raise log.Error('Process ended prematurely: %s' % process.name()) + if respawn == True: + process.respawn() + else: + process.log_stdout_tail() + process.log_stderr_tail() + log.ctx(process) + raise log.Error('Process ended prematurely: %s' % process.name()) def prompt(self, *msgs, **msg_details): 'ask for user interaction. Do not use in tests that should run automatically!' -- cgit v1.2.3