aboutsummaryrefslogtreecommitdiffstats
path: root/src/osmo_gsm_tester/core/suite.py
blob: 938471c910cb8067a3aaff071023b5a7db0061ea (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
# osmo_gsm_tester: test suite
#
# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH
#
# Author: Neels Hofmeyr <neels@hofmeyr.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import time
import pprint
from . import config
from . import log
from . import util
from . import schema
from . import resource
from . import scenario
from . import test

class SuiteDefinition(log.Origin):
    '''A test suite reserves resources for a number of tests.
       Each test requires a specific number of modems, BTSs etc., which are
       reserved beforehand by a test suite. This way several test suites can be
       scheduled dynamically without resource conflicts arising halfway through
       the tests.'''

    CONF_FILENAME = 'suite.conf'

    def __init__(self, suite_dir):
        self._suite_name = os.path.basename(suite_dir)
        super().__init__(log.C_CNF, self._suite_name)
        self.suite_dir = suite_dir
        self.conf = None
        self._schema = None
        self.test_basenames = []
        self.load_test_basenames()
        self.read_conf()

    def read_conf(self):
        self.dbg('reading %s' % SuiteDefinition.CONF_FILENAME)
        if not os.path.isdir(self.suite_dir):
            raise RuntimeError('No such directory: %r' % self.suite_dir)
        self.conf = config.read(os.path.join(self.suite_dir,
                                             SuiteDefinition.CONF_FILENAME))
        # Drop schema part since it's dynamically defining content, makes no sense to validate it.
        self._schema = self.conf.pop('schema', {})
        # Add per-test 'timeout' attribute:
        d = {t.rstrip('.py'):{'timeout': schema.DURATION} for t in self.test_basenames}
        schema.combine(self._schema, d)
        # Convert config file format to proper schema format and register it:
        sdef = schema.config_to_schema_def(self._schema, "%s." % self._suite_name)
        schema.register_config_schema('suite', sdef)
        # Finally validate the file:
        schema.validate(self.conf, schema.get_all_schema())

    def load_test_basenames(self):
        for basename in sorted(os.listdir(self.suite_dir)):
            if not basename.endswith('.py'):
                continue
            self.test_basenames.append(basename)


class SuiteRun(log.Origin):
    UNKNOWN = 'UNKNOWN'
    PASS = 'PASS'
    FAIL = 'FAIL'

    def __init__(self, trial, suite_scenario_str, suite_definition, scenarios=[]):
        super().__init__(log.C_TST, suite_scenario_str)
        self.start_timestamp = None
        self.duration = None
        self.reserved_resources = None
        self._resource_requirements = None
        self._resource_modifiers = None
        self._config = None
        self._run_dir = None
        self._trial = trial
        self.definition = suite_definition
        self.scenarios = scenarios
        self.resources_pool = resource.ResourcesPool()
        self.status = SuiteRun.UNKNOWN
        self.load_tests()

    def suite_name(self):
        'Return name of suite without scenarios'
        return self.definition.name()

    def trial(self):
        return self._trial

    def load_tests(self):
        self.tests = []
        for test_basename in self.definition.test_basenames:
            test_specific_config = self.config_suite_specific().get(test_basename.rstrip('.py'), {})
            self.tests.append(test.Test(self, test_basename, test_specific_config))

    def mark_start(self):
        self.start_timestamp = time.time()
        self.duration = 0
        self.status = SuiteRun.UNKNOWN

    def combined(self, conf_name, replicate_times=True):
        log.dbg(combining=conf_name)
        log.ctx(combining_scenarios=conf_name)
        combination = self.definition.conf.get(conf_name, {})
        if replicate_times:
            combination = config.replicate_times(combination)
        log.dbg(definition_conf=combination)
        for sc in self.scenarios:
            log.ctx(combining_scenarios=conf_name, scenario=sc.name())
            c = sc.get(conf_name, {})
            if replicate_times:
                c = config.replicate_times(c)
            log.dbg(scenario=sc.name(), conf=c)
            if c is None:
                continue
            schema.combine(combination, c)
        return combination

    def get_run_dir(self):
        if self._run_dir is None:
            self._run_dir = util.Dir(self._trial.get_run_dir().new_dir(self.name()))
        return self._run_dir

    def resource_requirements(self):
        if self._resource_requirements is None:
            self._resource_requirements = self.combined('resources')
        return self._resource_requirements

    def resource_modifiers(self):
        if self._resource_modifiers is None:
            self._resource_modifiers = self.combined('modifiers')
        return self._resource_modifiers

    def config(self):
        if self._config is None:
            self._config = self.combined('config', False)
        return self._config

    def config_suite_specific(self):
        return self.config().get('suite', {}).get(self.suite_name(), {})

    def resource_pool(self):
        return self.resources_pool

    def reserve_resources(self):
        if self.reserved_resources:
            raise RuntimeError('Attempt to reserve resources twice for a SuiteRun')
        self.log('reserving resources in', self.resources_pool.state_dir, '...')
        self.reserved_resources = self.resources_pool.reserve(self, self.resource_requirements(), self.resource_modifiers())

    def get_reserved_resource(self, resource_class_str, specifics):
        return self.reserved_resources.get(resource_class_str, specifics=specifics)

    def run_tests(self, names=None):
        suite_libdir = os.path.join(self.definition.suite_dir, 'lib')
        try:
            log.large_separator(self._trial.name(), self.name(), sublevel=2)
            self.mark_start()
            util.import_path_prepend(suite_libdir)
            if not self.reserved_resources:
                self.reserve_resources()
            for t in self.tests:
                if names and not t.name() in names:
                    t.set_skip()
                    continue
                self.current_test = t
                t.run()
        except Exception:
            log.log_exn()
        except BaseException as e:
            # when the program is aborted by a signal (like Ctrl-C), escalate to abort all.
            self.err('SUITE RUN ABORTED: %s' % type(e).__name__)
            raise
        finally:
            self.free_resources()
            util.import_path_remove(suite_libdir)
            self.duration = time.time() - self.start_timestamp

            passed, skipped, failed, errors = self.count_test_results()
            # if no tests ran, count it as failure
            if passed and not failed and not errors:
                self.status = SuiteRun.PASS
            else:
                self.status = SuiteRun.FAIL

            log.large_separator(self._trial.name(), self.name(), self.status, sublevel=2, space_above=False)

    def passed(self):
        return self.status == SuiteRun.PASS

    def count_test_results(self):
        passed = 0
        skipped = 0
        failed = 0
        errors = 0
        for t in self.tests:
            if t.status == test.Test.SKIP:
                skipped += 1
            elif t.status == test.Test.PASS:
                passed += 1
            elif t.status == test.Test.FAIL:
                failed += 1
            else: # error, could not run
                errors += 1
        return (passed, skipped, failed, errors)

    def free_resources(self):
        if self.reserved_resources is None:
            return
        self.reserved_resources.free()

    def resource_status_str(self):
        return '\n'.join(('',
            'SUITE RUN: %s' % self.origin_id(),
            'ASKED FOR:', pprint.pformat(self._resource_requirements),
            'RESERVED COUNT:', pprint.pformat(self.reserved_resources.counts()),
            'RESOURCES STATE:', repr(self.reserved_resources)))

loaded_suite_definitions = {}

def load(suite_name):
    global loaded_suite_definitions

    suite = loaded_suite_definitions.get(suite_name)
    if suite is not None:
        return suite

    suites_dirs = config.get_suites_dirs()
    suite_dir = None
    found = False
    for d in suites_dirs:
        suite_dir = d.child(suite_name)
        if d.exists(suite_name) and d.isdir(suite_name):
            found = True
            break
    if not found:
        raise RuntimeError('Suite not found: %r in %r' % (suite_name, suites_dirs))

    suite_def = SuiteDefinition(suite_dir)
    loaded_suite_definitions[suite_name] = suite_def
    return suite_def

def parse_suite_scenario_str(suite_scenario_str):
    tokens = suite_scenario_str.split(':')
    if len(tokens) > 2:
        raise RuntimeError('invalid combination string: %r' % suite_scenario_str)

    suite_name = tokens[0]
    if len(tokens) <= 1:
        scenario_names = []
    else:
        scenario_names = tokens[1].split('+')

    return suite_name, scenario_names

def load_suite_scenario_str(suite_scenario_str):
    suite_name, scenario_names = parse_suite_scenario_str(suite_scenario_str)
    suite = load(suite_name)
    scenarios = [scenario.get_scenario(scenario_name, schema.get_all_schema()) for scenario_name in scenario_names]
    return (suite_scenario_str, suite, scenarios)

# vim: expandtab tabstop=4 shiftwidth=4