aboutsummaryrefslogtreecommitdiffstats
path: root/src/osmo-gsm-tester.py
blob: 15b192309dc3bed0ce91f916eff6e139750d1347 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
#!/usr/bin/env python3

# osmo-gsm-tester.py: main program file
#
# Copyright (C) 2016-2020 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
#
# Author: Neels Hofmeyr <neels@hofmeyr.de>
# Author: Pau Espin Pedrol <pespin@sysmocom.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

'''osmo-gsm-tester.py: invoke a single test run

Examples:

./osmo-gsm-tester.py -c doc/examples/2g_osmocom/main.conf ~/my_trial_dir/ -s osmo_trx
./osmo-gsm-tester.py -c doc/examples/2g_osmocom/main.conf ~/my_trial_dir/ -s sms_tests:dyn_ts+eu_band+bts_sysmo
./osmo-gsm-tester.py -c sysmocom/main.conf ~/my_trial_dir/ -s sms_tests/mo_mt_sms:bts_trx

(The names for test suites and scenarios used in these examples must be defined
by the osmo-gsm-tester configuration.)

A trial package contains binaries (usually built by a jenkins job) of software
to be run by Osmo-Gsm-Tester, like core network programs as well as binaries for
the various BTS models on a 2G network.

A test suite defines specific actions to be taken and verifies their outcome.
Such a test suite may leave certain aspects of a setup undefined, e.g. it may
be BTS model agnostic or does not care which voice codecs are chosen.

A test scenario completes the picture in that it defines which specific choices
shall be made to run a test suite. Any one test suite may thus run on any
number of different scenarios, e.g. to test various voice codecs.

Test scenarios may be combined. For example, one scenario may define a timeslot
configuration to use, while another scenario may define the voice codec
configuration.

There may still be aspects that are neither required by a test suite nor
strictly defined by a scenario, which will be resolved automatically, e.g. by
choosing the first available item that matches the other constraints.

A test run thus needs to define:
* A trial package containing built binaries
* A set of test suites, each with its combinations of scenarios
* A main configuration file specifying paths to other files containing sets of
  resources, default configurations and paths on where to find suites,
  scenarios, etc.

If no combination of suites and scenarios is provided, the default list of
suites will be run as defined in the osmo-gsm-tester configuration.

The scenarios and suites run for a given trial will be recorded in a trial
package's directory: Upon launch, a '$trial_dir/run.<date>' directory will be
created, which will collect logs and reports.
'''

import sys
import argparse
from signal import *
from osmo_gsm_tester import __version__
from osmo_gsm_tester.core import log
from osmo_gsm_tester.core import trial
from osmo_gsm_tester.core import suite
from osmo_gsm_tester.core import config
from osmo_gsm_tester.core.schema import generate_schemas

def sig_handler_cleanup(signum, frame):
    print("killed by signal %d" % signum)
    # This sys.exit() will raise a SystemExit base exception at the current
    # point of execution. Code must be prepared to clean system-wide resources
    # by using the "finally" section. This allows at the end 'atexit' hooks to
    # be called before exiting.
    sys.exit(1)

def main():

    for sig in (SIGINT, SIGTERM, SIGQUIT, SIGPIPE, SIGHUP):
        signal(sig, sig_handler_cleanup)

    parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
    # Note: since we're using RawTextHelpFormatter to keep nicely separate
    # paragraphs in the long help text, we unfortunately also need to take care
    # of line wraps in the shorter cmdline options help.
    # The line width here is what remains of screen width after the list of
    # options placed by ArgumentParser. That's unfortunately subject to change
    # and undefined, so when things change, just run a local
    # ./osmo-gsm-tester.py --help and try to keep everything in 80 chars width.
    # The help text is indented automatically, but line width is manual.
    # Using multi-line strings here -- doesn't look nice in the python flow but
    # is easiest to maintain.
    parser.add_argument('-V', '--version', action='store_true',
            help='Show version')
    parser.add_argument('-c', '--conf-path', dest='conf_path',
            help='''Specify main configuration file path''')
    parser.add_argument('trial_dir', nargs='?', default=None,
            help='Directory containing binaries to test')
    parser.add_argument('-s', '--suite-scenario', dest='suite_scenario', action='append',
            help='''A suite-scenarios combination
like suite:scenario+scenario''')
    parser.add_argument('-S', '--suites-file', dest='suites_file', action='append',
            default=[],
            help='''Read suites to run from a yml listing,
like default-suites.conf. The path is relative to
--conf-path.''')
    parser.add_argument('-t', '--test', dest='test', action='append',
            help='''Run only tests matching this name.
Any test name that contains the given string is run.
To get an exact match, prepend a "=" like
"-t =my_exact_name". The ".py" suffix is always
optional.''')
    parser.add_argument('-l', '--log-level', dest='log_level', choices=log.LEVEL_STRS.keys(),
            default=None,
            help='Set logging level for all categories (on stdout)')
    parser.add_argument('-T', '--traceback', dest='trace', action='store_true',
            help='Enable stdout logging of tracebacks')
    parser.add_argument('-R', '--source', dest='source', action='store_true',
            help='Enable stdout logging of source file')
    args = parser.parse_args()

    if args.version:
        print(__version__)
        exit(0)

    print('combinations:', repr(args.suite_scenario))
    print('trial:', repr(args.trial_dir))
    print('tests:', repr(args.test))

    # create a default log to stdout
    log.LogTarget().style(all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK), src=False)

    if args.log_level:
        log.set_all_levels(log.LEVEL_STRS.get(args.log_level))
    if args.trace:
        log.style_change(trace=True)
    if args.source:
        log.style_change(src=True)
    if args.conf_path:
        config.override_conf = args.conf_path

    if args.trial_dir is not None:
        trial_dir = args.trial_dir
    else:
        trial_dir = config.get_main_config_value(config.CFG_TRIAL_DIR)

    combination_strs = list(args.suite_scenario or [])

    for suites_file in args.suites_file:
        suites_file = config.main_config_path_to_abspath(suites_file)
        from_this_file = config.read(suites_file)
        print(('Running suites from %r:\n  ' % suites_file) + ('\n  '.join(from_this_file)))
        combination_strs.extend(from_this_file)

    if not combination_strs:
        combination_strs = config.read_config_file(config.CFG_DEFAULT_SUITES_CONF, if_missing_return=[])

        if combination_strs:
            print('Running default suites:\n  ' + ('\n  '.join(combination_strs)))
        else:
            print('Failed to load default suites (%r)' % config.get_main_config_value(config.DEFAULT_SUITES_CONF, fail_if_missing=False))


    if not combination_strs:
        raise RuntimeError('Need at least one suite:scenario to run')

    # Generate supported schemas dynamically from objects:
    generate_schemas()

    # make sure all suite:scenarios exist
    suite_scenarios = []
    for combination_str in combination_strs:
        suite_scenarios.append(suite.load_suite_scenario_str(combination_str))

    # pick tests and make sure they exist
    test_names = []
    for test_name in (args.test or []):
        found = False
        if test_name.startswith('=') and not test_name.endswith('.py'):
            test_name = test_name + '.py'
        for suite_scenario_str, suite_def, scenarios in suite_scenarios:
            for def_test_name in suite_def.test_basenames:
                if test_name.startswith('='):
                    match = test_name[1:] == def_test_name
                else:
                    match = test_name in def_test_name
                if match:
                    found = True
                    test_names.append(def_test_name)
        if not found:
            raise RuntimeError('No test found for %r' % test_name)
    if test_names:
        test_names = sorted(set(test_names))
        print(repr(test_names))

    with trial.Trial(trial_dir) as current_trial:
        current_trial.verify()
        for suite_scenario_str, suite_def, scenarios in suite_scenarios:
            current_trial.add_suite_run(suite_scenario_str, suite_def, scenarios)
        current_trial.run_suites(test_names)

        if current_trial.status != trial.Trial.PASS:
            return 1
        return 0

if __name__ == '__main__':
    rc = 2
    try:
        rc = main()
    except:
        # Tell the log to show the exception, then terminate the program with the exception anyway.
        # Since exceptions within test runs should be caught and evaluated, this is basically about
        # exceptions during command line parsing and such, so it's appropriate to abort immediately.
        log.log_exn()
        raise
    exit(rc)

# vim: expandtab tabstop=4 shiftwidth=4