qemu-qemu-1/tests/qemu-iotests/check

#!/usr/bin/env python3
#
# Configure environment and run group of tests in it.
#
# Copyright (c) 2020-2021 Virtuozzo International GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import sys
import argparse
import shutil
from pathlib import Path
import warnings

from findtests import TestFinder
from testenv import TestEnv
from testrunner import TestRunner

def get_default_path(follow_link=False):
    """
    Try to automagically figure out the path we are running from.
    """
    # called from the build tree?
    if os.path.islink(sys.argv[0]):
        if follow_link:
            return os.path.dirname(os.readlink(sys.argv[0]))
        else:
            return os.path.dirname(os.path.abspath(sys.argv[0]))
    else:  # or source tree?
        return os.getcwd()

def make_argparser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
        description="Test run options",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    p.add_argument('-n', '--dry-run', action='store_true',
                   help='show me, do not run tests')
    p.add_argument('-j', dest='jobs', type=int, default=1,
                   help='run tests in multiple parallel jobs')

    p.add_argument('-d', dest='debug', action='store_true', help='debug')
    p.add_argument('-p', dest='print', action='store_true',
                   help='redirects qemu\'s stdout and stderr to '
                        'the test output')
    p.add_argument('-gdb', action='store_true',
                   help="start gdbserver with $GDB_OPTIONS options "
                        "('localhost:12345' if $GDB_OPTIONS is empty)")
    p.add_argument('-valgrind', action='store_true',
                   help='use valgrind, sets VALGRIND_QEMU environment '
                        'variable')

    p.add_argument('-misalign', action='store_true',
                   help='misalign memory allocations')
    p.add_argument('--color', choices=['on', 'off', 'auto'],
                   default='auto', help="use terminal colors. The default "
                   "'auto' value means use colors if terminal stdout detected")
    p.add_argument('-tap', action='store_true',
                   help='produce TAP output')

    g_env = p.add_argument_group('test environment options')
    mg = g_env.add_mutually_exclusive_group()
    # We don't set default for cachemode, as we need to distinguish default
    # from user input later.
    mg.add_argument('-nocache', dest='cachemode', action='store_const',
                    const='none', help='set cache mode "none" (O_DIRECT), '
                    'sets CACHEMODE environment variable')
    mg.add_argument('-c', dest='cachemode',
                    help='sets CACHEMODE environment variable')

    g_env.add_argument('-i', dest='aiomode', default='threads',
                       help='sets AIOMODE environment variable')

    p.set_defaults(imgproto='file')

    format_list = ['raw', 'bochs', 'cloop', 'parallels', 'qcow', 'qcow2',
                   'qed', 'vdi', 'vpc', 'vhdx', 'vmdk', 'luks', 'dmg', 'vvfat']
    g_fmt = p.add_argument_group(
        '  image format options',
        'The following options set the IMGFMT environment variable. '
        'At most one choice is allowed, default is "raw"')
    mg = g_fmt.add_mutually_exclusive_group()
    for fmt in format_list:
        mg.add_argument('-' + fmt, dest='imgfmt', action='store_const',
                        const=fmt, help=f'test {fmt}')

    protocol_list = ['file', 'rbd', 'nbd', 'ssh', 'nfs', 'fuse']
    g_prt = p.add_argument_group(
        '  image protocol options',
        'The following options set the IMGPROTO environment variable. '
        'At most one choice is allowed, default is "file"')
    mg = g_prt.add_mutually_exclusive_group()
    for prt in protocol_list:
        mg.add_argument('-' + prt, dest='imgproto', action='store_const',
                        const=prt, help=f'test {prt}')

    g_bash = p.add_argument_group('bash tests options',
                                  'The following options are ignored by '
                                  'python tests.')
    # TODO: make support for the following options in iotests.py
    g_bash.add_argument('-o', dest='imgopts',
                        help='options to pass to qemu-img create/convert, '
                        'sets IMGOPTS environment variable')

    g_sel = p.add_argument_group('test selecting options',
                                 'The following options specify test set '
                                 'to run.')
    g_sel.add_argument('-g', '--groups', metavar='group1,...',
                       help='include tests from these groups')
    g_sel.add_argument('-x', '--exclude-groups', metavar='group1,...',
                       help='exclude tests from these groups')
    g_sel.add_argument('--start-from', metavar='TEST',
                       help='Start from specified test: make sorted sequence '
                       'of tests as usual and then drop tests from the first '
                       'one to TEST (not inclusive). This may be used to '
                       'rerun failed ./check command, starting from the '
                       'middle of the process.')
    g_sel.add_argument('tests', metavar='TEST_FILES', nargs='*',
                       help='tests to run, or "--" followed by a command')
    g_sel.add_argument('--build-dir', default=get_default_path(),
                       help='Path to iotests build directory')
    g_sel.add_argument('--source-dir',
                       default=get_default_path(follow_link=True),
                       help='Path to iotests build directory')

    return p


def dry_run_list(test_dir, imgfmt, testlist):
    for t in testlist:
        if not imgfmt:
            print('\n'.join([os.path.basename(t)]))
            continue
        # If a format has been given, we look for the "supported_fmt"
        # and the "unsupported_fmt" lines in the test and try to find out
        # whether the format is supported or not. This is only heuristics
        # (it can e.g. fail if the "unsupported_fmts" and "supported_fmts"
        # statements are in the same line), but it should be good enough
        # to get a proper list for "make check-block"
        with open(os.path.join(test_dir, t), 'r', encoding='utf-8') as fh:
            supported = True
            check_next_line = False
            sd = "[ \t'\"]"           # Start delimiter
            ed = "([ \t'\"]|$)"       # End delimiter
            for line in fh:
                if 'unsupported_fmt' in line:
                    if re.search(sd + imgfmt + ed, line):
                        supported = False
                        break
                elif 'supported_fmt' in line or check_next_line:
                    if re.search(sd + 'generic' + ed, line):
                        continue      # Might be followed by "unsupported" line
                    supported = re.search(sd + imgfmt + ed, line)
                    check_next_line = not ']' in line and \
                            ('supported_fmts=[' in line or check_next_line)
                    if supported or not check_next_line:
                        break
            if supported:
                print('\n'.join([os.path.basename(t)]))


if __name__ == '__main__':
    warnings.simplefilter("default")
    os.environ["PYTHONWARNINGS"] = "default"

    args = make_argparser().parse_args()

    image_format = args.imgfmt or 'raw'

    env = TestEnv(source_dir=args.source_dir,
                  build_dir=args.build_dir,
                  imgfmt=image_format, imgproto=args.imgproto,
                  aiomode=args.aiomode, cachemode=args.cachemode,
                  imgopts=args.imgopts, misalign=args.misalign,
                  debug=args.debug, valgrind=args.valgrind,
                  gdb=args.gdb, qprint=args.print,
                  dry_run=args.dry_run)

    if len(sys.argv) > 1 and sys.argv[-len(args.tests)-1] == '--':
        if not args.tests:
            sys.exit("missing command after '--'")
        cmd = args.tests
        env.print_env()
        exec_pathstr = shutil.which(cmd[0])
        if exec_pathstr is None:
            sys.exit('command not found: ' + cmd[0])
        exec_path = Path(exec_pathstr).resolve()
        cmd[0] = str(exec_path)
        full_env = env.prepare_subprocess(cmd)
        os.chdir(exec_path.parent)
        os.execve(cmd[0], cmd, full_env)

    testfinder = TestFinder(test_dir=env.source_iotests)

    groups = args.groups.split(',') if args.groups else None
    x_groups = args.exclude_groups.split(',') if args.exclude_groups else None

    group_local = os.path.join(env.source_iotests, 'group.local')
    if os.path.isfile(group_local):
        try:
            testfinder.add_group_file(group_local)
        except ValueError as e:
            sys.exit(f"Failed to parse group file '{group_local}': {e}")

    try:
        tests = testfinder.find_tests(groups=groups, exclude_groups=x_groups,
                                      tests=args.tests,
                                      start_from=args.start_from)
        if not tests:
            raise ValueError('No tests selected')
    except ValueError as e:
        sys.exit(str(e))

    if args.dry_run:
        with env:
            dry_run_list(env.source_iotests, args.imgfmt, tests)
    else:
        with TestRunner(env, tap=args.tap,
                        color=args.color) as tr:
            paths = [os.path.join(env.source_iotests, t) for t in tests]
            ok = tr.run_tests(paths, args.jobs)
            if not ok:
                sys.exit(1)