mirror of
https://github.com/qemu/qemu.git
synced 2026-02-04 05:35:39 +00:00
da949d495d
When reviewing tracetool patches it is often very unclear what the expected output will be for the generated backends. Compounding this is that a default build will only enable the 'log' trace backend, so developers won't see generated code for other backends without making a special effort. Some backends are also platform specific, so can't be enabled in QEMU builds, even though tracetool could generate the code. To address this, introduce a test suite for tracetool which is conceptually similar to the qapi-schema test. It is a simple python program that runs tracetool and compares the actual output to historical reference output kept in git. The test directly emits TAP format logs for ease of integration with meson. This can be run with make check-tracetool to make it easier for developers changing generated output, the sample expected content can be auto-recreated QEMU_TEST_REGENERATE=1 make check-tracetool and the changes reviewed and added to the commit. This will also assist reviewers interpreting the change. Developers are reminded of this in the test output on failure: $ make check-tracetool 1/6 qemu:tracetool / dtrace OK 0.14s 2/6 qemu:tracetool / ftrace FAIL 0.06s exit status 1 ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― 1..2 ok 1 - ftrace.c # not ok 1 - ftrace.h (set QEMU_TEST_REGENERATE=1 to recreate reference output if tracetool generator was intentionally changed) ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――― 3/6 qemu:tracetool / log OK 0.06s 4/6 qemu:tracetool / simple OK 0.06s 5/6 qemu:tracetool / syslog OK 0.06s 6/6 qemu:tracetool / ust OK 0.11s Summary of Failures: 2/6 qemu:tracetool / ftrace FAIL 0.06s exit status 1 Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> Message-id: 20250916081638.764020-6-berrange@redhat.com Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
130 lines
4.5 KiB
Makefile
130 lines
4.5 KiB
Makefile
# -*- Mode: makefile -*-
|
|
|
|
.PHONY: check-help
|
|
check-help:
|
|
@echo "Regression testing targets:"
|
|
@echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
|
|
@echo " $(MAKE) bench Run speed tests"
|
|
@echo
|
|
@echo "Individual test suites:"
|
|
@echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
|
|
@echo " $(MAKE) check-qtest Run qtest tests"
|
|
@echo " $(MAKE) check-functional Run python-based functional tests"
|
|
@echo " $(MAKE) check-functional-TARGET Run functional tests for a given target"
|
|
@echo " $(MAKE) check-unit Run qobject tests"
|
|
@echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
|
|
@echo " $(MAKE) check-tracetool Run tracetool generator tests"
|
|
@echo " $(MAKE) check-block Run block tests"
|
|
ifneq ($(filter $(all-check-targets), check-softfloat),)
|
|
@echo " $(MAKE) check-tcg Run TCG tests"
|
|
@echo " $(MAKE) check-softfloat Run FPU emulation tests"
|
|
endif
|
|
@echo
|
|
@echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report"
|
|
@echo " $(MAKE) check-venv Creates a Python venv for tests"
|
|
@echo " $(MAKE) check-clean Clean the tests and related data"
|
|
@echo
|
|
@echo "The following are useful for CI builds"
|
|
@echo " $(MAKE) check-build Build most test binaries"
|
|
@echo
|
|
@echo
|
|
@echo "The variable SPEED can be set to control the gtester speed setting."
|
|
@echo "Default options are -k and (for $(MAKE) V=1) --verbose; they can be"
|
|
@echo "changed with variable GTESTER_OPTIONS."
|
|
|
|
ifneq ($(wildcard config-host.mak),)
|
|
export SRC_PATH
|
|
|
|
SPEED = quick
|
|
|
|
# Per guest TCG tests
|
|
BUILD_TCG_TARGET_RULES=$(patsubst %,build-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
CLEAN_TCG_TARGET_RULES=$(patsubst %,clean-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
DISTCLEAN_TCG_TARGET_RULES=$(patsubst %,distclean-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
RUN_TCG_TARGET_RULES=$(patsubst %,run-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
|
|
$(foreach TARGET,$(TCG_TESTS_TARGETS), \
|
|
$(eval $(BUILD_DIR)/tests/tcg/config-$(TARGET).mak: config-host.mak))
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=build-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=build-tcg-tests-%): build-tcg-tests-%: $(BUILD_DIR)/tests/tcg/config-%.mak
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS), \
|
|
"BUILD","$* guest-tests")
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=run-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=run-tcg-tests-%): run-tcg-tests-%: build-tcg-tests-%
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) SPEED=$(SPEED) run, \
|
|
"RUN", "$* guest-tests")
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=clean-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=clean-tcg-tests-%): clean-tcg-tests-%:
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) clean, \
|
|
"CLEAN", "$* guest-tests")
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%): distclean-tcg-tests-%:
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) distclean, \
|
|
"CLEAN", "$* guest-tests")
|
|
|
|
.PHONY: build-tcg
|
|
build-tcg: $(BUILD_TCG_TARGET_RULES)
|
|
|
|
.PHONY: check-tcg
|
|
.ninja-goals.check-tcg = all test-plugins
|
|
check-tcg: $(RUN_TCG_TARGET_RULES)
|
|
|
|
.PHONY: clean-tcg
|
|
clean-tcg: $(CLEAN_TCG_TARGET_RULES)
|
|
|
|
.PHONY: distclean-tcg
|
|
distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES)
|
|
|
|
# Python venv for running tests
|
|
|
|
.PHONY: check-venv
|
|
|
|
# Build up our target list from the filtered list of ninja targets
|
|
TARGETS=$(patsubst libqemu-%.a, %, $(filter libqemu-%.a, $(ninja-targets)))
|
|
|
|
TESTS_VENV_TOKEN=$(BUILD_DIR)/pyvenv/tests.group
|
|
|
|
quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
|
|
$(PYTHON) -m pip -q --disable-pip-version-check $1, \
|
|
"VENVPIP","$1")
|
|
|
|
$(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml
|
|
$(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
|
|
$(MKVENV_ENSUREGROUP) $< testdeps
|
|
$(call quiet-command, touch $@)
|
|
|
|
check-venv: $(TESTS_VENV_TOKEN)
|
|
|
|
FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS)))
|
|
.PHONY: $(FUNCTIONAL_TARGETS)
|
|
$(FUNCTIONAL_TARGETS):
|
|
@$(MAKE) SPEED=thorough $(subst -functional,-func,$@)
|
|
|
|
.PHONY: check-functional
|
|
check-functional:
|
|
@$(NINJA) precache-functional
|
|
@QEMU_TEST_NO_DOWNLOAD=1 $(MAKE) SPEED=thorough check-func check-func-quick
|
|
|
|
# Consolidated targets
|
|
|
|
.PHONY: check check-clean
|
|
check:
|
|
|
|
check-build: run-ninja
|
|
|
|
check-clean:
|
|
rm -rf $(BUILD_DIR)/tests/functional
|
|
|
|
clean: check-clean clean-tcg
|
|
distclean: distclean-tcg
|
|
|
|
endif
|