diff --git a/MAINTAINERS b/MAINTAINERS index 2b5b581e17..9920ab24f1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2520,6 +2520,13 @@ F: include/system/rng*.h F: backends/rng*.c F: tests/qtest/virtio-rng-test.c +virtio-rtc +M: Kuan-Wei Chiu +S: Maintained +F: hw/virtio/virtio-rtc.c +F: hw/virtio/virtio-rtc-pci.c +F: include/hw/virtio/virtio-rtc.h + virtio-nsm M: Alexander Graf M: Dorjoy Chowdhury @@ -3192,8 +3199,8 @@ F: tests/qtest/migration/cpr* F: docs/devel/migration/CPR.rst Compute Express Link -M: Jonathan Cameron -R: Fan Ni +M: Jonathan Cameron +L: linux-cxl@vger.kernel.org S: Supported F: hw/cxl/ F: hw/mem/cxl_type3.c @@ -4038,6 +4045,7 @@ M: Alejandro Jimenez R: Sairaj Kodilkar S: Supported F: hw/i386/amd_iommu* +F: tests/qtest/amd-iommu-test.c OpenSBI Firmware M: Alistair Francis diff --git a/backends/cryptodev-vhost.c b/backends/cryptodev-vhost.c index 943680a23a..c6069f4e5b 100644 --- a/backends/cryptodev-vhost.c +++ b/backends/cryptodev-vhost.c @@ -60,7 +60,6 @@ cryptodev_vhost_init( crypto->cc = options->cc; - crypto->dev.protocol_features = 0; crypto->backend = -1; /* vhost-user needs vq_index to initiate a specific queue pair */ @@ -152,7 +151,6 @@ vhost_set_vring_enable(CryptoDevBackendClient *cc, { CryptoDevBackendVhost *crypto = cryptodev_get_vhost(cc, b, queue); - const VhostOps *vhost_ops; cc->vring_enable = enable; @@ -160,12 +158,7 @@ vhost_set_vring_enable(CryptoDevBackendClient *cc, return 0; } - vhost_ops = crypto->dev.vhost_ops; - if (vhost_ops->vhost_set_vring_enable) { - return vhost_ops->vhost_set_vring_enable(&crypto->dev, enable); - } - - return 0; + return vhost_dev_set_vring_enable(&crypto->dev, enable); } int cryptodev_vhost_start(VirtIODevice *dev, int total_queues) diff --git a/backends/iommufd.c b/backends/iommufd.c index 410b044370..cfde6f2b2c 100644 --- a/backends/iommufd.c +++ b/backends/iommufd.c @@ -539,23 +539,24 @@ bool iommufd_backend_alloc_veventq(IOMMUFDBackend *be, uint32_t viommu_id, } bool host_iommu_device_iommufd_attach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, - uint32_t hwpt_id, Error **errp) -{ - HostIOMMUDeviceIOMMUFDClass *hiodic = - HOST_IOMMU_DEVICE_IOMMUFD_GET_CLASS(hiodi); - - g_assert(hiodic->attach_hwpt); - return hiodic->attach_hwpt(hiodi, hwpt_id, errp); -} - -bool host_iommu_device_iommufd_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, + uint32_t pasid, uint32_t hwpt_id, Error **errp) { HostIOMMUDeviceIOMMUFDClass *hiodic = HOST_IOMMU_DEVICE_IOMMUFD_GET_CLASS(hiodi); + g_assert(hiodic->attach_hwpt); + return hiodic->attach_hwpt(hiodi, pasid, hwpt_id, errp); +} + +bool host_iommu_device_iommufd_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, + uint32_t pasid, Error **errp) +{ + HostIOMMUDeviceIOMMUFDClass *hiodic = + HOST_IOMMU_DEVICE_IOMMUFD_GET_CLASS(hiodi); + g_assert(hiodic->detach_hwpt); - return hiodic->detach_hwpt(hiodi, errp); + return hiodic->detach_hwpt(hiodi, pasid, errp); } static int hiod_iommufd_get_cap(HostIOMMUDevice *hiod, int cap, Error **errp) diff --git a/backends/vhost-user.c b/backends/vhost-user.c index 42845329e7..380d825023 100644 --- a/backends/vhost-user.c +++ b/backends/vhost-user.c @@ -102,7 +102,7 @@ vhost_user_backend_stop(VhostUserBackend *b) { BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(b->vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int ret; + int ret, err; if (!b->started) { return 0; @@ -111,9 +111,9 @@ vhost_user_backend_stop(VhostUserBackend *b) ret = vhost_dev_stop(&b->dev, b->vdev, true); if (k->set_guest_notifiers && - k->set_guest_notifiers(qbus->parent, b->dev.nvqs, false) < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); - return -1; + (err = k->set_guest_notifiers(qbus->parent, b->dev.nvqs, false)) < 0) { + error_report("vhost guest notifier cleanup failed: %d", err); + return err; } vhost_dev_disable_notifiers(&b->dev, b->vdev); diff --git a/block/export/vhost-user-blk-server.c b/block/export/vhost-user-blk-server.c index 67912a3e17..46c0d303df 100644 --- a/block/export/vhost-user-blk-server.c +++ b/block/export/vhost-user-blk-server.c @@ -119,7 +119,7 @@ static uint64_t vu_blk_get_features(VuDev *dev) uint64_t features; VuServer *server = container_of(dev, VuServer, vu_dev); VuBlkExport *vexp = container_of(server, VuBlkExport, vu_server); - features = 1ull << VIRTIO_BLK_F_SIZE_MAX | + features = 1ull << VIRTIO_BLK_F_SEG_MAX | 1ull << VIRTIO_BLK_F_TOPOLOGY | 1ull << VIRTIO_BLK_F_BLK_SIZE | diff --git a/docs/devel/vfio-iommufd.rst b/docs/devel/vfio-iommufd.rst index 78bcdffac7..d5538461ba 100644 --- a/docs/devel/vfio-iommufd.rst +++ b/docs/devel/vfio-iommufd.rst @@ -153,22 +153,22 @@ RAM discarding for mdev. ``vfio-ap`` and ``vfio-ccw`` devices don't have same issue as their backend devices are always mdev and RAM discarding is force enabled. -Usage with intel_iommu featuring x-flts=on +Usage with intel_iommu featuring fsts=on ------------------------------------------ Only IOMMUFD backed VFIO device is supported when intel_iommu is configured -with x-flts=on, for legacy container backed VFIO device, below error shows: +with fsts=on, for legacy container backed VFIO device, below error shows: .. code-block:: none - qemu-system-x86_64: -device vfio-pci,host=0000:02:00.0: vfio 0000:02:00.0: Failed to set vIOMMU: Need IOMMUFD backend when x-flts=on + qemu-system-x86_64: -device vfio-pci,host=0000:02:00.0: vfio 0000:02:00.0: Failed to set vIOMMU: Need IOMMUFD backend when fsts=on VFIO device under PCI bridge is unsupported, use PCIE bridge if necessary, otherwise below error shows: .. code-block:: none - qemu-system-x86_64: -device vfio-pci,host=0000:02:00.0,bus=bridge1,iommufd=iommufd0: vfio 0000:02:00.0: Failed to set vIOMMU: Host device downstream to a PCI bridge is unsupported when x-flts=on + qemu-system-x86_64: -device vfio-pci,host=0000:02:00.0,bus=bridge1,iommufd=iommufd0: vfio 0000:02:00.0: Failed to set vIOMMU: Host device downstream to a PCI bridge is unsupported when fsts=on If host IOMMU has ERRATA_772415_SPR17, running guest with "intel_iommu=on,sm_off" is unsupported, kexec or reboot guest from "intel_iommu=on,sm_on" to @@ -177,4 +177,4 @@ below if it's not needed by guest: .. code-block:: bash - -device intel-iommu,x-scalable-mode=off + -device intel-iommu,scalable-mode=off diff --git a/docs/interop/vhost-user.rst b/docs/interop/vhost-user.rst index 137c9f3669..c83ae2accb 100644 --- a/docs/interop/vhost-user.rst +++ b/docs/interop/vhost-user.rst @@ -164,8 +164,16 @@ A vring address description :log: a 64-bit guest address for logging -Note that a ring address is an IOVA if ``VIRTIO_F_IOMMU_PLATFORM`` has -been negotiated. Otherwise it is a user address. +.. Note:: + When ``VIRTIO_F_IOMMU_PLATFORM`` is negotiated, ring addresses are IOVAs. + + Otherwise, when ``VHOST_USER_PROTOCOL_F_GPA_ADDRESSES`` is negotiated, the + ring addresses are guest physical addresses for frontend messages. That + does not apply to backend replies. + + Finally, when neither ``VIRTIO_F_IOMMU_PLATFORM`` nor + ``VHOST_USER_PROTOCOL_F_GPA_ADDRESSES`` features are negotiated, ring + addresses are user virtual addresses. .. _memory_region_description: @@ -180,7 +188,9 @@ Memory region description :size: a 64-bit size -:user address: a 64-bit user address +:user address: a 64-bit user address. When ``VHOST_USER_PROTOCOL_F_GPA_ADDRESSES`` + is negotiated, this field contain guest physical address instead and must + duplicate ``guest address`` field. :mmap offset: a 64-bit offset where region starts in the mapped memory @@ -252,7 +262,9 @@ An IOTLB message :size: a 64-bit size -:user address: a 64-bit user address +:user address: a 64-bit user address. When ``VHOST_USER_PROTOCOL_F_GPA_ADDRESSES`` + is negotiated, this field contain guest physical address instead, except for + ``VHOST_USER_BACKEND_IOTLB_MSG``, where it's user address anyway. :permissions flags: an 8-bit value: - 0: No access @@ -350,6 +362,44 @@ Device state transfer parameters In the future, additional phases might be added e.g. to allow iterative migration while the device is running. +MMAP request +^^^^^^^^^^^^ + ++-------+---------+-----------+------------+-----+-------+ +| shmid | padding | fd_offset | shm_offset | len | flags | ++-------+---------+-----------+------------+-----+-------+ + +:shmid: a 8-bit shared memory region identifier + +:fd_offset: a 64-bit offset of this area from the start + of the supplied file descriptor + +:shm_offset: a 64-bit offset from the start of the + pointed shared memory region + +:len: a 64-bit size of the memory to map + +:flags: a 64-bit value: + + - 0: Pages are mapped read-only + - 1: Pages are mapped read-write + +VIRTIO Shared Memory Region configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++-------------+---------+------------+----+--------------+ +| num regions | padding | mem size 0 | .. | mem size 255 | ++-------------+---------+------------+----+--------------+ + +:num regions: a 32-bit number of regions + +:padding: 32-bit + +:mem size: an array of 256 64-bit fields representing the size of each + VIRTIO Shared Memory Region. ``num regions`` specifies the + number of valid regions (non-zero size). The array index + corresponds to the shared memory ID (shmid). + C structure ----------- @@ -366,11 +416,17 @@ In QEMU the vhost-user message is implemented with the following struct: struct vhost_vring_state state; struct vhost_vring_addr addr; VhostUserMemory memory; + VhostUserMemRegMsg mem_reg; VhostUserLog log; struct vhost_iotlb_msg iotlb; VhostUserConfig config; + VhostUserCryptoSession session; VhostUserVringArea area; VhostUserInflight inflight; + VhostUserShared object; + VhostUserTransferDeviceState transfer_state; + VhostUserMMap mmap; + VhostUserShMemConfig shmem; }; } QEMU_PACKED VhostUserMsg; @@ -656,7 +712,10 @@ destination, following the usual protocol for establishing a connection to a vhost-user back-end: This includes, for example, setting up memory mappings and kick and call FDs as necessary, negotiating protocol features, or setting the initial vring base indices (to the same value -as on the source side, so that operation can resume). +as on the source side, so that operation can resume). The vhost-user front-end +may also write to the kick FDs of vrings containing unused buffers or send +``VHOST_USER_VRING_KICK`` if negotiated to start those vrings in the destination +since the driver likely already kicked them in the source and won't do it again. Both on the source and on the destination side, after the respective front-end has seen all data transferred (when the transfer FD has been @@ -1063,6 +1122,8 @@ Protocol features #define VHOST_USER_PROTOCOL_F_SHARED_OBJECT 18 #define VHOST_USER_PROTOCOL_F_DEVICE_STATE 19 #define VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT 20 + #define VHOST_USER_PROTOCOL_F_GPA_ADDRESSES 21 + #define VHOST_USER_PROTOCOL_F_SHMEM_MAP 22 Front-end message types ----------------------- @@ -1268,7 +1329,7 @@ Front-end message types How to suspend an in-flight request depends on the implementation of the back-end but it typically can be done by aborting or cancelling the underlying I/O request. The ``VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT`` - protocol feature must only be neogotiated if + protocol feature must only be negotiated if ``VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD`` is also negotiated. ``VHOST_USER_SET_VRING_KICK`` @@ -1750,6 +1811,28 @@ Front-end message types Using this function requires prior negotiation of the ``VHOST_USER_PROTOCOL_F_DEVICE_STATE`` feature. +``VHOST_USER_GET_SHMEM_CONFIG`` + :id: 44 + :equivalent ioctl: N/A + :request payload: N/A + :reply payload: ``struct VhostUserShMemConfig`` + + When the ``VHOST_USER_PROTOCOL_F_SHMEM`` protocol feature has been + successfully negotiated, this message can be submitted by the front-end + to gather the VIRTIO Shared Memory Region configuration. The back-end will + respond with the number of VIRTIO Shared Memory Regions it requires, and + each shared memory region size in an array. The shared memory IDs are + represented by the array index. The information returned shall comply + with the following rules: + + * The shared information will remain valid and unchanged for the entire + lifetime of the connection. + + * The Shared Memory Region size must be a multiple of the page size + supported by mmap(2). + + * The size may be 0 if the region is unused. + Back-end message types ---------------------- @@ -1884,6 +1967,46 @@ is sent by the front-end. when the operation is successful, or non-zero otherwise. Note that if the operation fails, no fd is sent to the backend. +``VHOST_USER_BACKEND_SHMEM_MAP`` + :id: 9 + :equivalent ioctl: N/A + :request payload: fd and ``struct VhostUserMMap`` + :reply payload: N/A + + When the ``VHOST_USER_PROTOCOL_F_SHMEM`` protocol feature has been + successfully negotiated, this message can be submitted by the backends to + advertise a new mapping to be made in a given VIRTIO Shared Memory Region. + Upon receiving the message, the front-end will mmap the given fd into the + VIRTIO Shared Memory Region with the requested ``shmid``. + + If ``VHOST_USER_PROTOCOL_F_REPLY_ACK`` is negotiated, and + back-end set the ``VHOST_USER_NEED_REPLY`` flag, the front-end + must respond with zero when operation is successfully completed, + or non-zero otherwise. + + Mapping over an already existing map is not allowed and requests shall fail. + Therefore, the memory range in the request must correspond with a valid, + free region of the VIRTIO Shared Memory Region. Also, note that mappings + consume resources and that the request can fail when there are no resources + available. Lastly, mappings are automatically unmapped by the front-end + across device reset operation. + +``VHOST_USER_BACKEND_SHMEM_UNMAP`` + :id: 10 + :equivalent ioctl: N/A + :request payload: ``struct VhostUserMMap`` + :reply payload: N/A + + When the ``VHOST_USER_PROTOCOL_F_SHMEM`` protocol feature has been + successfully negotiated, this message can be submitted by the backends so + that the front-end un-mmaps a given range (``shm_offset``, ``len``) in the + VIRTIO Shared Memory Region with the requested ``shmid``. Note that the + given range shall correspond to the entirety of a valid mapped region. + + If ``VHOST_USER_PROTOCOL_F_REPLY_ACK`` is negotiated, and the back-end + sets the ``VHOST_USER_NEED_REPLY`` flag, the front-end must respond with + zero when operation is successfully completed, or non-zero otherwise. + .. _reply_ack: VHOST_USER_PROTOCOL_F_REPLY_ACK diff --git a/hw/acpi/aml-build-stub.c b/hw/acpi/aml-build-stub.c index 89a8fec4af..3180c7c962 100644 --- a/hw/acpi/aml-build-stub.c +++ b/hw/acpi/aml-build-stub.c @@ -67,6 +67,12 @@ Aml *aml_irq_no_flags(uint8_t irq) return NULL; } +Aml *aml_irq(uint8_t irq, AmlLevelAndEdge level_and_edge, + AmlActiveHighAndLow high_and_low, AmlShared shared) +{ + return NULL; +} + Aml *aml_interrupt(AmlConsumerAndProducer con_and_pro, AmlLevelAndEdge level_and_edge, AmlActiveHighAndLow high_and_low, AmlShared shared, diff --git a/hw/acpi/aml-build.c b/hw/acpi/aml-build.c index 7edc8aed42..9b3cdd3781 100644 --- a/hw/acpi/aml-build.c +++ b/hw/acpi/aml-build.c @@ -35,6 +35,7 @@ #include "hw/acpi/acpi_aml_interface.h" #include "qemu/cutils.h" #include "hw/core/cpu.h" +#include "hw/acpi/wdat.h" static GArray *build_alloc_array(void) { @@ -1061,6 +1062,33 @@ Aml *aml_irq_no_flags(uint8_t irq) return var; } +/* + * ACPI 1.0b: 6.4.2.1.1 ASL Macro for IRQ Descriptor + * + * More verbose description at: + * ACPI 5.0: 19.5.63 IRQ (Interrupt Resource Descriptor Macro) + * 6.4.2.1 IRQ Descriptor + */ +Aml *aml_irq(uint8_t irq, AmlLevelAndEdge level_and_edge, + AmlActiveHighAndLow high_and_low, AmlShared shared) +{ + uint16_t irq_mask; + Aml *var = aml_alloc(); + uint8_t irq_flags = level_and_edge | (high_and_low << 3) | + (shared << 4); + + assert((level_and_edge == AML_EDGE && high_and_low == AML_ACTIVE_HIGH) || + (level_and_edge == AML_LEVEL && high_and_low == AML_ACTIVE_LOW)); + assert(irq < 16); + build_append_byte(var->buf, 0x23); /* IRQ descriptor 3 byte form */ + + irq_mask = 1U << irq; + build_append_byte(var->buf, irq_mask & 0xFF); /* IRQ mask bits[7:0] */ + build_append_byte(var->buf, irq_mask >> 8); /* IRQ mask bits[15:8] */ + build_append_byte(var->buf, irq_flags); /* IRQ flags */ + return var; +} + /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLNot */ Aml *aml_lnot(Aml *arg) { @@ -2845,3 +2873,16 @@ void qbus_build_aml(BusState *bus, Aml *scope) call_dev_aml_func(DEVICE(kid->child), scope); } } + +void build_append_wdat_ins(GArray *table_data, + WDATAction action, uint8_t flags, + struct AcpiGenericAddress as, + uint32_t val, uint32_t mask) +{ + build_append_int_noprefix(table_data, action, 1); /* Watchdog Action */ + build_append_int_noprefix(table_data, flags, 1); /* Instruction Flags */ + build_append_int_noprefix(table_data, 0, 2); /* Reserved */ + build_append_gas_from_struct(table_data, &as); /* Register Region */ + build_append_int_noprefix(table_data, val, 4); /* Value */ + build_append_int_noprefix(table_data, mask, 4); /* Mask */ +} diff --git a/hw/acpi/meson.build b/hw/acpi/meson.build index 1c5251909b..0c7bfb278a 100644 --- a/hw/acpi/meson.build +++ b/hw/acpi/meson.build @@ -25,7 +25,8 @@ acpi_ss.add(when: 'CONFIG_ACPI_PCI_BRIDGE', if_true: files('pci-bridge.c')) acpi_ss.add(when: 'CONFIG_ACPI_PCIHP', if_true: files('pcihp.c')) stub_ss.add(files('acpi-pci-hotplug-stub.c')) acpi_ss.add(when: 'CONFIG_ACPI_VIOT', if_true: files('viot.c')) -acpi_ss.add(when: 'CONFIG_ACPI_ICH9', if_true: files('ich9.c', 'ich9_tco.c', 'ich9_timer.c')) +acpi_ss.add(when: 'CONFIG_ACPI_ICH9', if_true: files('ich9.c', 'ich9_tco.c', 'ich9_timer.c', 'wdat-ich9.c')) +stub_ss.add(files('wdat-ich9-stub.c')) acpi_ss.add(when: 'CONFIG_ACPI_ERST', if_true: files('erst.c')) acpi_ss.add(when: 'CONFIG_IPMI', if_true: files('ipmi.c')) stub_ss.add(files('ipmi-stub.c')) diff --git a/hw/acpi/wdat-ich9-stub.c b/hw/acpi/wdat-ich9-stub.c new file mode 100644 index 0000000000..7c30ef3558 --- /dev/null +++ b/hw/acpi/wdat-ich9-stub.c @@ -0,0 +1,15 @@ +/* + * Copyright Red Hat, Inc. 2026 + * Author(s): Igor Mammedov + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "hw/acpi/wdat-ich9.h" + +void build_ich9_wdat(GArray *table_data, BIOSLinker *linker, const char *oem_id, + const char *oem_table_id, uint64_t tco_base) +{ + g_assert_not_reached(); +} diff --git a/hw/acpi/wdat-ich9.c b/hw/acpi/wdat-ich9.c new file mode 100644 index 0000000000..259c45c62d --- /dev/null +++ b/hw/acpi/wdat-ich9.c @@ -0,0 +1,90 @@ +/* + * TCO Watchdog Action Table (WDAT) + * + * Copyright Red Hat, Inc. 2026 + * Author(s): Igor Mammedov + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "hw/acpi/wdat.h" +#include "hw/acpi/wdat-ich9.h" +#include "hw/southbridge/ich9.h" + +#define TCO_REG(base, reg_offset, reg_width) { .space_id = AML_AS_SYSTEM_IO, \ + .address = base + reg_offset, .bit_width = reg_width, \ + .access_width = AML_WORD_ACC, }; + +/* + * "Hardware Watchdog Timers Design Specification" + * https://uefi.org/acpi 'Watchdog Action Table (WDAT)' + * + * ICH9 specific implementation. + */ +void build_ich9_wdat(GArray *table_data, BIOSLinker *linker, const char *oem_id, + const char *oem_table_id, uint64_t tco_base) +{ + AcpiTable table = { .sig = "WDAT", .rev = 1, .oem_id = oem_id, + .oem_table_id = oem_table_id }; + struct AcpiGenericAddress tco_rld = TCO_REG(tco_base, 0x0, 16); + struct AcpiGenericAddress tco2_sts = TCO_REG(tco_base, 0x6, 16); + struct AcpiGenericAddress tco1_cnt = TCO_REG(tco_base, 0x8, 16); + struct AcpiGenericAddress tco_tmr = TCO_REG(tco_base, 0x12, 16); + + acpi_table_begin(&table, table_data); + build_append_int_noprefix(table_data, 0x20, 4); /* Watchdog Header Length */ + build_append_int_noprefix(table_data, 0xff, 2); /* PCI Segment */ + build_append_int_noprefix(table_data, 0xff, 1); /* PCI Bus Number */ + build_append_int_noprefix(table_data, 0xff, 1); /* PCI Device Number */ + build_append_int_noprefix(table_data, 0xff, 1); /* PCI Function Number */ + build_append_int_noprefix(table_data, 0, 3); /* Reserved */ + /* + * limits/resolution are defined by ICH9 TCO spec + */ + build_append_int_noprefix(table_data, 0x258, 4);/* Timer Period, ms */ + build_append_int_noprefix(table_data, 0x3ff, 4);/* Maximum Count */ + build_append_int_noprefix(table_data, 0x4, 4); /* Minimum Count */ + /* + * WATCHDOG_ENABLED & WATCHDOG_STOPPED_IN_SLEEP_STATE + */ + build_append_int_noprefix(table_data, 0x81, 1); /* Watchdog Flags */ + build_append_int_noprefix(table_data, 0, 3); /* Reserved */ + /* + * watchdog instruction entries + */ + build_append_int_noprefix(table_data, 10 /* # of actions below */, 4); + /* Action table */ + build_append_wdat_ins(table_data, WDAT_ACTION_RESET, + WDAT_INS_WRITE_VALUE, + tco_rld, 0x1, 0x1ff); + build_append_wdat_ins(table_data, WDAT_ACTION_QUERY_RUNNING_STATE, + WDAT_INS_READ_VALUE, + tco1_cnt, 0x0, 0x800); + build_append_wdat_ins(table_data, WDAT_ACTION_SET_RUNNING_STATE, + WDAT_INS_WRITE_VALUE | WDAT_INS_PRESERVE_REGISTER, + tco1_cnt, 0, 0x800); + build_append_wdat_ins(table_data, WDAT_ACTION_QUERY_STOPPED_STATE, + WDAT_INS_READ_VALUE, + tco1_cnt, 0x800, 0x800); + build_append_wdat_ins(table_data, WDAT_ACTION_SET_STOPPED_STATE, + WDAT_INS_WRITE_VALUE | WDAT_INS_PRESERVE_REGISTER, + tco1_cnt, 0x800, 0x800); + build_append_wdat_ins(table_data, WDAT_ACTION_SET_COUNTDOWN_PERIOD, + WDAT_INS_WRITE_COUNTDOWN, + tco_tmr, 0x0, 0x3FF); + build_append_wdat_ins(table_data, WDAT_ACTION_QUERY_COUNTDOWN_PERIOD, + WDAT_INS_READ_COUNTDOWN, + tco_tmr, 0x0, 0x3FF); + build_append_wdat_ins(table_data, WDAT_ACTION_QUERY_WATCHDOG_STATUS, + WDAT_INS_READ_VALUE, + tco2_sts, 0x2, 0x2); + build_append_wdat_ins(table_data, WDAT_ACTION_SET_WATCHDOG_STATUS, + WDAT_INS_WRITE_VALUE | WDAT_INS_PRESERVE_REGISTER, + tco2_sts, 0x2, 0x2); + build_append_wdat_ins(table_data, WDAT_ACTION_SET_WATCHDOG_STATUS, + WDAT_INS_WRITE_VALUE | WDAT_INS_PRESERVE_REGISTER, + tco2_sts, 0x4, 0x4); + + acpi_table_end(linker, &table); +} diff --git a/hw/arm/smmuv3-accel.c b/hw/arm/smmuv3-accel.c index 862be814a0..2947e2e5dc 100644 --- a/hw/arm/smmuv3-accel.c +++ b/hw/arm/smmuv3-accel.c @@ -300,7 +300,8 @@ bool smmuv3_accel_install_ste(SMMUv3State *s, SMMUDevice *sdev, int sid, return false; } - if (!host_iommu_device_iommufd_attach_hwpt(hiodi, hwpt_id, errp)) { + if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, hwpt_id, + errp)) { if (s1_hwpt) { iommufd_backend_free_id(hiodi->iommufd, s1_hwpt->hwpt_id); g_free(s1_hwpt); @@ -575,7 +576,8 @@ smmuv3_accel_alloc_viommu(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *hiodi, /* Attach a HWPT based on SMMUv3 GBPA.ABORT value */ hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel); - if (!host_iommu_device_iommufd_attach_hwpt(hiodi, hwpt_id, errp)) { + if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, hwpt_id, + errp)) { goto free_veventq; } return true; @@ -665,7 +667,8 @@ static void smmuv3_accel_unset_iommu_device(PCIBus *bus, void *opaque, hiodi = accel_dev->hiodi; accel = accel_dev->s_accel; /* Re-attach the default s2 hwpt id */ - if (!host_iommu_device_iommufd_attach_hwpt(hiodi, hiodi->hwpt_id, NULL)) { + if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, + hiodi->hwpt_id, NULL)) { error_report("Unable to attach the default HW pagetable: hiodi devid " "0x%x", hiodi->devid); } @@ -879,7 +882,8 @@ bool smmuv3_accel_attach_gbpa_hwpt(SMMUv3State *s, Error **errp) hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel); QLIST_FOREACH(accel_dev, &accel->device_list, next) { - if (!host_iommu_device_iommufd_attach_hwpt(accel_dev->hiodi, hwpt_id, + if (!host_iommu_device_iommufd_attach_hwpt(accel_dev->hiodi, + IOMMU_NO_PASID, hwpt_id, &local_err)) { error_append_hint(&local_err, "Failed to attach GBPA hwpt %u for " "hiodi devid %u", hwpt_id, diff --git a/hw/block/trace-events b/hw/block/trace-events index cc9a9f2460..dbaa5ca6cb 100644 --- a/hw/block/trace-events +++ b/hw/block/trace-events @@ -58,6 +58,16 @@ virtio_blk_handle_zone_mgmt(void *vdev, void *req, uint8_t op, int64_t sector, i virtio_blk_handle_zone_reset_all(void *vdev, void *req, int64_t sector, int64_t len) "vdev %p req %p sector 0x%" PRIx64 " cap 0x%" PRIx64 "" virtio_blk_handle_zone_append(void *vdev, void *req, int64_t sector) "vdev %p req %p, append sector 0x%" PRIx64 "" +# vhost-user-blk.c +vhost_user_blk_start_in(void *vdev) "vdev %p" +vhost_user_blk_start_out(void *vdev) "vdev %p" +vhost_user_blk_stop_in(void *vdev) "vdev %p" +vhost_user_blk_stop_out(void *vdev) "vdev %p" +vhost_user_blk_connect_in(void *vdev) "vdev %p" +vhost_user_blk_connect_out(void *vdev) "vdev %p" +vhost_user_blk_device_realize_in(void *vdev) "vdev %p" +vhost_user_blk_device_realize_out(void *vdev) "vdev %p" + # hd-geometry.c hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d" hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "blk %p CHS %u %u %u trans %d" diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c index c151e83677..b2e46eb3f9 100644 --- a/hw/block/vhost-user-blk.c +++ b/hw/block/vhost-user-blk.c @@ -31,6 +31,7 @@ #include "hw/virtio/virtio-access.h" #include "system/system.h" #include "system/runstate.h" +#include "trace.h" static const int user_feature_bits[] = { VIRTIO_BLK_F_SIZE_MAX, @@ -137,6 +138,8 @@ static int vhost_user_blk_start(VirtIODevice *vdev, Error **errp) VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); int i, ret; + trace_vhost_user_blk_start_in(vdev); + if (!k->set_guest_notifiers) { error_setg(errp, "binding does not support guest notifiers"); return -ENOSYS; @@ -192,6 +195,8 @@ static int vhost_user_blk_start(VirtIODevice *vdev, Error **errp) } s->started_vu = true; + trace_vhost_user_blk_start_out(vdev); + return ret; err_guest_notifiers: @@ -209,9 +214,11 @@ static int vhost_user_blk_stop(VirtIODevice *vdev) VHostUserBlk *s = VHOST_USER_BLK(vdev); BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int ret; + int ret, err; bool force_stop = false; + trace_vhost_user_blk_stop_in(vdev); + if (!s->started_vu) { return 0; } @@ -227,12 +234,16 @@ static int vhost_user_blk_stop(VirtIODevice *vdev) ret = force_stop ? vhost_dev_force_stop(&s->dev, vdev, true) : vhost_dev_stop(&s->dev, vdev, true); - if (k->set_guest_notifiers(qbus->parent, s->dev.nvqs, false) < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); - return -1; + err = k->set_guest_notifiers(qbus->parent, s->dev.nvqs, false); + if (err < 0) { + error_report("vhost guest notifier cleanup failed: %d", err); + return err; } vhost_dev_disable_notifiers(&s->dev, vdev); + + trace_vhost_user_blk_stop_out(vdev); + return ret; } @@ -273,7 +284,6 @@ static uint64_t vhost_user_blk_get_features(VirtIODevice *vdev, VHostUserBlk *s = VHOST_USER_BLK(vdev); /* Turn on pre-defined features */ - virtio_add_feature(&features, VIRTIO_BLK_F_SIZE_MAX); virtio_add_feature(&features, VIRTIO_BLK_F_SEG_MAX); virtio_add_feature(&features, VIRTIO_BLK_F_GEOMETRY); virtio_add_feature(&features, VIRTIO_BLK_F_TOPOLOGY); @@ -340,6 +350,8 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp) VHostUserBlk *s = VHOST_USER_BLK(vdev); int ret = 0; + trace_vhost_user_blk_connect_in(vdev); + if (s->connected) { return 0; } @@ -348,7 +360,6 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp) s->dev.nvqs = s->num_queues; s->dev.vqs = s->vhost_vqs; s->dev.vq_index = 0; - s->dev.backend_features = 0; vhost_dev_set_config_notifier(&s->dev, &blk_ops); @@ -367,6 +378,8 @@ static int vhost_user_blk_connect(DeviceState *dev, Error **errp) ret = vhost_user_blk_start(vdev, errp); } + trace_vhost_user_blk_connect_out(vdev); + return ret; } @@ -457,6 +470,8 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) int retries; int i, ret; + trace_vhost_user_blk_device_realize_in(vdev); + if (!s->chardev.chr) { error_setg(errp, "chardev is mandatory"); return; @@ -516,6 +531,9 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) qemu_chr_fe_set_handlers(&s->chardev, NULL, NULL, vhost_user_blk_event, NULL, (void *)dev, NULL, true); + + trace_vhost_user_blk_device_realize_out(vdev); + return; virtio_err: @@ -573,10 +591,8 @@ static bool vhost_user_blk_inflight_needed(void *opaque) { struct VHostUserBlk *s = opaque; - bool inflight_migration = virtio_has_feature(s->dev.protocol_features, - VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT); - - return inflight_migration; + return vhost_user_has_protocol_feature( + &s->dev, VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT); } static const VMStateDescription vmstate_vhost_user_blk_inflight = { diff --git a/hw/char/serial-isa.c b/hw/char/serial-isa.c index a4be0492c5..eaa4e843c0 100644 --- a/hw/char/serial-isa.c +++ b/hw/char/serial-isa.c @@ -92,7 +92,8 @@ static void serial_isa_build_aml(AcpiDevAmlIf *adev, Aml *scope) crs = aml_resource_template(); aml_append(crs, aml_io(AML_DECODE16, isa->iobase, isa->iobase, 0x00, 0x08)); - aml_append(crs, aml_irq_no_flags(isa->isairq)); + aml_append(crs, aml_irq(isa->isairq, AML_LEVEL, AML_ACTIVE_LOW, + AML_SHARED)); dev = aml_device("COM%d", isa->index + 1); aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0501"))); diff --git a/hw/cxl/cxl-component-utils.c b/hw/cxl/cxl-component-utils.c index 07aabe331c..a624357978 100644 --- a/hw/cxl/cxl-component-utils.c +++ b/hw/cxl/cxl-component-utils.c @@ -143,6 +143,12 @@ static void dumb_hdm_handler(CXLComponentState *cxl_cstate, hwaddr offset, value = FIELD_DP32(value, CXL_HDM_DECODER0_CTRL, COMMITTED, 0); } stl_le_p((uint8_t *)cache_mem + offset, value); + + if (should_commit) { + cfmws_update_non_interleaved(true); + } else if (should_uncommit) { + cfmws_update_non_interleaved(false); + } } static void bi_handler(CXLComponentState *cxl_cstate, hwaddr offset, diff --git a/hw/cxl/cxl-host.c b/hw/cxl/cxl-host.c index a94b893e99..7e744312f1 100644 --- a/hw/cxl/cxl-host.c +++ b/hw/cxl/cxl-host.c @@ -104,7 +104,7 @@ void cxl_fmws_link_targets(Error **errp) } static bool cxl_hdm_find_target(uint32_t *cache_mem, hwaddr addr, - uint8_t *target) + uint8_t *target, bool *interleaved) { int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO; unsigned int hdm_count; @@ -138,6 +138,11 @@ static bool cxl_hdm_find_target(uint32_t *cache_mem, hwaddr addr, found = true; ig_enc = FIELD_EX32(ctrl, CXL_HDM_DECODER0_CTRL, IG); iw_enc = FIELD_EX32(ctrl, CXL_HDM_DECODER0_CTRL, IW); + + if (interleaved) { + *interleaved = iw_enc != 0; + } + target_idx = (addr / cxl_decode_ig(ig_enc)) % (1 << iw_enc); if (target_idx < 4) { @@ -157,7 +162,8 @@ static bool cxl_hdm_find_target(uint32_t *cache_mem, hwaddr addr, return found; } -static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) +static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr, + bool allow_interleave) { CXLComponentState *hb_cstate, *usp_cstate; PCIHostState *hb; @@ -165,9 +171,13 @@ static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) int rb_index; uint32_t *cache_mem; uint8_t target; - bool target_found; + bool target_found, interleaved; PCIDevice *rp, *d; + if ((fw->num_targets > 1) && !allow_interleave) { + return NULL; + } + rb_index = (addr / cxl_decode_ig(fw->enc_int_gran)) % fw->num_targets; hb = PCI_HOST_BRIDGE(fw->target_hbs[rb_index]->cxl_host_bridge); if (!hb || !hb->bus || !pci_bus_is_cxl(hb->bus)) { @@ -187,11 +197,16 @@ static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) cache_mem = hb_cstate->crb.cache_mem_registers; - target_found = cxl_hdm_find_target(cache_mem, addr, &target); + target_found = cxl_hdm_find_target(cache_mem, addr, &target, + &interleaved); if (!target_found) { return NULL; } + if (interleaved && !allow_interleave) { + return NULL; + } + rp = pcie_find_port_by_pn(hb->bus, target); if (!rp) { return NULL; @@ -223,11 +238,15 @@ static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) cache_mem = usp_cstate->crb.cache_mem_registers; - target_found = cxl_hdm_find_target(cache_mem, addr, &target); + target_found = cxl_hdm_find_target(cache_mem, addr, &target, &interleaved); if (!target_found) { return NULL; } + if (interleaved && !allow_interleave) { + return NULL; + } + d = pcie_find_port_by_pn(&PCI_BRIDGE(d)->sec_bus, target); if (!d) { return NULL; @@ -245,13 +264,214 @@ static PCIDevice *cxl_cfmws_find_device(CXLFixedWindow *fw, hwaddr addr) return d; } +typedef struct CXLDirectPTState { + CXLType3Dev *ct3d; + hwaddr decoder_base; + hwaddr decoder_size; + hwaddr dpa_base; + unsigned int hdm_decoder_idx; +} CXLDirectPTState; + +static void cxl_fmws_direct_passthrough_setup(CXLDirectPTState *state, + CXLFixedWindow *fw) +{ + CXLType3Dev *ct3d = state->ct3d; + MemoryRegion *mr = NULL; + uint64_t vmr_size = 0, pmr_size = 0, offset = 0; + MemoryRegion *direct_mr; + g_autofree char *direct_mr_name; + unsigned int idx = state->hdm_decoder_idx; + + if (ct3d->hostvmem) { + MemoryRegion *vmr = host_memory_backend_get_memory(ct3d->hostvmem); + + vmr_size = memory_region_size(vmr); + if (state->dpa_base < vmr_size) { + mr = vmr; + offset = state->dpa_base; + } + } + if (!mr && ct3d->hostpmem) { + MemoryRegion *pmr = host_memory_backend_get_memory(ct3d->hostpmem); + + pmr_size = memory_region_size(pmr); + if (state->dpa_base - vmr_size < pmr_size) { + mr = pmr; + offset = state->dpa_base - vmr_size; + } + } + if (!mr) { + return; + } + + if (ct3d->direct_mr_fw[idx]) { + return; + } + + direct_mr = &ct3d->direct_mr[idx]; + direct_mr_name = g_strdup_printf("cxl-direct-mapping-alias-%u", idx); + if (!direct_mr_name) { + return; + } + + memory_region_init_alias(direct_mr, OBJECT(ct3d), direct_mr_name, mr, + offset, state->decoder_size); + memory_region_transaction_begin(); + memory_region_add_subregion(&fw->mr, + state->decoder_base - fw->base, direct_mr); + memory_region_transaction_commit(); + ct3d->direct_mr_fw[idx] = fw; +} + +static void cxl_fmws_direct_passthrough_remove(CXLType3Dev *ct3d, + uint64_t decoder_base, + unsigned int idx) +{ + CXLFixedWindow *owner_fw = ct3d->direct_mr_fw[idx]; + MemoryRegion *direct_mr = &ct3d->direct_mr[idx]; + + if (!owner_fw) { + return; + } + + if (!memory_region_is_mapped(direct_mr)) { + return; + } + + if (cxl_cfmws_find_device(owner_fw, decoder_base, false)) { + return; + } + + memory_region_transaction_begin(); + memory_region_del_subregion(&owner_fw->mr, direct_mr); + object_unparent(OBJECT(direct_mr)); + memory_region_transaction_commit(); + ct3d->direct_mr_fw[idx] = NULL; +} + +static int cxl_fmws_direct_passthrough(Object *obj, void *opaque) +{ + CXLDirectPTState *state = opaque; + CXLFixedWindow *fw; + + if (!object_dynamic_cast(obj, TYPE_CXL_FMW)) { + return 0; + } + + fw = CXL_FMW(obj); + + /* Verify not interleaved */ + if (!cxl_cfmws_find_device(fw, state->decoder_base, false)) { + return 0; + } + + cxl_fmws_direct_passthrough_setup(state, fw); + + return 0; +} + +static int update_non_interleaved(Object *obj, void *opaque) +{ + const int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO; + bool commit = *(bool *)opaque; + CXLType3Dev *ct3d; + uint32_t *cache_mem; + unsigned int hdm_count, i; + int interleave_ways_dec; + uint32_t cap; + uint64_t dpa_base = 0; + + if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) { + return 0; + } + + ct3d = CXL_TYPE3(obj); + cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers; + cap = ldl_le_p(cache_mem + R_CXL_HDM_DECODER_CAPABILITY); + hdm_count = cxl_decoder_count_dec(FIELD_EX32(cap, + CXL_HDM_DECODER_CAPABILITY, + DECODER_COUNT)); + for (i = 0; i < hdm_count; i++) { + uint64_t decoder_base, decoder_size, skip; + uint32_t hdm_ctrl, low, high; + int iw, committed; + + hdm_ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + i * hdm_inc); + committed = FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED); + + /* + * Optimization: Looking for a fully committed path; if the type 3 HDM + * decoder is not commmitted, it cannot lie on such a path. + */ + if (commit && !committed) { + return 0; + } + + low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_DPA_SKIP_LO + + i * hdm_inc); + high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_DPA_SKIP_HI + + i * hdm_inc); + skip = ((uint64_t)high << 32) | (low & 0xf0000000); + dpa_base += skip; + + low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_LO + i * hdm_inc); + high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_HI + i * hdm_inc); + decoder_size = ((uint64_t)high << 32) | (low & 0xf0000000); + + low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_LO + i * hdm_inc); + high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_HI + i * hdm_inc); + decoder_base = ((uint64_t)high << 32) | (low & 0xf0000000); + + iw = FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, IW); + + if (iw == 0) { + if (!commit) { + cxl_fmws_direct_passthrough_remove(ct3d, decoder_base, i); + } else { + CXLDirectPTState state = { + .ct3d = ct3d, + .decoder_base = decoder_base, + .decoder_size = decoder_size, + .dpa_base = dpa_base, + .hdm_decoder_idx = i, + }; + + object_child_foreach_recursive(object_get_root(), + cxl_fmws_direct_passthrough, + &state); + } + } + + interleave_ways_dec = cxl_interleave_ways_dec(iw, &error_fatal); + if (interleave_ways_dec == 0) { + return 0; + } + + dpa_base += decoder_size / interleave_ways_dec; + } + + return 0; +} + +void cfmws_update_non_interleaved(bool commit) +{ + /* + * Walk endpoints to find both committed and uncommitted decoders, + * then check if they are not interleaved (but the path is fully set up). + */ + object_child_foreach_recursive(object_get_root(), + update_non_interleaved, &commit); + + return; +} + static MemTxResult cxl_read_cfmws(void *opaque, hwaddr addr, uint64_t *data, unsigned size, MemTxAttrs attrs) { CXLFixedWindow *fw = opaque; PCIDevice *d; - d = cxl_cfmws_find_device(fw, addr + fw->base); + d = cxl_cfmws_find_device(fw, addr + fw->base, true); if (d == NULL) { *data = 0; /* Reads to invalid address return poison */ @@ -268,7 +488,7 @@ static MemTxResult cxl_write_cfmws(void *opaque, hwaddr addr, CXLFixedWindow *fw = opaque; PCIDevice *d; - d = cxl_cfmws_find_device(fw, addr + fw->base); + d = cxl_cfmws_find_device(fw, addr + fw->base, true); if (d == NULL) { /* Writes to invalid address are silent */ return MEMTX_OK; diff --git a/hw/cxl/cxl-mailbox-utils.c b/hw/cxl/cxl-mailbox-utils.c index d8ba7e8625..20e0b7e476 100644 --- a/hw/cxl/cxl-mailbox-utils.c +++ b/hw/cxl/cxl-mailbox-utils.c @@ -1994,8 +1994,8 @@ static void cxl_create_mem_sparing_event_records(CXLType3Dev *ct3d, stw_le_p(&event_rec.column, ent->column); event_rec.sub_channel = ent->sub_channel; if (ent->validity_flags & CXL_MSER_VALID_COMP_ID) { - strncpy((char *)event_rec.component_id, (char *)ent->component_id, - sizeof(event_rec.component_id)); + memcpy(event_rec.component_id, ent->component_id, + sizeof(event_rec.component_id)); } } else if (sparing_pi) { event_rec.flags = CXL_MSER_FLAGS_QUERY_RESOURCES; @@ -2068,6 +2068,8 @@ static void cxl_perform_ppr(CXLType3Dev *ct3d, uint64_t dpa) CXL_MEMDEV_MAINT_SUBCLASS_CACHELINE_SPARING, ent, NULL); } + QLIST_REMOVE(ent, node); + g_free(ent); break; } } diff --git a/hw/display/vhost-user-gpu.c b/hw/display/vhost-user-gpu.c index 6e5e6540a4..57360898ca 100644 --- a/hw/display/vhost-user-gpu.c +++ b/hw/display/vhost-user-gpu.c @@ -631,21 +631,16 @@ vhost_user_gpu_device_realize(DeviceState *qdev, Error **errp) /* existing backend may send DMABUF, so let's add that requirement */ g->parent_obj.conf.flags |= 1 << VIRTIO_GPU_FLAG_DMABUF_ENABLED; - if (virtio_has_feature(g->vhost->dev.features, VIRTIO_GPU_F_VIRGL)) { + if (vhost_dev_has_feature(&g->vhost->dev, VIRTIO_GPU_F_VIRGL)) { g->parent_obj.conf.flags |= 1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED; } - if (virtio_has_feature(g->vhost->dev.features, VIRTIO_GPU_F_EDID)) { + if (vhost_dev_has_feature(&g->vhost->dev, VIRTIO_GPU_F_EDID)) { g->parent_obj.conf.flags |= 1 << VIRTIO_GPU_FLAG_EDID_ENABLED; } else { error_report("EDID requested but the backend doesn't support it."); g->parent_obj.conf.flags &= ~(1 << VIRTIO_GPU_FLAG_EDID_ENABLED); } - if (virtio_has_feature(g->vhost->dev.features, - VIRTIO_GPU_F_RESOURCE_UUID)) { - g->parent_obj.conf.flags |= 1 << VIRTIO_GPU_FLAG_RESOURCE_UUID_ENABLED; - } - if (virtio_has_feature(g->vhost->dev.features, - VIRTIO_GPU_F_RESOURCE_UUID)) { + if (vhost_dev_has_feature(&g->vhost->dev, VIRTIO_GPU_F_RESOURCE_UUID)) { g->parent_obj.conf.flags |= 1 << VIRTIO_GPU_FLAG_RESOURCE_UUID_ENABLED; } diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index 0d7c83d5e9..2ee061558c 100644 --- a/hw/i386/acpi-build.c +++ b/hw/i386/acpi-build.c @@ -78,6 +78,7 @@ #include "hw/acpi/hmat.h" #include "hw/acpi/viot.h" +#include "hw/acpi/wdat-ich9.h" #include CONFIG_DEVICES @@ -110,6 +111,7 @@ typedef struct AcpiPmInfo { uint16_t cpu_hp_io_base; uint16_t pcihp_io_base; uint16_t pcihp_io_len; + uint64_t tco_io_base; } AcpiPmInfo; typedef struct AcpiMiscInfo { @@ -204,6 +206,7 @@ static void acpi_get_pm_info(MachineState *machine, AcpiPmInfo *pm) pm->pcihp_io_len = 0; pm->smi_on_cpuhp = false; pm->smi_on_cpu_unplug = false; + pm->tco_io_base = 0; assert(obj); init_common_fadt_data(machine, obj, &pm->fadt); @@ -225,6 +228,8 @@ static void acpi_get_pm_info(MachineState *machine, AcpiPmInfo *pm) !!(smi_features & BIT_ULL(ICH9_LPC_SMI_F_CPU_HOTPLUG_BIT)); pm->smi_on_cpu_unplug = !!(smi_features & BIT_ULL(ICH9_LPC_SMI_F_CPU_HOT_UNPLUG_BIT)); + pm->tco_io_base = object_property_get_uint(obj, ACPI_PM_PROP_PM_IO_BASE, + NULL) + ICH9_PMIO_TCO_RLD; } pm->pcihp_io_base = object_property_get_uint(obj, ACPI_PCIHP_IO_BASE_PROP, NULL); @@ -2078,6 +2083,13 @@ void acpi_build(AcpiBuildTables *tables, MachineState *machine) acpi_add_table(table_offsets, tables_blob); build_waet(tables_blob, tables->linker, x86ms->oem_id, x86ms->oem_table_id); + if (pcms->wdat_enabled == true) { + g_assert(pm.tco_io_base); + acpi_add_table(table_offsets, tables_blob); + build_ich9_wdat(tables_blob, tables->linker, x86ms->oem_id, + x86ms->oem_table_id, pm.tco_io_base); + } + /* Add tables supplied by user (if any) */ for (u = acpi_table_first(); u; u = acpi_table_next(u)) { unsigned len = acpi_table_len(u); diff --git a/hw/i386/amd_iommu.c b/hw/i386/amd_iommu.c index 789e09d6f2..79216fb305 100644 --- a/hw/i386/amd_iommu.c +++ b/hw/i386/amd_iommu.c @@ -35,29 +35,6 @@ #include "kvm/kvm_i386.h" #include "qemu/iova-tree.h" -/* used AMD-Vi MMIO registers */ -const char *amdvi_mmio_low[] = { - "AMDVI_MMIO_DEVTAB_BASE", - "AMDVI_MMIO_CMDBUF_BASE", - "AMDVI_MMIO_EVTLOG_BASE", - "AMDVI_MMIO_CONTROL", - "AMDVI_MMIO_EXCL_BASE", - "AMDVI_MMIO_EXCL_LIMIT", - "AMDVI_MMIO_EXT_FEATURES", - "AMDVI_MMIO_PPR_BASE", - "UNHANDLED" -}; -const char *amdvi_mmio_high[] = { - "AMDVI_MMIO_COMMAND_HEAD", - "AMDVI_MMIO_COMMAND_TAIL", - "AMDVI_MMIO_EVTLOG_HEAD", - "AMDVI_MMIO_EVTLOG_TAIL", - "AMDVI_MMIO_STATUS", - "AMDVI_MMIO_PPR_HEAD", - "AMDVI_MMIO_PPR_TAIL", - "UNHANDLED" -}; - struct AMDVIAddressSpace { PCIBus *bus; /* PCIBus (for bus number) */ uint8_t devfn; /* device function */ @@ -215,18 +192,38 @@ static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val) amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val); } +static void amdvi_build_xt_msi_msg(AMDVIState *s, MSIMessage *msg) +{ + union mmio_xt_intr xt_reg; + struct X86IOMMUIrq irq; + + xt_reg.val = amdvi_readq(s, AMDVI_MMIO_XT_GEN_INTR); + + irq.vector = xt_reg.vector; + irq.delivery_mode = xt_reg.delivery_mode; + irq.dest_mode = xt_reg.destination_mode; + irq.dest = (xt_reg.destination_hi << 24) | xt_reg.destination_lo; + irq.trigger_mode = 0; + irq.redir_hint = 0; + + x86_iommu_irq_to_msi_message(&irq, msg); +} + static void amdvi_generate_msi_interrupt(AMDVIState *s) { MSIMessage msg = {}; - MemTxAttrs attrs = { - .requester_id = pci_requester_id(&s->pci->dev) - }; - if (msi_enabled(&s->pci->dev)) { + if (s->intcapxten) { + trace_amdvi_generate_msi_interrupt("XT GEN"); + amdvi_build_xt_msi_msg(s, &msg); + } else if (msi_enabled(&s->pci->dev)) { + trace_amdvi_generate_msi_interrupt("MSI"); msg = msi_get_message(&s->pci->dev, 0); - address_space_stl_le(&address_space_memory, msg.address, msg.data, - attrs, NULL); + } else { + trace_amdvi_generate_msi_interrupt("NO MSI"); + return; } + apic_get_class(NULL)->send_msi(&msg); } static uint32_t get_next_eventlog_entry(AMDVIState *s) @@ -648,6 +645,52 @@ static uint64_t large_pte_page_size(uint64_t pte) return PTE_LARGE_PAGE_SIZE(pte); } +/* + * Validate DTE fields and extract permissions and top level data required to + * initiate the page table walk. + * + * On success, returns 0 and stores: + * - top_level: highest page-table level encoded in DTE[Mode] + * - dte_perms: effective permissions from the DTE + * + * On failure, returns -AMDVI_FR_PT_ROOT_INV. This includes cases where: + * - DTE permissions disallow read AND write + * - DTE[Mode] is invalid for translation + * - IOVA exceeds the address width supported by DTE[Mode] + * In all such cases a page walk must be aborted. + */ +static uint64_t amdvi_get_top_pt_level_and_perms(hwaddr address, uint64_t dte, + uint8_t *top_level, + IOMMUAccessFlags *dte_perms) +{ + *dte_perms = amdvi_get_perms(dte); + if (*dte_perms == IOMMU_NONE) { + return -AMDVI_FR_PT_ROOT_INV; + } + + /* Verifying a valid mode is encoded in DTE */ + *top_level = get_pte_translation_mode(dte); + + /* + * Page Table Root pointer is only valid for GPA->SPA translation on + * supported modes. + */ + if (*top_level == 0 || *top_level > 6) { + return -AMDVI_FR_PT_ROOT_INV; + } + + /* + * If IOVA is larger than the max supported by the highest pgtable level, + * there is nothing to do. + */ + if (address > PT_LEVEL_MAX_ADDR(*top_level)) { + /* IOVA too large for the current DTE */ + return -AMDVI_FR_PT_ROOT_INV; + } + + return 0; +} + /* * Helper function to fetch a PTE using AMD v1 pgtable format. * On successful page walk, returns 0 and pte parameter points to a valid PTE. @@ -662,40 +705,49 @@ static uint64_t large_pte_page_size(uint64_t pte) static uint64_t fetch_pte(AMDVIAddressSpace *as, hwaddr address, uint64_t dte, uint64_t *pte, hwaddr *page_size) { - IOMMUAccessFlags perms = amdvi_get_perms(dte); - - uint8_t level, mode; uint64_t pte_addr; + uint8_t pt_level, next_pt_level; + IOMMUAccessFlags perms; + int ret; - *pte = dte; *page_size = 0; - if (perms == IOMMU_NONE) { + /* + * Verify the DTE is properly configured before page walk, and extract + * top pagetable level and permissions. + */ + ret = amdvi_get_top_pt_level_and_perms(address, dte, &pt_level, &perms); + if (ret < 0) { + return ret; + } + + /* + * Retrieve the top pagetable entry by following the DTE Page Table Root + * Pointer and indexing the top level table using the IOVA from the request. + */ + pte_addr = NEXT_PTE_ADDR(dte, pt_level, address); + *pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn); + + if (*pte == (uint64_t)-1) { + /* + * A returned PTE of -1 here indicates a failure to read the top level + * page table from guest memory. A page walk is not possible and page + * size must be returned as 0. + */ return -AMDVI_FR_PT_ROOT_INV; } /* - * The Linux kernel driver initializes the default mode to 3, corresponding - * to a 39-bit GPA space, where each entry in the pagetable translates to a - * 1GB (2^30) page size. + * Calculate page size for the top level page table entry. + * This ensures correct results for a single level Page Table setup. */ - level = mode = get_pte_translation_mode(dte); - assert(mode > 0 && mode < 7); + *page_size = PTE_LEVEL_PAGE_SIZE(pt_level); /* - * If IOVA is larger than the max supported by the current pgtable level, - * there is nothing to do. + * The root page table entry and its level have been determined. Begin the + * page walk. */ - if (address > PT_LEVEL_MAX_ADDR(mode - 1)) { - /* IOVA too large for the current DTE */ - return -AMDVI_FR_PT_ROOT_INV; - } - - do { - level -= 1; - - /* Update the page_size */ - *page_size = PTE_LEVEL_PAGE_SIZE(level); + while (pt_level > 0) { /* Permission bits are ANDed at every level, including the DTE */ perms &= amdvi_get_perms(*pte); @@ -708,37 +760,38 @@ static uint64_t fetch_pte(AMDVIAddressSpace *as, hwaddr address, uint64_t dte, return 0; } + next_pt_level = PTE_NEXT_LEVEL(*pte); + /* Large or Leaf PTE found */ - if (PTE_NEXT_LEVEL(*pte) == 7 || PTE_NEXT_LEVEL(*pte) == 0) { + if (next_pt_level == 0 || next_pt_level == 7) { /* Leaf PTE found */ break; } + /* Next level must always be less than current level */ + if (pt_level <= next_pt_level) { + return -AMDVI_FR_PT_ENTRY_INV; + } + pt_level = next_pt_level; + /* - * Index the pgtable using the IOVA bits corresponding to current level - * and walk down to the lower level. + * The current entry is a Page Directory Entry. Descend to the lower + * page table level encoded in current pte, and index the new table + * using the appropriate IOVA bits to retrieve the new entry. */ - pte_addr = NEXT_PTE_ADDR(*pte, level, address); + *page_size = PTE_LEVEL_PAGE_SIZE(pt_level); + + pte_addr = NEXT_PTE_ADDR(*pte, pt_level, address); *pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn); if (*pte == (uint64_t)-1) { - /* - * A returned PTE of -1 indicates a failure to read the page table - * entry from guest memory. - */ - if (level == mode - 1) { - /* Failure to retrieve the Page Table from Root Pointer */ - *page_size = 0; - return -AMDVI_FR_PT_ROOT_INV; - } else { - /* Failure to read PTE. Page walk skips a page_size chunk */ - return -AMDVI_FR_PT_ENTRY_INV; - } + /* Failure to read PTE. Page walk skips a page_size chunk */ + return -AMDVI_FR_PT_ENTRY_INV; } - } while (level > 0); + } + + assert(PTE_NEXT_LEVEL(*pte) == 0 || PTE_NEXT_LEVEL(*pte) == 7); - assert(PTE_NEXT_LEVEL(*pte) == 0 || PTE_NEXT_LEVEL(*pte) == 7 || - level == 0); /* * Page walk ends when Next Level field on PTE shows that either a leaf PTE * or a series of large PTEs have been reached. In the latter case, even if @@ -1475,40 +1528,41 @@ static void amdvi_cmdbuf_run(AMDVIState *s) trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf); amdvi_cmdbuf_exec(s); s->cmdbuf_head += AMDVI_COMMAND_SIZE; - amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head); /* wrap head pointer */ if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) { s->cmdbuf_head = 0; } + amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head); } } -static inline uint8_t amdvi_mmio_get_index(hwaddr addr) +static inline +const char *amdvi_mmio_get_name(hwaddr addr) { - uint8_t index = (addr & ~0x2000) / 8; - - if ((addr & 0x2000)) { - /* high table */ - index = index >= AMDVI_MMIO_REGS_HIGH ? AMDVI_MMIO_REGS_HIGH : index; - } else { - index = index >= AMDVI_MMIO_REGS_LOW ? AMDVI_MMIO_REGS_LOW : index; + /* Return MMIO names as string literals */ + switch (addr) { +#define MMIO_REG_TO_STRING(mmio_reg) case mmio_reg: return #mmio_reg + MMIO_REG_TO_STRING(AMDVI_MMIO_DEVICE_TABLE); + MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_BASE); + MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_BASE); + MMIO_REG_TO_STRING(AMDVI_MMIO_CONTROL); + MMIO_REG_TO_STRING(AMDVI_MMIO_EXCL_BASE); + MMIO_REG_TO_STRING(AMDVI_MMIO_EXCL_LIMIT); + MMIO_REG_TO_STRING(AMDVI_MMIO_EXT_FEATURES); + MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_HEAD); + MMIO_REG_TO_STRING(AMDVI_MMIO_COMMAND_TAIL); + MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_HEAD); + MMIO_REG_TO_STRING(AMDVI_MMIO_EVENT_TAIL); + MMIO_REG_TO_STRING(AMDVI_MMIO_STATUS); + MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_BASE); + MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_HEAD); + MMIO_REG_TO_STRING(AMDVI_MMIO_PPR_TAIL); + MMIO_REG_TO_STRING(AMDVI_MMIO_XT_GEN_INTR); +#undef MMIO_REG_TO_STRING + default: + return "UNHANDLED"; } - - return index; -} - -static void amdvi_mmio_trace_read(hwaddr addr, unsigned size) -{ - uint8_t index = amdvi_mmio_get_index(addr); - trace_amdvi_mmio_read(amdvi_mmio_low[index], addr, size, addr & ~0x07); -} - -static void amdvi_mmio_trace_write(hwaddr addr, unsigned size, uint64_t val) -{ - uint8_t index = amdvi_mmio_get_index(addr); - trace_amdvi_mmio_write(amdvi_mmio_low[index], addr, size, val, - addr & ~0x07); } static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size) @@ -1528,7 +1582,7 @@ static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size) } else if (size == 8) { val = amdvi_readq(s, addr); } - amdvi_mmio_trace_read(addr, size); + trace_amdvi_mmio_read(amdvi_mmio_get_name(addr), addr, size, addr & ~0x07); return val; } @@ -1546,6 +1600,17 @@ static void amdvi_handle_control_write(AMDVIState *s) s->cmdbuf_enabled = s->enabled && !!(control & AMDVI_MMIO_CONTROL_CMDBUFLEN); s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN); + s->xten = !!(control & AMDVI_MMIO_CONTROL_XTEN) && s->xtsup && + s->ga_enabled; + /* + * IntCapXTEn controls whether IOMMU-originated interrupts are sent based + * on the information in XT IOMMU Interrupt Control Registers rather than + * the IOMMU’s MSI capability registers. Therefore it requires IOMMU + * x2APIC support capabilities (i.e. XTSup=1), but it is independent of + * whether a driver chooses to enable x2APIC mode for interrupt remapping + * (i.e. XTEn=1). + */ + s->intcapxten = !!(control & AMDVI_MMIO_CONTROL_INTCAPXTEN) && s->xtsup; /* update the flags depending on the control register */ if (s->cmdbuf_enabled) { @@ -1578,7 +1643,8 @@ static inline void amdvi_handle_devtab_write(AMDVIState *s) static inline void amdvi_handle_cmdhead_write(AMDVIState *s) { s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD) - & AMDVI_MMIO_CMDBUF_HEAD_MASK; + & AMDVI_MMIO_CMDBUF_HEAD_MASK + & (s->cmdbuf_len * AMDVI_COMMAND_SIZE - 1); amdvi_cmdbuf_run(s); } @@ -1594,7 +1660,8 @@ static inline void amdvi_handle_cmdbase_write(AMDVIState *s) static inline void amdvi_handle_cmdtail_write(AMDVIState *s) { s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL) - & AMDVI_MMIO_CMDBUF_TAIL_MASK; + & AMDVI_MMIO_CMDBUF_TAIL_MASK + & (s->cmdbuf_len * AMDVI_COMMAND_SIZE - 1); amdvi_cmdbuf_run(s); } @@ -1684,7 +1751,8 @@ static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val, return; } - amdvi_mmio_trace_write(addr, size, val); + trace_amdvi_mmio_write(amdvi_mmio_get_name(addr), addr, size, val, offset); + switch (addr & ~0x07) { case AMDVI_MMIO_CONTROL: amdvi_mmio_reg_write(s, size, val, addr); @@ -1752,6 +1820,9 @@ static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val, case AMDVI_MMIO_STATUS: amdvi_mmio_reg_write(s, size, val, addr); break; + case AMDVI_MMIO_XT_GEN_INTR: + amdvi_mmio_reg_write(s, size, val, addr); + break; } } @@ -2018,7 +2089,7 @@ static int amdvi_int_remap_ga(AMDVIState *iommu, irq->vector = irte.hi.fields.vector; irq->dest_mode = irte.lo.fields_remap.dm; irq->redir_hint = irte.lo.fields_remap.rq_eoi; - if (iommu->xtsup) { + if (iommu->xten) { irq->dest = irte.lo.fields_remap.destination | (irte.hi.fields.destination_hi << 24); } else { @@ -2401,6 +2472,8 @@ static void amdvi_init(AMDVIState *s) s->mmio_enabled = false; s->enabled = false; s->cmdbuf_enabled = false; + s->xten = false; + s->intcapxten = false; /* reset MMIO */ memset(s->mmior, 0, AMDVI_MMIO_SIZE); @@ -2465,6 +2538,17 @@ static void amdvi_sysbus_reset(DeviceState *dev) amdvi_reset_address_translation_all(s); } +static const VMStateDescription vmstate_xt = { + .name = "amd-iommu-xt", + .version_id = 1, + .minimum_version_id = 1, + .fields = (VMStateField[]) { + VMSTATE_BOOL(xten, AMDVIState), + VMSTATE_BOOL(intcapxten, AMDVIState), + VMSTATE_END_OF_LIST() + } +}; + static const VMStateDescription vmstate_amdvi_sysbus_migratable = { .name = "amd-iommu", .version_id = 1, @@ -2509,7 +2593,11 @@ static const VMStateDescription vmstate_amdvi_sysbus_migratable = { VMSTATE_UINT8_ARRAY(romask, AMDVIState, AMDVI_MMIO_SIZE), VMSTATE_UINT8_ARRAY(w1cmask, AMDVIState, AMDVI_MMIO_SIZE), VMSTATE_END_OF_LIST() - } + }, + .subsections = (const VMStateDescription *const []) { + &vmstate_xt, + NULL + } }; static void amdvi_sysbus_realize(DeviceState *dev, Error **errp) diff --git a/hw/i386/amd_iommu.h b/hw/i386/amd_iommu.h index 302ccca512..3cab04a6d4 100644 --- a/hw/i386/amd_iommu.h +++ b/hw/i386/amd_iommu.h @@ -45,10 +45,6 @@ #define AMDVI_CAPAB_FLAG_IOTLBSUP (1 << 24) #define AMDVI_CAPAB_INIT_TYPE (3 << 16) -/* No. of used MMIO registers */ -#define AMDVI_MMIO_REGS_HIGH 7 -#define AMDVI_MMIO_REGS_LOW 8 - /* MMIO registers */ #define AMDVI_MMIO_DEVICE_TABLE 0x0000 #define AMDVI_MMIO_COMMAND_BASE 0x0008 @@ -57,6 +53,7 @@ #define AMDVI_MMIO_EXCL_BASE 0x0020 #define AMDVI_MMIO_EXCL_LIMIT 0x0028 #define AMDVI_MMIO_EXT_FEATURES 0x0030 +#define AMDVI_MMIO_XT_GEN_INTR 0x0170 #define AMDVI_MMIO_COMMAND_HEAD 0x2000 #define AMDVI_MMIO_COMMAND_TAIL 0x2008 #define AMDVI_MMIO_EVENT_HEAD 0x2010 @@ -106,6 +103,8 @@ #define AMDVI_MMIO_CONTROL_COMWAITINTEN (1ULL << 4) #define AMDVI_MMIO_CONTROL_CMDBUFLEN (1ULL << 12) #define AMDVI_MMIO_CONTROL_GAEN (1ULL << 17) +#define AMDVI_MMIO_CONTROL_XTEN (1ULL << 50) +#define AMDVI_MMIO_CONTROL_INTCAPXTEN (1ULL << 51) /* MMIO status register bits */ #define AMDVI_MMIO_STATUS_CMDBUF_RUN (1 << 4) @@ -186,17 +185,16 @@ #define IOMMU_PTE_PRESENT(pte) ((pte) & AMDVI_PTE_PR) -/* Using level=0 for leaf PTE at 4K page size */ -#define PT_LEVEL_SHIFT(level) (12 + ((level) * 9)) +/* Using level=1 for leaf PTE at 4K page size */ +#define PT_LEVEL_SHIFT(level) (12 + (((level) - 1) * 9)) /* Return IOVA bit group used to index the Page Table at specific level */ #define PT_LEVEL_INDEX(level, iova) (((iova) >> PT_LEVEL_SHIFT(level)) & \ GENMASK64(8, 0)) -/* Return the max address for a specified level i.e. max_oaddr */ -#define PT_LEVEL_MAX_ADDR(x) (((x) < 5) ? \ - ((1ULL << PT_LEVEL_SHIFT((x + 1))) - 1) : \ - (~(0ULL))) +/* Return the maximum output address for a specified page table level */ +#define PT_LEVEL_MAX_ADDR(level) (((level) > 5) ? (~(0ULL)) : \ + ((1ULL << PT_LEVEL_SHIFT((level) + 1)) - 1)) /* Extract the NextLevel field from PTE/PDE */ #define PTE_NEXT_LEVEL(pte) (((pte) & AMDVI_PTE_NEXT_LEVEL_MASK) >> 9) @@ -342,6 +340,20 @@ struct irte_ga { union irte_ga_hi hi; }; +union mmio_xt_intr { + uint64_t val; + struct { + uint64_t rsvd_1:2, + destination_mode:1, + rsvd_2:5, + destination_lo:24, + vector:8, + delivery_mode:1, + rsvd_3:15, + destination_hi:8; + }; +}; + #define TYPE_AMD_IOMMU_DEVICE "amd-iommu" OBJECT_DECLARE_SIMPLE_TYPE(AMDVIState, AMD_IOMMU_DEVICE) @@ -418,7 +430,9 @@ struct AMDVIState { /* Interrupt remapping */ bool ga_enabled; - bool xtsup; + bool xtsup; /* xtsup=on command line */ + bool xten; /* guest controlled, x2apic mode enabled */ + bool intcapxten; /* guest controlled, IOMMU x2apic interrupts enabled */ /* DMA address translation */ bool dma_remap; diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c index b784c5f10a..744cdfd2e6 100644 --- a/hw/i386/intel_iommu.c +++ b/hw/i386/intel_iommu.c @@ -21,6 +21,7 @@ #include "qemu/osdep.h" #include "qemu/error-report.h" +#include "qemu/log.h" #include "qemu/main-loop.h" #include "qapi/error.h" #include "hw/core/sysbus.h" @@ -42,12 +43,6 @@ #include "migration/vmstate.h" #include "trace.h" -/* context entry operations */ -#define VTD_CE_GET_PASID_DIR_TABLE(ce) \ - ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK) -#define VTD_CE_GET_PRE(ce) \ - ((ce)->val[0] & VTD_SM_CONTEXT_ENTRY_PRE) - /* * Paging mode for first-stage translation (VTD spec Figure 9-6) * 00: 4-level paging, 01: 5-level paging @@ -92,8 +87,6 @@ static void vtd_pasid_cache_reset_locked(IntelIOMMUState *s) VTDPASIDCacheEntry *pc_entry = &vtd_as->pasid_cache_entry; if (pc_entry->valid) { pc_entry->valid = false; - /* It's fatal to get failure during reset */ - vtd_propagate_guest_pasid(vtd_as, &error_fatal); } } } @@ -397,6 +390,8 @@ static void vtd_reset_caches(IntelIOMMUState *s) vtd_reset_context_cache_locked(s); vtd_pasid_cache_reset_locked(s); vtd_iommu_unlock(s); + + vtd_accel_pasid_cache_reset(s); } static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) @@ -831,18 +826,12 @@ static inline bool vtd_pe_type_check(IntelIOMMUState *s, VTDPASIDEntry *pe) } } -static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire) -{ - return pdire->val & 1; -} - /** * Caller of this function should check present bit if wants * to use pdir entry for further usage except for fpd bit check. */ -static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, - uint32_t pasid, - VTDPASIDDirEntry *pdire) +int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, uint32_t pasid, + VTDPASIDDirEntry *pdire) { uint32_t index; dma_addr_t addr, entry_size; @@ -860,15 +849,8 @@ static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, return 0; } -static inline bool vtd_pe_present(VTDPASIDEntry *pe) -{ - return pe->val[0] & VTD_PASID_ENTRY_P; -} - -static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, - uint32_t pasid, - dma_addr_t addr, - VTDPASIDEntry *pe) +int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, uint32_t pasid, + dma_addr_t addr, VTDPASIDEntry *pe) { uint8_t pgtt; uint32_t index; @@ -957,12 +939,8 @@ static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s, static int vtd_ce_get_pasid_entry(IntelIOMMUState *s, VTDContextEntry *ce, VTDPASIDEntry *pe, uint32_t pasid) { - dma_addr_t pasid_dir_base; + dma_addr_t pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); - if (pasid == PCI_NO_PASID) { - pasid = PASID_0; - } - pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); return vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe); } @@ -972,15 +950,10 @@ static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s, uint32_t pasid) { int ret; - dma_addr_t pasid_dir_base; + dma_addr_t pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); VTDPASIDDirEntry pdire; VTDPASIDEntry pe; - if (pasid == PCI_NO_PASID) { - pasid = PASID_0; - } - pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); - /* * No present bit check since fpd is meaningful even * if the present bit is clear. @@ -1520,14 +1493,14 @@ static int vtd_ce_pasid_0_check(IntelIOMMUState *s, VTDContextEntry *ce) /* * Make sure in Scalable Mode, a present context entry - * has valid pasid entry setting at PASID_0. + * has valid pasid entry setting at IOMMU_NO_PASID. */ - return vtd_ce_get_pasid_entry(s, ce, &pe, PASID_0); + return vtd_ce_get_pasid_entry(s, ce, &pe, IOMMU_NO_PASID); } /* Map a device to its corresponding domain (context-entry) */ -static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, - uint8_t devfn, VTDContextEntry *ce) +int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, + uint8_t devfn, VTDContextEntry *ce) { VTDRootEntry re; int ret_fr; @@ -1583,7 +1556,7 @@ static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, } } else { /* - * Check if the programming of pasid setting of PASID_0 + * Check if the programming of pasid setting of IOMMU_NO_PASID * is valid, and thus avoids to check pasid entry fetching * result in future helper function calling. */ @@ -1769,7 +1742,7 @@ static bool vtd_switch_address_space(VTDAddressSpace *as) * * Need to disable ir for as with PASID. */ - if (as->pasid != PCI_NO_PASID) { + if (as->pasid != IOMMU_NO_PASID) { memory_region_set_enabled(&as->iommu_ir, false); } else { memory_region_set_enabled(&as->iommu_ir, true); @@ -1799,7 +1772,7 @@ static bool vtd_switch_address_space(VTDAddressSpace *as) * We enable per as memory region (iommu_ir_fault) for catching * the translation for interrupt range through PASID + PT. */ - if (pt && as->pasid != PCI_NO_PASID) { + if (pt && as->pasid != IOMMU_NO_PASID) { memory_region_set_enabled(&as->iommu_ir_fault, true); } else { memory_region_set_enabled(&as->iommu_ir_fault, false); @@ -1909,9 +1882,9 @@ static VTDAddressSpace *vtd_get_as_by_sid_and_pasid(IntelIOMMUState *s, vtd_find_as_by_sid_and_pasid, &key); } -static VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid) +VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid) { - return vtd_get_as_by_sid_and_pasid(s, sid, PCI_NO_PASID); + return vtd_get_as_by_sid_and_pasid(s, sid, IOMMU_NO_PASID); } static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) @@ -2128,6 +2101,7 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, bool is_fpd_set = false; bool reads = true; bool writes = true; + bool is_pasid = pasid != IOMMU_NO_PASID; uint8_t access_flags, pgtt; VTDIOTLBEntry *iotlb_entry; uint64_t xlat, size; @@ -2140,10 +2114,6 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, vtd_iommu_lock(s); - if (pasid == PCI_NO_PASID && s->root_scalable) { - pasid = PASID_0; - } - /* Try to fetch pte from IOTLB */ iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); if (iotlb_entry) { @@ -2254,7 +2224,7 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, if (ret_fr) { if (!vtd_is_recoverable_fault(-ret_fr, iommu_idx)) { vtd_report_fault(s, -ret_fr, is_fpd_set, source_id, - addr, is_write, pasid != PCI_NO_PASID, pasid); + addr, is_write, is_pasid, pasid); } goto error; } @@ -2506,9 +2476,9 @@ static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) } /* - * There is no pasid field in iotlb invalidation descriptor, so PCI_NO_PASID + * There is no pasid field in iotlb invalidation descriptor, so IOMMU_NO_PASID * is passed as parameter. Piotlb invalidation supports pasid, pasid in its - * descriptor is passed which should not be PCI_NO_PASID. + * descriptor is passed. */ static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, uint16_t domain_id, hwaddr addr, @@ -2522,48 +2492,41 @@ static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), vtd_as->devfn, &ce); - if (!ret && domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { - /* - * In legacy mode, vtd_as->pasid == pasid is always true. - * In scalable mode, for vtd address space backing a PCI - * device without pasid, needs to compare pasid with - * PASID_0 of this device. - */ - if (!(vtd_as->pasid == pasid || - (vtd_as->pasid == PCI_NO_PASID && pasid == PASID_0))) { - continue; - } + if (ret || vtd_as->pasid != pasid || + domain_id != vtd_get_domain_id(s, &ce, pasid)) { + continue; + } - if (vtd_as_has_map_notifier(vtd_as)) { - /* - * When first stage translation is off, as long as we have MAP - * notifications registered in any of our IOMMU notifiers, - * we need to sync the shadow page table. Otherwise VFIO - * device attaches to nested page table instead of shadow - * page table, so no need to sync. - */ - if (!s->fsts || !s->root_scalable) { - vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); - } - } else { - /* - * For UNMAP-only notifiers, we don't need to walk the - * page tables. We just deliver the PSI down to - * invalidate caches. - */ - const IOMMUTLBEvent event = { - .type = IOMMU_NOTIFIER_UNMAP, - .entry = { - .target_as = &address_space_memory, - .iova = addr, - .translated_addr = 0, - .addr_mask = size - 1, - .perm = IOMMU_NONE, - .pasid = vtd_as->pasid, - }, - }; - memory_region_notify_iommu(&vtd_as->iommu, 0, event); + if (vtd_as_has_map_notifier(vtd_as)) { + /* + * When first stage translation is off, as long as we have MAP + * notifications registered in any of our IOMMU notifiers, + * we need to sync the shadow page table. Otherwise VFIO + * device attaches to nested page table instead of shadow + * page table, so no need to sync. + */ + if (!s->fsts || !s->root_scalable) { + vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); } + } else { + /* + * For UNMAP-only notifiers, we don't need to walk the + * page tables. We just deliver the PSI down to + * invalidate caches. + */ + const IOMMUTLBEvent event = { + .type = IOMMU_NOTIFIER_UNMAP, + .entry = { + .target_as = &address_space_memory, + .iova = addr, + .translated_addr = 0, + .addr_mask = size - 1, + .perm = IOMMU_NONE, + /* Other sub-systems use PCI pasid */ + .pasid = pasid == IOMMU_NO_PASID ? PCI_NO_PASID : pasid, + }, + }; + memory_region_notify_iommu(&vtd_as->iommu, 0, event); } } } @@ -2582,7 +2545,7 @@ static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, vtd_iommu_lock(s); g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); vtd_iommu_unlock(s); - vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, PCI_NO_PASID); + vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, IOMMU_NO_PASID); } /* Flush IOTLB @@ -3026,6 +2989,7 @@ static void vtd_piotlb_pasid_invalidate(IntelIOMMUState *s, VTDIOTLBPageInvInfo info; VTDAddressSpace *vtd_as; VTDContextEntry ce; + int ret; info.domain_id = domain_id; info.pasid = pasid; @@ -3038,17 +3002,15 @@ static void vtd_piotlb_pasid_invalidate(IntelIOMMUState *s, vtd_iommu_unlock(s); QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { - if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), - vtd_as->devfn, &ce) && - domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { - if ((vtd_as->pasid != PCI_NO_PASID || pasid != PASID_0) && - vtd_as->pasid != pasid) { - continue; - } + ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), + vtd_as->devfn, &ce); + if (ret || vtd_as->pasid != pasid || + domain_id != vtd_get_domain_id(s, &ce, pasid)) { + continue; + } - if (!s->fsts || !vtd_as_has_map_notifier(vtd_as)) { - vtd_address_space_sync(vtd_as); - } + if (!s->fsts || !vtd_as_has_map_notifier(vtd_as)) { + vtd_address_space_sync(vtd_as); } } } @@ -3113,10 +3075,9 @@ static bool vtd_process_piotlb_desc(IntelIOMMUState *s, return true; } -static inline int vtd_dev_get_pe_from_pasid(VTDAddressSpace *vtd_as, - VTDPASIDEntry *pe) +int vtd_dev_get_pe_from_pasid(IntelIOMMUState *s, PCIBus *bus, uint8_t devfn, + uint32_t pasid, VTDPASIDEntry *pe) { - IntelIOMMUState *s = vtd_as->iommu_state; VTDContextEntry ce; int ret; @@ -3124,18 +3085,12 @@ static inline int vtd_dev_get_pe_from_pasid(VTDAddressSpace *vtd_as, return -VTD_FR_RTADDR_INV_TTM; } - ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), vtd_as->devfn, - &ce); + ret = vtd_dev_to_context_entry(s, pci_bus_num(bus), devfn, &ce); if (ret) { return ret; } - return vtd_ce_get_pasid_entry(s, &ce, pe, vtd_as->pasid); -} - -static int vtd_pasid_entry_compare(VTDPASIDEntry *p1, VTDPASIDEntry *p2) -{ - return memcmp(p1, p2, sizeof(*p1)); + return vtd_ce_get_pasid_entry(s, &ce, pe, pasid); } /* Update or invalidate pasid cache based on the pasid entry in guest memory. */ @@ -3148,10 +3103,9 @@ static void vtd_pasid_cache_sync_locked(gpointer key, gpointer value, VTDPASIDEntry pe; IOMMUNotifier *n; uint16_t did; - const char *err_prefix = "Attaching to HWPT failed: "; - Error *local_err = NULL; - if (vtd_dev_get_pe_from_pasid(vtd_as, &pe)) { + if (vtd_dev_get_pe_from_pasid(vtd_as->iommu_state, vtd_as->bus, + vtd_as->devfn, vtd_as->pasid, &pe)) { if (!pc_entry->valid) { return; } @@ -3170,9 +3124,6 @@ static void vtd_pasid_cache_sync_locked(gpointer key, gpointer value, vtd_address_space_unmap(vtd_as, n); } vtd_switch_address_space(vtd_as); - - err_prefix = "Detaching from HWPT failed: "; - goto do_bind_unbind; } /* @@ -3200,20 +3151,12 @@ static void vtd_pasid_cache_sync_locked(gpointer key, gpointer value, if (!pc_entry->valid) { pc_entry->pasid_entry = pe; pc_entry->valid = true; - } else if (vtd_pasid_entry_compare(&pe, &pc_entry->pasid_entry)) { - err_prefix = "Replacing HWPT attachment failed: "; - } else { + } else if (!vtd_pasid_entry_compare(&pe, &pc_entry->pasid_entry)) { return; } vtd_switch_address_space(vtd_as); vtd_address_space_sync(vtd_as); - -do_bind_unbind: - /* TODO: Fault event injection into guest, report error to QEMU for now */ - if (!vtd_propagate_guest_pasid(vtd_as, &local_err)) { - error_reportf_err(local_err, "%s", err_prefix); - } } static void vtd_pasid_cache_sync(IntelIOMMUState *s, VTDPASIDCacheInfo *pc_info) @@ -3226,6 +3169,8 @@ static void vtd_pasid_cache_sync(IntelIOMMUState *s, VTDPASIDCacheInfo *pc_info) g_hash_table_foreach(s->vtd_address_spaces, vtd_pasid_cache_sync_locked, pc_info); vtd_iommu_unlock(s); + + vtd_accel_pasid_cache_sync(s, pc_info); } static void vtd_replay_pasid_bindings_all(IntelIOMMUState *s) @@ -3263,7 +3208,7 @@ static bool vtd_process_pasid_desc(IntelIOMMUState *s, /* PASID selective implies a DID selective */ trace_vtd_inv_desc_pasid_cache_psi(did, pasid); pc_info.did = did; - pc_info.pasid = pasid ?: PCI_NO_PASID; + pc_info.pasid = pasid; break; case VTD_INV_DESC_PASIDC_G_GLOBAL: @@ -3315,6 +3260,7 @@ static void do_invalidate_device_tlb(VTDAddressSpace *vtd_dev_as, * ... */ + uint32_t pasid = vtd_dev_as->pasid; IOMMUTLBEvent event; uint64_t sz; @@ -3331,7 +3277,8 @@ static void do_invalidate_device_tlb(VTDAddressSpace *vtd_dev_as, event.entry.iova = addr; event.entry.perm = IOMMU_NONE; event.entry.translated_addr = 0; - event.entry.pasid = vtd_dev_as->pasid; + /* Other sub-systems use PCI pasid */ + event.entry.pasid = pasid == IOMMU_NO_PASID ? PCI_NO_PASID : pasid; memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event); } @@ -3359,7 +3306,7 @@ static bool vtd_process_device_piotlb_desc(IntelIOMMUState *s, sid = VTD_INV_DESC_PASID_DEVICE_IOTLB_SID(inv_desc->lo); if (global) { QLIST_FOREACH(vtd_dev_as, &s->vtd_as_with_notifiers, next) { - if ((vtd_dev_as->pasid != PCI_NO_PASID) && + if ((vtd_dev_as->pasid != IOMMU_NO_PASID) && (PCI_BUILD_BDF(pci_bus_num(vtd_dev_as->bus), vtd_dev_as->devfn) == sid)) { do_invalidate_device_tlb(vtd_dev_as, size, addr); @@ -3713,7 +3660,6 @@ static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) break; case DMAR_RTADDR_REG_HI: - assert(size == 4); val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32; break; @@ -3728,12 +3674,10 @@ static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) break; case DMAR_IQA_REG_HI: - assert(size == 4); val = s->iq >> 32; break; case DMAR_PEUADDR_REG: - assert(size == 4); val = vtd_get_long_raw(s, DMAR_PEUADDR_REG); break; @@ -3779,7 +3723,6 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_CCMD_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); vtd_handle_ccmd_write(s); break; @@ -3795,13 +3738,11 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_IOTLB_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); vtd_handle_iotlb_write(s); break; case DMAR_PEUADDR_REG: - assert(size == 4); vtd_set_long(s, addr, val); break; @@ -3815,27 +3756,35 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_IVA_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); break; /* Fault Status Register, 32-bit */ case DMAR_FSTS_REG: - assert(size == 4); vtd_set_long(s, addr, val); vtd_handle_fsts_write(s); break; /* Fault Event Control Register, 32-bit */ case DMAR_FECTL_REG: - assert(size == 4); + /* + * 32-bit register at an 8-byte-aligned offset: a well-formed + * 8-byte guest access reaches this handler. vtd_set_long() + * takes uint32_t and truncates the high half -- undefined per + * the VT-d spec but harmless here. Flag it under + * -d guest_errors so the guest-side bug surfaces. + */ + if (size != 4) { + qemu_log_mask(LOG_GUEST_ERROR, + "%s: invalid %u-byte access to 32-bit reg " + "addr=0x%" PRIx64 "\n", __func__, size, addr); + } vtd_set_long(s, addr, val); vtd_handle_fectl_write(s); break; /* Fault Event Data Register, 32-bit */ case DMAR_FEDATA_REG: - assert(size == 4); vtd_set_long(s, addr, val); break; @@ -3854,13 +3803,11 @@ static void vtd_mem_write(void *opaque, hwaddr addr, /* Fault Event Upper Address Register, 32-bit */ case DMAR_FEUADDR_REG: - assert(size == 4); vtd_set_long(s, addr, val); break; /* Protected Memory Enable Register, 32-bit */ case DMAR_PMEN_REG: - assert(size == 4); vtd_set_long(s, addr, val); break; @@ -3874,7 +3821,6 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_RTADDR_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); break; @@ -3889,7 +3835,6 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_IQT_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); /* 19:63 of IQT_REG is RsvdZ, do nothing here */ break; @@ -3905,39 +3850,57 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_IQA_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); break; /* Invalidation Completion Status Register, 32-bit */ case DMAR_ICS_REG: - assert(size == 4); vtd_set_long(s, addr, val); vtd_handle_ics_write(s); break; /* Invalidation Event Control Register, 32-bit */ case DMAR_IECTL_REG: - assert(size == 4); + /* + * 32-bit register at an 8-byte-aligned offset: a well-formed + * 8-byte guest access reaches this handler. vtd_set_long() + * takes uint32_t and truncates the high half -- undefined per + * the VT-d spec but harmless here. Flag it under + * -d guest_errors so the guest-side bug surfaces. + */ + if (size != 4) { + qemu_log_mask(LOG_GUEST_ERROR, + "%s: invalid %u-byte access to 32-bit reg " + "addr=0x%" PRIx64 "\n", __func__, size, addr); + } vtd_set_long(s, addr, val); vtd_handle_iectl_write(s); break; /* Invalidation Event Data Register, 32-bit */ case DMAR_IEDATA_REG: - assert(size == 4); vtd_set_long(s, addr, val); break; /* Invalidation Event Address Register, 32-bit */ case DMAR_IEADDR_REG: - assert(size == 4); + /* + * 32-bit register at an 8-byte-aligned offset: a well-formed + * 8-byte guest access reaches this handler. vtd_set_long() + * takes uint32_t and truncates the high half -- undefined per + * the VT-d spec but harmless here. Flag it under + * -d guest_errors so the guest-side bug surfaces. + */ + if (size != 4) { + qemu_log_mask(LOG_GUEST_ERROR, + "%s: invalid %u-byte access to 32-bit reg " + "addr=0x%" PRIx64 "\n", __func__, size, addr); + } vtd_set_long(s, addr, val); break; /* Invalidation Event Upper Address Register, 32-bit */ case DMAR_IEUADDR_REG: - assert(size == 4); vtd_set_long(s, addr, val); break; @@ -3951,7 +3914,6 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_FRCD_REG_0_1: - assert(size == 4); vtd_set_long(s, addr, val); break; @@ -3966,7 +3928,6 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_FRCD_REG_0_3: - assert(size == 4); vtd_set_long(s, addr, val); /* May clear bit 127 (Fault), update PPF */ vtd_update_fsts_ppf(s); @@ -3981,18 +3942,27 @@ static void vtd_mem_write(void *opaque, hwaddr addr, break; case DMAR_IRTA_REG_HI: - assert(size == 4); vtd_set_long(s, addr, val); break; case DMAR_PRS_REG: - assert(size == 4); vtd_set_long(s, addr, val); vtd_handle_prs_write(s); break; case DMAR_PECTL_REG: - assert(size == 4); + /* + * 32-bit register at an 8-byte-aligned offset: a well-formed + * 8-byte guest access reaches this handler. vtd_set_long() + * takes uint32_t and truncates the high half -- undefined per + * the VT-d spec but harmless here. Flag it under + * -d guest_errors so the guest-side bug surfaces. + */ + if (size != 4) { + qemu_log_mask(LOG_GUEST_ERROR, + "%s: invalid %u-byte access to 32-bit reg " + "addr=0x%" PRIx64 "\n", __func__, size, addr); + } vtd_set_long(s, addr, val); vtd_handle_pectl_write(s); break; @@ -4007,13 +3977,12 @@ static void vtd_mem_write(void *opaque, hwaddr addr, } static void vtd_prepare_identity_entry(hwaddr addr, IOMMUAccessFlags perm, - uint32_t pasid, IOMMUTLBEntry *iotlb) + IOMMUTLBEntry *iotlb) { iotlb->iova = addr & VTD_PAGE_MASK_4K; iotlb->translated_addr = addr & VTD_PAGE_MASK_4K; iotlb->addr_mask = ~VTD_PAGE_MASK_4K; iotlb->perm = perm; - iotlb->pasid = pasid; } static inline void vtd_prepare_error_entry(IOMMUTLBEntry *entry) @@ -4025,6 +3994,10 @@ static inline void vtd_prepare_error_entry(IOMMUTLBEntry *entry) entry->pasid = PCI_NO_PASID; } +/* + * This function returns translation result to other sub-system such as PCI, + * so iommu pasid is converted to PCI pasid and returned in IOMMUTLBEntry. + */ static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, IOMMUAccessFlags flag, int iommu_idx) { @@ -4033,7 +4006,7 @@ static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, IOMMUTLBEntry iotlb = { /* We'll fill in the rest later. */ .target_as = &address_space_memory, - .pasid = vtd_as->pasid, + .pasid = vtd_as->pasid == IOMMU_NO_PASID ? PCI_NO_PASID : vtd_as->pasid, }; bool success; bool is_write = flag & IOMMU_WO; @@ -4041,9 +4014,8 @@ static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, if (likely(s->dmar_enabled)) { /* Only support translated requests in scalable mode */ if (iommu_idx == VTD_IDX_TRANSLATED && s->root_scalable) { - if (vtd_as->pasid == PCI_NO_PASID) { - vtd_prepare_identity_entry(addr, IOMMU_RW, PCI_NO_PASID, - &iotlb); + if (vtd_as->pasid == IOMMU_NO_PASID) { + vtd_prepare_identity_entry(addr, IOMMU_RW, &iotlb); success = true; } else { vtd_prepare_error_entry(&iotlb); @@ -4058,7 +4030,7 @@ static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, } } else { /* DMAR disabled, passthrough, use 4k-page*/ - vtd_prepare_identity_entry(addr, IOMMU_RW, vtd_as->pasid, &iotlb); + vtd_prepare_identity_entry(addr, IOMMU_RW, &iotlb); success = true; } @@ -4200,10 +4172,10 @@ static const Property vtd_properties[] = { DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits, VTD_HOST_ADDRESS_WIDTH), DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), - DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE), - DEFINE_PROP_BOOL("x-flts", IntelIOMMUState, fsts, FALSE), + DEFINE_PROP_BOOL("scalable-mode", IntelIOMMUState, scalable_mode, FALSE), + DEFINE_PROP_BOOL("fsts", IntelIOMMUState, fsts, FALSE), DEFINE_PROP_BOOL("snoop-control", IntelIOMMUState, snoop_control, false), - DEFINE_PROP_BOOL("x-pasid-mode", IntelIOMMUState, pasid, false), + DEFINE_PROP_UINT8("pasid-bits", IntelIOMMUState, pasid, 0), DEFINE_PROP_BOOL("svm", IntelIOMMUState, svm, false), DEFINE_PROP_BOOL("stale-tm", IntelIOMMUState, stale_tm, false), DEFINE_PROP_BOOL("fs1gp", IntelIOMMUState, fs1gp, true), @@ -4484,7 +4456,7 @@ static void vtd_report_sid_ir_illegal_access(IntelIOMMUState *s, uint16_t sid, } vtd_report_fault(s, VTD_FR_SM_INTERRUPT_ADDR, is_fpd_set, sid, addr, - is_write, pasid != PCI_NO_PASID, pasid); + is_write, pasid != IOMMU_NO_PASID, pasid); } static void vtd_report_ir_illegal_access(VTDAddressSpace *vtd_as, @@ -4512,7 +4484,6 @@ static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, int ret = 0; MSIMessage from = {}, to = {}; uint16_t sid = X86_IOMMU_SID_INVALID; - uint32_t pasid; from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; from.data = (uint32_t) value; @@ -4520,11 +4491,11 @@ static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, if (!attrs.unspecified) { /* We have explicit Source ID */ sid = attrs.requester_id; - pasid = attrs.pid != 0 ? attrs.pid : PCI_NO_PASID; if (attrs.address_type == PCI_AT_TRANSLATED && sid != X86_IOMMU_SID_INVALID) { - vtd_report_sid_ir_illegal_access(s, sid, pasid, from.address, true); + vtd_report_sid_ir_illegal_access(s, sid, attrs.pid, from.address, + true); return MEMTX_ERROR; } } @@ -4586,9 +4557,19 @@ static const MemoryRegionOps vtd_mem_ir_fault_ops = { }, }; +/* + * This function is called by many PCIIOMMUOps callbacks to get + * VTDAddressSpace or create one if non-exist. Those callbacks are + * used by PCI sub-system and are passed in a PCI pasid value. + * + * VTD honors iommu pasid, so the first thing is to convert PCI + * pasid to iommu pasid. + */ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn, unsigned int pasid) { + pasid = pasid == PCI_NO_PASID ? IOMMU_NO_PASID : pasid; + /* * We can't simply use sid here since the bus number might not be * initialized by the guest. @@ -4630,7 +4611,7 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, new_key->devfn = devfn; new_key->pasid = pasid; - if (pasid == PCI_NO_PASID) { + if (pasid == IOMMU_NO_PASID) { snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), PCI_FUNC(devfn)); } else { @@ -4783,6 +4764,7 @@ static bool vtd_dev_set_iommu_device(PCIBus *bus, void *opaque, int devfn, vtd_hiod->devfn = (uint8_t)devfn; vtd_hiod->iommu_state = s; vtd_hiod->hiod = hiod; + QLIST_INIT(&vtd_hiod->pasid_cache_list); if (!vtd_check_hiod(s, vtd_hiod, errp)) { g_free(vtd_hiod); @@ -4830,6 +4812,11 @@ static uint64_t vtd_get_viommu_flags(void *opaque) if (s->fsts) { flags = VIOMMU_FLAG_WANT_NESTING_PARENT | VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING; + + if (s->pasid) { + flags |= VIOMMU_FLAG_PASID_SUPPORTED | + VIOMMU_FLAG_WANT_PASID_ATTACH; + } } return flags; @@ -5045,7 +5032,8 @@ static void vtd_cap_init(IntelIOMMUState *s) } if (s->pasid) { - s->ecap |= VTD_ECAP_PASID | VTD_ECAP_PSS; + VTD_ECAP_SET_PSS(s, s->pasid - 1); + s->ecap |= VTD_ECAP_PASID; } } @@ -5313,7 +5301,7 @@ error_get_fpd_and_report: vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, vtd_as->pasid); error_report: vtd_report_fault(s, -ret, is_fpd_set, sid, addr, is_write, - vtd_as->pasid != PCI_NO_PASID, vtd_as->pasid); + vtd_as->pasid != IOMMU_NO_PASID, vtd_as->pasid); return false; } @@ -5404,7 +5392,7 @@ static int vtd_pri_request_page(PCIBus *bus, void *opaque, int devfn, */ /* We do not support PRI without PASID */ - if (vtd_as->pasid == PCI_NO_PASID) { + if (vtd_as->pasid == IOMMU_NO_PASID) { return -EPERM; } if (exec_req && !is_read) { @@ -5563,20 +5551,20 @@ static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) } if (!s->scalable_mode && s->fsts) { - error_setg(errp, "x-flts is only available in scalable mode"); + error_setg(errp, "fsts is only available in scalable mode"); return false; } if (!s->fsts && s->aw_bits != VTD_HOST_AW_39BIT && s->aw_bits != VTD_HOST_AW_48BIT) { error_setg(errp, "%s: supported values for aw-bits are: %d, %d", - s->scalable_mode ? "Scalable mode(flts=off)" : "Legacy mode", + s->scalable_mode ? "Scalable mode(fsts=off)" : "Legacy mode", VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); return false; } if (s->fsts && s->aw_bits != VTD_HOST_AW_48BIT) { - error_setg(errp, "Scalable mode(x-flts=on): supported value for " + error_setg(errp, "Scalable mode(fsts=on): supported value for " "aw-bits is: %d", VTD_HOST_AW_48BIT); return false; } @@ -5586,6 +5574,12 @@ static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) return false; } + if (s->pasid > PCI_EXT_CAP_PASID_MAX_WIDTH) { + error_setg(errp, "PASID width %d exceeds Max PASID Width %d allowed " + "in PCI spec", s->pasid, PCI_EXT_CAP_PASID_MAX_WIDTH); + return false; + } + if (s->svm) { if (!x86_iommu->dt_supported) { error_setg(errp, "Need to set device IOTLB for svm"); @@ -5593,7 +5587,7 @@ static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) } if (!s->fsts) { - error_setg(errp, "Need to set flts for svm"); + error_setg(errp, "Need to set fsts for svm"); return false; } diff --git a/hw/i386/intel_iommu_accel.c b/hw/i386/intel_iommu_accel.c index ed3793602b..c03435f8a2 100644 --- a/hw/i386/intel_iommu_accel.c +++ b/hw/i386/intel_iommu_accel.c @@ -22,11 +22,12 @@ bool vtd_check_hiod_accel(IntelIOMMUState *s, VTDHostIOMMUDevice *vtd_hiod, HostIOMMUDevice *hiod = vtd_hiod->hiod; struct HostIOMMUDeviceCaps *caps = &hiod->caps; struct iommu_hw_info_vtd *vtd = &caps->vendor_caps.vtd; + uint8_t hpasid = VTD_ECAP_GET_PSS(vtd->ecap_reg) + 1; PCIBus *bus = vtd_hiod->bus; PCIDevice *pdev = bus->devices[vtd_hiod->devfn]; if (!object_dynamic_cast(OBJECT(hiod), TYPE_HOST_IOMMU_DEVICE_IOMMUFD)) { - error_setg(errp, "Need IOMMUFD backend when x-flts=on"); + error_setg(errp, "Need IOMMUFD backend when fsts=on"); return false; } @@ -42,9 +43,16 @@ bool vtd_check_hiod_accel(IntelIOMMUState *s, VTDHostIOMMUDevice *vtd_hiod, return false; } + /* Only do the check when host device support PASIDs */ + if (caps->max_pasid_log2 && s->pasid > hpasid) { + error_setg(errp, "PASID bits size %d > host IOMMU PASID bits size %d", + s->pasid, hpasid); + return false; + } + if (pci_device_get_iommu_bus_devfn(pdev, &bus, NULL, NULL)) { error_setg(errp, "Host device downstream to a PCI bridge is " - "unsupported when x-flts=on"); + "unsupported when fsts=on"); return false; } @@ -69,11 +77,13 @@ VTDHostIOMMUDevice *vtd_find_hiod_iommufd(VTDAddressSpace *as) return NULL; } -static bool vtd_create_fs_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, +static bool vtd_create_fs_hwpt(VTDHostIOMMUDevice *vtd_hiod, VTDPASIDEntry *pe, uint32_t *fs_hwpt_id, Error **errp) { + HostIOMMUDeviceIOMMUFD *hiodi = HOST_IOMMU_DEVICE_IOMMUFD(vtd_hiod->hiod); struct iommu_hwpt_vtd_s1 vtd = {}; + uint32_t flags = vtd_hiod->iommu_state->pasid ? IOMMU_HWPT_ALLOC_PASID : 0; vtd.flags = (VTD_SM_PASID_ENTRY_SRE(pe) ? IOMMU_VTD_S1_SRE : 0) | (VTD_SM_PASID_ENTRY_WPE(pe) ? IOMMU_VTD_S1_WPE : 0) | @@ -82,30 +92,34 @@ static bool vtd_create_fs_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, vtd.pgtbl_addr = (uint64_t)vtd_pe_get_fspt_base(pe); return iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid, - hiodi->hwpt_id, 0, IOMMU_HWPT_DATA_VTD_S1, - sizeof(vtd), &vtd, fs_hwpt_id, errp); + hiodi->hwpt_id, flags, + IOMMU_HWPT_DATA_VTD_S1, sizeof(vtd), &vtd, + fs_hwpt_id, errp); } -static void vtd_destroy_old_fs_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, - VTDAddressSpace *vtd_as) +static void vtd_destroy_old_fs_hwpt(VTDAccelPASIDCacheEntry *vtd_pce) { - if (!vtd_as->fs_hwpt_id) { + HostIOMMUDeviceIOMMUFD *hiodi = + HOST_IOMMU_DEVICE_IOMMUFD(vtd_pce->vtd_hiod->hiod); + + if (!vtd_pce->fs_hwpt_id) { return; } - iommufd_backend_free_id(hiodi->iommufd, vtd_as->fs_hwpt_id); - vtd_as->fs_hwpt_id = 0; + iommufd_backend_free_id(hiodi->iommufd, vtd_pce->fs_hwpt_id); + vtd_pce->fs_hwpt_id = 0; } -static bool vtd_device_attach_iommufd(VTDHostIOMMUDevice *vtd_hiod, - VTDAddressSpace *vtd_as, Error **errp) +static bool vtd_device_attach_iommufd(VTDAccelPASIDCacheEntry *vtd_pce, + Error **errp) { + VTDHostIOMMUDevice *vtd_hiod = vtd_pce->vtd_hiod; HostIOMMUDeviceIOMMUFD *hiodi = HOST_IOMMU_DEVICE_IOMMUFD(vtd_hiod->hiod); - VTDPASIDEntry *pe = &vtd_as->pasid_cache_entry.pasid_entry; - uint32_t hwpt_id = hiodi->hwpt_id; + VTDPASIDEntry *pe = &vtd_pce->pasid_entry; + uint32_t hwpt_id = hiodi->hwpt_id, pasid = vtd_pce->pasid; bool ret; /* - * We can get here only if flts=on, the supported PGTT is FST or PT. + * We can get here only if fsts=on, the supported PGTT is FST or PT. * Catch invalid PGTT when processing invalidation request to avoid * attaching to wrong hwpt. */ @@ -116,18 +130,18 @@ static bool vtd_device_attach_iommufd(VTDHostIOMMUDevice *vtd_hiod, } if (vtd_pe_pgtt_is_fst(pe)) { - if (!vtd_create_fs_hwpt(hiodi, pe, &hwpt_id, errp)) { + if (!vtd_create_fs_hwpt(vtd_hiod, pe, &hwpt_id, errp)) { return false; } } - ret = host_iommu_device_iommufd_attach_hwpt(hiodi, hwpt_id, errp); - trace_vtd_device_attach_hwpt(hiodi->devid, vtd_as->pasid, hwpt_id, ret); + ret = host_iommu_device_iommufd_attach_hwpt(hiodi, pasid, hwpt_id, errp); + trace_vtd_device_attach_hwpt(hiodi->devid, pasid, hwpt_id, ret); if (ret) { /* Destroy old fs_hwpt if it's a replacement */ - vtd_destroy_old_fs_hwpt(hiodi, vtd_as); + vtd_destroy_old_fs_hwpt(vtd_pce); if (vtd_pe_pgtt_is_fst(pe)) { - vtd_as->fs_hwpt_id = hwpt_id; + vtd_pce->fs_hwpt_id = hwpt_id; } } else if (vtd_pe_pgtt_is_fst(pe)) { iommufd_backend_free_id(hiodi->iommufd, hwpt_id); @@ -136,16 +150,18 @@ static bool vtd_device_attach_iommufd(VTDHostIOMMUDevice *vtd_hiod, return ret; } -static bool vtd_device_detach_iommufd(VTDHostIOMMUDevice *vtd_hiod, - VTDAddressSpace *vtd_as, Error **errp) +static bool vtd_device_detach_iommufd(VTDAccelPASIDCacheEntry *vtd_pce, + Error **errp) { + VTDHostIOMMUDevice *vtd_hiod = vtd_pce->vtd_hiod; HostIOMMUDeviceIOMMUFD *hiodi = HOST_IOMMU_DEVICE_IOMMUFD(vtd_hiod->hiod); - IntelIOMMUState *s = vtd_as->iommu_state; - uint32_t pasid = vtd_as->pasid; + + IntelIOMMUState *s = vtd_hiod->iommu_state; + uint32_t pasid = vtd_pce->pasid; bool ret; - if (s->dmar_enabled && s->root_scalable) { - ret = host_iommu_device_iommufd_detach_hwpt(hiodi, errp); + if (pasid != IOMMU_NO_PASID || (s->dmar_enabled && s->root_scalable)) { + ret = host_iommu_device_iommufd_detach_hwpt(hiodi, pasid, errp); trace_vtd_device_detach_hwpt(hiodi->devid, pasid, ret); } else { /* @@ -153,65 +169,39 @@ static bool vtd_device_detach_iommufd(VTDHostIOMMUDevice *vtd_hiod, * we fallback to the default HWPT which contains shadow page table. * So guest DMA could still work. */ - ret = host_iommu_device_iommufd_attach_hwpt(hiodi, hiodi->hwpt_id, - errp); - trace_vtd_device_reattach_def_hwpt(hiodi->devid, pasid, hiodi->hwpt_id, - ret); + ret = host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, + hiodi->hwpt_id, errp); + trace_vtd_device_reattach_def_hwpt(hiodi->devid, IOMMU_NO_PASID, + hiodi->hwpt_id, ret); } if (ret) { - vtd_destroy_old_fs_hwpt(hiodi, vtd_as); + vtd_destroy_old_fs_hwpt(vtd_pce); } return ret; } -bool vtd_propagate_guest_pasid(VTDAddressSpace *vtd_as, Error **errp) -{ - VTDPASIDCacheEntry *pc_entry = &vtd_as->pasid_cache_entry; - VTDHostIOMMUDevice *vtd_hiod = vtd_find_hiod_iommufd(vtd_as); - - /* Ignore emulated device or legacy VFIO backed device */ - if (!vtd_as->iommu_state->fsts || !vtd_hiod) { - return true; - } - - if (pc_entry->valid) { - return vtd_device_attach_iommufd(vtd_hiod, vtd_as, errp); - } - - return vtd_device_detach_iommufd(vtd_hiod, vtd_as, errp); -} - /* - * This function is a loop function for the s->vtd_address_spaces - * list with VTDPIOTLBInvInfo as execution filter. It propagates - * the piotlb invalidation to host. + * This function is a loop function for the s->vtd_host_iommu_dev + * and vtd_hiod->pasid_cache_list lists with VTDPIOTLBInvInfo as + * execution filter. It propagates the piotlb invalidation to host. */ -static void vtd_flush_host_piotlb_locked(gpointer key, gpointer value, - gpointer user_data) +static void vtd_flush_host_piotlb_locked(VTDAccelPASIDCacheEntry *vtd_pce, + VTDPIOTLBInvInfo *piotlb_info) { - VTDPIOTLBInvInfo *piotlb_info = user_data; - VTDAddressSpace *vtd_as = value; - VTDHostIOMMUDevice *vtd_hiod = vtd_find_hiod_iommufd(vtd_as); - VTDPASIDCacheEntry *pc_entry = &vtd_as->pasid_cache_entry; + VTDHostIOMMUDevice *vtd_hiod = vtd_pce->vtd_hiod; + VTDPASIDEntry *pe = &vtd_pce->pasid_entry; uint16_t did; - if (!vtd_hiod) { - return; - } - - assert(vtd_as->pasid == PCI_NO_PASID); - /* Nothing to do if there is no first stage HWPT attached */ - if (!pc_entry->valid || - !vtd_pe_pgtt_is_fst(&pc_entry->pasid_entry)) { + if (!vtd_pe_pgtt_is_fst(pe)) { return; } - did = VTD_SM_PASID_ENTRY_DID(&pc_entry->pasid_entry); + did = VTD_SM_PASID_ENTRY_DID(pe); - if (piotlb_info->domain_id == did && piotlb_info->pasid == PASID_0) { + if (piotlb_info->domain_id == did && piotlb_info->pasid == vtd_pce->pasid) { HostIOMMUDeviceIOMMUFD *hiodi = HOST_IOMMU_DEVICE_IOMMUFD(vtd_hiod->hiod); uint32_t entry_num = 1; /* Only implement one request for simplicity */ @@ -219,7 +209,7 @@ static void vtd_flush_host_piotlb_locked(gpointer key, gpointer value, struct iommu_hwpt_vtd_s1_invalidate *cache = piotlb_info->inv_data; if (!iommufd_backend_invalidate_cache(hiodi->iommufd, - vtd_as->fs_hwpt_id, + vtd_pce->fs_hwpt_id, IOMMU_HWPT_INVALIDATE_DATA_VTD_S1, sizeof(*cache), &entry_num, cache, &local_err)) { @@ -235,6 +225,8 @@ void vtd_flush_host_piotlb_all_locked(IntelIOMMUState *s, uint16_t domain_id, { struct iommu_hwpt_vtd_s1_invalidate cache_info = { 0 }; VTDPIOTLBInvInfo piotlb_info; + VTDHostIOMMUDevice *vtd_hiod; + GHashTableIter hiod_it; cache_info.addr = addr; cache_info.npages = npages; @@ -245,12 +237,275 @@ void vtd_flush_host_piotlb_all_locked(IntelIOMMUState *s, uint16_t domain_id, piotlb_info.inv_data = &cache_info; /* - * Go through each vtd_as instance in s->vtd_address_spaces, find out - * affected host devices which need host piotlb invalidation. Piotlb - * invalidation should check pasid cache per architecture point of view. + * Go through each vtd_pce in vtd_hiod->pasid_cache_list for each host + * device, find out affected host device pasid which need host piotlb + * invalidation. Piotlb invalidation should check pasid cache per + * architecture point of view. */ - g_hash_table_foreach(s->vtd_address_spaces, - vtd_flush_host_piotlb_locked, &piotlb_info); + g_hash_table_iter_init(&hiod_it, s->vtd_host_iommu_dev); + while (g_hash_table_iter_next(&hiod_it, NULL, (void **)&vtd_hiod)) { + VTDAccelPASIDCacheEntry *vtd_pce; + + QLIST_FOREACH(vtd_pce, &vtd_hiod->pasid_cache_list, next) { + vtd_flush_host_piotlb_locked(vtd_pce, &piotlb_info); + } + } +} + +static void vtd_accel_fill_pc(VTDHostIOMMUDevice *vtd_hiod, uint32_t pasid, + VTDPASIDEntry *pe) +{ + VTDAccelPASIDCacheEntry *vtd_pce; + Error *local_err = NULL; + + QLIST_FOREACH(vtd_pce, &vtd_hiod->pasid_cache_list, next) { + if (vtd_pce->pasid == pasid) { + if (vtd_pasid_entry_compare(pe, &vtd_pce->pasid_entry)) { + vtd_pce->pasid_entry = *pe; + + if (!vtd_device_attach_iommufd(vtd_pce, &local_err)) { + error_reportf_err(local_err, "%s", + "Replacing HWPT attachment failed: "); + } + } + return; + } + } + + vtd_pce = g_malloc0(sizeof(VTDAccelPASIDCacheEntry)); + vtd_pce->vtd_hiod = vtd_hiod; + vtd_pce->pasid = pasid; + vtd_pce->pasid_entry = *pe; + QLIST_INSERT_HEAD(&vtd_hiod->pasid_cache_list, vtd_pce, next); + + if (!vtd_device_attach_iommufd(vtd_pce, &local_err)) { + error_reportf_err(local_err, "%s", "Attaching to HWPT failed: "); + } +} + +static void vtd_accel_delete_pc(VTDAccelPASIDCacheEntry *vtd_pce, + VTDPASIDCacheInfo *pc_info) +{ + Error *local_err = NULL; + + if (!vtd_device_detach_iommufd(vtd_pce, &local_err)) { + error_reportf_err(local_err, "%s", "Detaching from HWPT failed: "); + } + + QLIST_REMOVE(vtd_pce, next); + g_free(vtd_pce); + + if (pc_info->type == VTD_INV_DESC_PASIDC_G_PASID_SI) { + pc_info->accel_pce_deleted = true; + } +} + +static void +vtd_accel_pasid_cache_invalidate_one(VTDAccelPASIDCacheEntry *vtd_pce, + VTDPASIDCacheInfo *pc_info) +{ + VTDHostIOMMUDevice *vtd_hiod = vtd_pce->vtd_hiod; + VTDPASIDEntry pe; + uint16_t did; + + /* + * VTD_INV_DESC_PASIDC_G_DSI and VTD_INV_DESC_PASIDC_G_PASID_SI require + * DID check. If DID doesn't match the value in cache or memory, then + * it's not a pasid entry we want to invalidate. + */ + switch (pc_info->type) { + case VTD_INV_DESC_PASIDC_G_PASID_SI: + if (pc_info->pasid != vtd_pce->pasid) { + return; + } + /* Fall through */ + case VTD_INV_DESC_PASIDC_G_DSI: + did = VTD_SM_PASID_ENTRY_DID(&vtd_pce->pasid_entry); + if (pc_info->did != did) { + return; + } + } + + if (vtd_dev_get_pe_from_pasid(vtd_hiod->iommu_state, vtd_hiod->bus, + vtd_hiod->devfn, vtd_pce->pasid, &pe)) { + /* + * No valid pasid entry in guest memory. e.g. pasid entry was modified + * to be either all-zero or non-present. Either case means existing + * pasid cache should be invalidated. + */ + vtd_accel_delete_pc(vtd_pce, pc_info); + } +} + +static void vtd_accel_pasid_cache_invalidate(VTDHostIOMMUDevice *vtd_hiod, + VTDPASIDCacheInfo *pc_info) +{ + VTDAccelPASIDCacheEntry *vtd_pce, *next; + + QLIST_FOREACH_SAFE(vtd_pce, &vtd_hiod->pasid_cache_list, next, next) { + vtd_accel_pasid_cache_invalidate_one(vtd_pce, pc_info); + } +} + +/* + * This function walks over PASID range within [start, end) in a single + * PASID table for entries matching @info type/did, then create + * VTDAccelPASIDCacheEntry if not exist yet. + */ +static void vtd_sm_pasid_table_walk_one(VTDHostIOMMUDevice *vtd_hiod, + dma_addr_t pt_base, int start, int end, + VTDPASIDCacheInfo *info) +{ + IntelIOMMUState *s = vtd_hiod->iommu_state; + VTDPASIDEntry pe; + int pasid; + + for (pasid = start; pasid < end; pasid++) { + if (vtd_get_pe_in_pasid_leaf_table(s, pasid, pt_base, &pe) || + !vtd_pe_present(&pe)) { + continue; + } + + if ((info->type == VTD_INV_DESC_PASIDC_G_DSI || + info->type == VTD_INV_DESC_PASIDC_G_PASID_SI) && + (info->did != VTD_SM_PASID_ENTRY_DID(&pe))) { + /* + * VTD_PASID_CACHE_DOMSI and VTD_PASID_CACHE_PASIDSI + * requires domain id check. If domain id check fail, + * go to next pasid. + */ + continue; + } + + vtd_accel_fill_pc(vtd_hiod, pasid, &pe); + } +} + +/* + * In VT-d scalable mode translation, PASID dir + PASID table is used. + * This function aims at looping over a range of PASIDs in the given + * two level table to identify the pasid config in guest. + */ +static void vtd_sm_pasid_table_walk(VTDHostIOMMUDevice *vtd_hiod, + dma_addr_t pdt_base, int start, int end, + VTDPASIDCacheInfo *info) +{ + VTDPASIDDirEntry pdire; + int pasid = start; + int pasid_next; + dma_addr_t pt_base; + + while (pasid < end) { + pasid_next = (pasid + VTD_PASID_TABLE_ENTRY_NUM) & + ~(VTD_PASID_TABLE_ENTRY_NUM - 1); + pasid_next = pasid_next < end ? pasid_next : end; + + if (!vtd_get_pdire_from_pdir_table(pdt_base, pasid, &pdire) + && vtd_pdire_present(&pdire)) { + pt_base = pdire.val & VTD_PASID_TABLE_BASE_ADDR_MASK; + vtd_sm_pasid_table_walk_one(vtd_hiod, pt_base, pasid, pasid_next, + info); + } + pasid = pasid_next; + } +} + +static void vtd_accel_replay_pasid_bind_for_dev(VTDHostIOMMUDevice *vtd_hiod, + int start, int end, + VTDPASIDCacheInfo *pc_info) +{ + IntelIOMMUState *s = vtd_hiod->iommu_state; + VTDContextEntry ce; + int dev_max_pasid = 1 << vtd_hiod->hiod->caps.max_pasid_log2; + + if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_hiod->bus), + vtd_hiod->devfn, &ce)) { + VTDPASIDCacheInfo walk_info = *pc_info; + uint32_t ce_max_pasid = vtd_sm_ce_get_pdt_entry_num(&ce) * + VTD_PASID_TABLE_ENTRY_NUM; + + end = MIN(end, MIN(dev_max_pasid, ce_max_pasid)); + + vtd_sm_pasid_table_walk(vtd_hiod, VTD_CE_GET_PASID_DIR_TABLE(&ce), + start, end, &walk_info); + } +} + +/* + * This function replays the guest pasid bindings by walking the two level + * guest PASID table. For each valid pasid entry, it creates an entry + * VTDAccelPASIDCacheEntry dynamically if not exist yet. This entry holds + * info specific to a pasid + */ +void vtd_accel_pasid_cache_sync(IntelIOMMUState *s, VTDPASIDCacheInfo *pc_info) +{ + int start = IOMMU_NO_PASID, end = 1 << s->pasid; + VTDHostIOMMUDevice *vtd_hiod; + GHashTableIter hiod_it; + + if (!s->fsts) { + return; + } + + switch (pc_info->type) { + case VTD_INV_DESC_PASIDC_G_PASID_SI: + start = pc_info->pasid; + end = pc_info->pasid + 1; + /* fall through */ + case VTD_INV_DESC_PASIDC_G_DSI: + /* + * loop all assigned devices, do domain id check in + * vtd_sm_pasid_table_walk_one() after get pasid entry. + */ + break; + case VTD_INV_DESC_PASIDC_G_GLOBAL: + /* loop all assigned devices */ + break; + default: + g_assert_not_reached(); + } + + /* + * Loop all the vtd_hiod instances to sync the "pasid cache" per the + * guest pasid configuration. + * + * VTD translation callback never accesses vtd_hiod and its corresponding + * cached pasid entry, so no iommu lock needed here. + */ + g_hash_table_iter_init(&hiod_it, s->vtd_host_iommu_dev); + while (g_hash_table_iter_next(&hiod_it, NULL, (void **)&vtd_hiod)) { + if (!object_dynamic_cast(OBJECT(vtd_hiod->hiod), + TYPE_HOST_IOMMU_DEVICE_IOMMUFD)) { + continue; + } + + /* + * The replay path inevitably needs to iterate through existing + * PASID cache entries. Since cached PASID entries that are marked + * for removal don't need to be iterated, we intentionally handle + * removals before additions to optimize the replay process. + */ + vtd_accel_pasid_cache_invalidate(vtd_hiod, pc_info); + + if (pc_info->accel_pce_deleted) { + pc_info->accel_pce_deleted = false; + } else { + vtd_accel_replay_pasid_bind_for_dev(vtd_hiod, start, end, pc_info); + } + } +} + +/* Fake a global pasid cache invalidation to remove all pasid cache entries */ +void vtd_accel_pasid_cache_reset(IntelIOMMUState *s) +{ + VTDPASIDCacheInfo pc_info = { .type = VTD_INV_DESC_PASIDC_G_GLOBAL }; + VTDHostIOMMUDevice *vtd_hiod; + GHashTableIter hiod_it; + + g_hash_table_iter_init(&hiod_it, s->vtd_host_iommu_dev); + while (g_hash_table_iter_next(&hiod_it, NULL, (void **)&vtd_hiod)) { + vtd_accel_pasid_cache_invalidate(vtd_hiod, &pc_info); + } } static uint64_t vtd_get_host_iommu_quirks(uint32_t type, diff --git a/hw/i386/intel_iommu_accel.h b/hw/i386/intel_iommu_accel.h index e5f0b077b4..4a9003c92d 100644 --- a/hw/i386/intel_iommu_accel.h +++ b/hw/i386/intel_iommu_accel.h @@ -12,14 +12,23 @@ #define HW_I386_INTEL_IOMMU_ACCEL_H #include CONFIG_DEVICES +typedef struct VTDAccelPASIDCacheEntry { + VTDHostIOMMUDevice *vtd_hiod; + VTDPASIDEntry pasid_entry; + uint32_t pasid; + uint32_t fs_hwpt_id; + QLIST_ENTRY(VTDAccelPASIDCacheEntry) next; +} VTDAccelPASIDCacheEntry; + #ifdef CONFIG_VTD_ACCEL bool vtd_check_hiod_accel(IntelIOMMUState *s, VTDHostIOMMUDevice *vtd_hiod, Error **errp); VTDHostIOMMUDevice *vtd_find_hiod_iommufd(VTDAddressSpace *as); -bool vtd_propagate_guest_pasid(VTDAddressSpace *vtd_as, Error **errp); void vtd_flush_host_piotlb_all_locked(IntelIOMMUState *s, uint16_t domain_id, uint32_t pasid, hwaddr addr, uint64_t npages, bool ih); +void vtd_accel_pasid_cache_sync(IntelIOMMUState *s, VTDPASIDCacheInfo *pc_info); +void vtd_accel_pasid_cache_reset(IntelIOMMUState *s); void vtd_iommu_ops_update_accel(PCIIOMMUOps *ops); #else static inline bool vtd_check_hiod_accel(IntelIOMMUState *s, @@ -49,6 +58,15 @@ static inline void vtd_flush_host_piotlb_all_locked(IntelIOMMUState *s, { } +static inline void vtd_accel_pasid_cache_sync(IntelIOMMUState *s, + VTDPASIDCacheInfo *pc_info) +{ +} + +static inline void vtd_accel_pasid_cache_reset(IntelIOMMUState *s) +{ +} + static inline void vtd_iommu_ops_update_accel(PCIIOMMUOps *ops) { } diff --git a/hw/i386/intel_iommu_internal.h b/hw/i386/intel_iommu_internal.h index 11a53aa369..924e91cb8a 100644 --- a/hw/i386/intel_iommu_internal.h +++ b/hw/i386/intel_iommu_internal.h @@ -195,7 +195,8 @@ #define VTD_ECAP_MHMV (15ULL << 20) #define VTD_ECAP_SRS (1ULL << 31) #define VTD_ECAP_NWFS (1ULL << 33) -#define VTD_ECAP_PSS (7ULL << 35) /* limit: MemTxAttrs::pid */ +#define VTD_ECAP_SET_PSS(x, v) ((x)->ecap = deposit64((x)->ecap, 35, 5, v)) +#define VTD_ECAP_GET_PSS(ecap) extract64(ecap, 35, 5) #define VTD_ECAP_PASID (1ULL << 40) #define VTD_ECAP_PDS (1ULL << 42) #define VTD_ECAP_SMTS (1ULL << 43) @@ -615,15 +616,22 @@ typedef struct VTDRootEntry VTDRootEntry; #define VTD_CTX_ENTRY_LEGACY_SIZE 16 #define VTD_CTX_ENTRY_SCALABLE_SIZE 32 -#define PASID_0 0 +#define VTD_SM_CONTEXT_ENTRY_PDTS(x) extract64((x)->val[0], 9, 3) #define VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(aw) (0x1e0ULL | ~VTD_HAW_MASK(aw)) #define VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 0xffffffffffe00000ULL #define VTD_SM_CONTEXT_ENTRY_PRE 0x10ULL +/* context entry operations */ +#define VTD_CE_GET_PASID_DIR_TABLE(ce) \ + ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK) +#define VTD_CE_GET_PRE(ce) \ + ((ce)->val[0] & VTD_SM_CONTEXT_ENTRY_PRE) + typedef struct VTDPASIDCacheInfo { uint8_t type; uint16_t did; uint32_t pasid; + bool accel_pce_deleted; } VTDPASIDCacheInfo; typedef struct VTDPIOTLBInvInfo { @@ -640,6 +648,7 @@ typedef struct VTDPIOTLBInvInfo { #define VTD_PASID_DIR_BITS_MASK (0x3fffULL) #define VTD_PASID_DIR_INDEX(pasid) (((pasid) >> 6) & VTD_PASID_DIR_BITS_MASK) #define VTD_PASID_DIR_FPD (1ULL << 1) /* Fault Processing Disable */ +#define VTD_PASID_TABLE_ENTRY_NUM (1ULL << 6) #define VTD_PASID_TABLE_BITS_MASK (0x3fULL) #define VTD_PASID_TABLE_INDEX(pasid) ((pasid) & VTD_PASID_TABLE_BITS_MASK) #define VTD_PASID_ENTRY_FPD (1ULL << 1) /* Fault Processing Disable */ @@ -705,6 +714,7 @@ typedef struct VTDHostIOMMUDevice { PCIBus *bus; uint8_t devfn; HostIOMMUDevice *hiod; + QLIST_HEAD(, VTDAccelPASIDCacheEntry) pasid_cache_list; } VTDHostIOMMUDevice; /* @@ -746,4 +756,34 @@ static inline bool vtd_pe_pgtt_is_fst(VTDPASIDEntry *pe) { return (VTD_SM_PASID_ENTRY_PGTT(pe) == VTD_SM_PASID_ENTRY_FST); } + +static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire) +{ + return pdire->val & 1; +} + +static inline bool vtd_pe_present(VTDPASIDEntry *pe) +{ + return pe->val[0] & VTD_PASID_ENTRY_P; +} + +static inline int vtd_pasid_entry_compare(VTDPASIDEntry *p1, VTDPASIDEntry *p2) +{ + return memcmp(p1, p2, sizeof(*p1)); +} + +static inline uint32_t vtd_sm_ce_get_pdt_entry_num(VTDContextEntry *ce) +{ + return 1U << (VTD_SM_CONTEXT_ENTRY_PDTS(ce) + 7); +} + +int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, uint32_t pasid, + VTDPASIDDirEntry *pdire); +int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, uint32_t pasid, + dma_addr_t addr, VTDPASIDEntry *pe); +int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, + uint8_t devfn, VTDContextEntry *ce); +VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid); +int vtd_dev_get_pe_from_pasid(IntelIOMMUState *s, PCIBus *bus, uint8_t devfn, + uint32_t pasid, VTDPASIDEntry *pe); #endif diff --git a/hw/i386/pc.c b/hw/i386/pc.c index 7b6ad97e5a..73a625327c 100644 --- a/hw/i386/pc.c +++ b/hw/i386/pc.c @@ -1051,10 +1051,12 @@ void pc_basic_device_init(struct PCMachineState *pcms, MemoryRegion *ioportF0_io = g_new(MemoryRegion, 1); X86MachineState *x86ms = X86_MACHINE(pcms); - memory_region_init_io(ioport80_io, NULL, &ioport80_io_ops, NULL, "ioport80", 1); + memory_region_init_io(ioport80_io, OBJECT(pcms), &ioport80_io_ops, NULL, + "ioport80", 1); memory_region_add_subregion(isa_bus->address_space_io, 0x80, ioport80_io); - memory_region_init_io(ioportF0_io, NULL, &ioportF0_io_ops, NULL, "ioportF0", 1); + memory_region_init_io(ioportF0_io, OBJECT(pcms), &ioportF0_io_ops, NULL, + "ioportF0", 1); memory_region_add_subregion(isa_bus->address_space_io, 0xf0, ioportF0_io); /* diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index d8fed698c7..6c1e4eff5f 100644 --- a/hw/i386/pc_q35.c +++ b/hw/i386/pc_q35.c @@ -331,6 +331,18 @@ static void pc_q35_init(MachineState *machine) } } +static bool pc_q35_machine_get_wdat(Object *o, Error **errp) +{ + PCMachineState *pcms = PC_MACHINE(o); + return pcms->wdat_enabled; +} + +static void pc_q35_machine_set_wdat(Object *o, bool value, Error **errp) +{ + PCMachineState *pcms = PC_MACHINE(o); + pcms->wdat_enabled = value; +} + #define DEFINE_Q35_MACHINE(major, minor) \ DEFINE_PC_VER_MACHINE(pc_q35, "pc-q35", pc_q35_init, false, NULL, major, minor); @@ -343,6 +355,7 @@ static void pc_q35_init(MachineState *machine) static void pc_q35_machine_options(MachineClass *m) { PCMachineClass *pcmc = PC_MACHINE_CLASS(m); + ObjectClass *oc = OBJECT_CLASS(m); pcmc->pci_root_uid = 0; pcmc->default_cpu_version = 1; @@ -361,6 +374,11 @@ static void pc_q35_machine_options(MachineClass *m) machine_class_allow_dynamic_sysbus_dev(m, TYPE_RAMFB_DEVICE); machine_class_allow_dynamic_sysbus_dev(m, TYPE_VMBUS_BRIDGE); machine_class_allow_dynamic_sysbus_dev(m, TYPE_UEFI_VARS_X64); + object_class_property_add_bool(oc, "wdat", + pc_q35_machine_get_wdat, pc_q35_machine_set_wdat); + object_class_property_set_description(oc, "wdat", + "Enable WDAT watchdog support. Default: off"); + compat_props_add(m->compat_props, pc_q35_compat_defaults, pc_q35_compat_defaults_len); } diff --git a/hw/i386/trace-events b/hw/i386/trace-events index 5fa5e93b68..a1dfade20f 100644 --- a/hw/i386/trace-events +++ b/hw/i386/trace-events @@ -118,6 +118,7 @@ amdvi_ir_intctl(uint8_t val) "int_ctl 0x%"PRIx8 amdvi_ir_target_abort(const char *str) "%s" amdvi_ir_delivery_mode(const char *str) "%s" amdvi_ir_irte_ga_val(uint64_t hi, uint64_t lo) "hi 0x%"PRIx64" lo 0x%"PRIx64 +amdvi_generate_msi_interrupt(const char *str) "Mode: %s" # vmport.c vmport_register(unsigned char command, void *func, void *opaque) "command: 0x%02x func: %p opaque: %p" diff --git a/hw/mem/cxl_type3.c b/hw/mem/cxl_type3.c index 4739239da3..cba05ec57d 100644 --- a/hw/mem/cxl_type3.c +++ b/hw/mem/cxl_type3.c @@ -427,6 +427,8 @@ static void hdm_decoder_commit(CXLType3Dev *ct3d, int which) ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 1); stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl); + + cfmws_update_non_interleaved(true); } static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which) @@ -442,6 +444,8 @@ static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which) ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 0); stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl); + + cfmws_update_non_interleaved(false); } static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err) @@ -1767,7 +1771,6 @@ static void cxl_maintenance_insert(CXLType3Dev *ct3d, uint64_t dpa, } } m = g_new0(CXLMaintenance, 1); - memset(m, 0, sizeof(*m)); m->dpa = dpa; m->validity_flags = 0; @@ -1804,8 +1807,7 @@ static void cxl_maintenance_insert(CXLType3Dev *ct3d, uint64_t dpa, m->validity_flags |= CXL_MSER_VALID_SUB_CHANNEL; } if (component_id) { - strncpy((char *)m->component_id, component_id, - sizeof(m->component_id) - 1); + memcpy(m->component_id, component_id, sizeof(m->component_id)); m->validity_flags |= CXL_MSER_VALID_COMP_ID; if (has_comp_id_pldm && is_comp_id_pldm) { m->validity_flags |= CXL_MSER_VALID_COMP_ID_FORMAT; @@ -1897,8 +1899,7 @@ void qmp_cxl_inject_general_media_event(const char *path, CxlEventLog log, } if (component_id) { - strncpy((char *)gem.component_id, component_id, - sizeof(gem.component_id) - 1); + memcpy(gem.component_id, component_id, sizeof(gem.component_id)); valid_flags |= CXL_GMER_VALID_COMPONENT; if (has_comp_id_pldm && is_comp_id_pldm) { valid_flags |= CXL_GMER_VALID_COMPONENT_ID_FORMAT; @@ -2068,8 +2069,7 @@ void qmp_cxl_inject_dram_event(const char *path, CxlEventLog log, } if (component_id) { - strncpy((char *)dram.component_id, component_id, - sizeof(dram.component_id) - 1); + memcpy(dram.component_id, component_id, sizeof(dram.component_id)); valid_flags |= CXL_DRAM_VALID_COMPONENT; if (has_comp_id_pldm && is_comp_id_pldm) { valid_flags |= CXL_DRAM_VALID_COMPONENT_ID_FORMAT; @@ -2187,8 +2187,7 @@ void qmp_cxl_inject_memory_module_event(const char *path, CxlEventLog log, corrected_persist_error_count); if (component_id) { - strncpy((char *)module.component_id, component_id, - sizeof(module.component_id) - 1); + memcpy(module.component_id, component_id, sizeof(module.component_id)); valid_flags |= CXL_MMER_VALID_COMPONENT; if (has_comp_id_pldm && is_comp_id_pldm) { valid_flags |= CXL_MMER_VALID_COMPONENT_ID_FORMAT; diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index a8ee18a912..323d117735 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -52,8 +52,14 @@ int vhost_net_set_config(struct vhost_net *net, const uint8_t *data, void vhost_net_ack_features_ex(struct vhost_net *net, const uint64_t *features) { - virtio_features_copy(net->dev.acked_features_ex, - net->dev.backend_features_ex); + virtio_features_clear(net->dev.acked_features_ex); + if (net->backend == -1) { + net->dev.acked_features = (vhost_dev_features(&net->dev) & + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)); + } else if (!qemu_has_vnet_hdr(net->nc)) { + net->dev.acked_features = 1ULL << VHOST_NET_F_VIRTIO_NET_HDR; + } + vhost_ack_features_ex(&net->dev, net->feature_bits, features); } @@ -258,13 +264,8 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options) if (r < 0) { goto fail; } - net->dev.backend_features = qemu_has_vnet_hdr(options->net_backend) - ? 0 : (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); net->backend = r; - net->dev.protocol_features = 0; } else { - virtio_features_clear(net->dev.backend_features_ex); - net->dev.protocol_features = 0; net->backend = -1; /* vhost-user needs vq_index to initiate a specific queue pair */ @@ -281,14 +282,13 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options) if (backend_kernel) { if (!qemu_has_vnet_hdr_len(options->net_backend, sizeof(struct virtio_net_hdr_mrg_rxbuf))) { - net->dev.features &= ~(1ULL << VIRTIO_NET_F_MRG_RXBUF); + vhost_dev_clear_feature(&net->dev, VIRTIO_NET_F_MRG_RXBUF); } - if (virtio_features_andnot(missing_features, - net->dev.backend_features_ex, - net->dev.features_ex)) { - fprintf(stderr, "vhost lacks feature mask 0x" VIRTIO_FEATURES_FMT - " for backend\n", VIRTIO_FEATURES_PR(missing_features)); + if (!qemu_has_vnet_hdr(options->net_backend) && + !vhost_dev_has_feature(&net->dev, VHOST_NET_F_VIRTIO_NET_HDR)) { + fprintf(stderr, "vhost lacks VHOST_NET_F_VIRTIO_NET_HDR " + "feature for backend\n"); goto fail; } } @@ -298,7 +298,7 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options) virtio_features_from_u64(features, options->get_acked_features(net->nc)); if (virtio_features_andnot(missing_features, features, - net->dev.features_ex)) { + vhost_dev_features_ex(&net->dev))) { fprintf(stderr, "vhost lacks feature mask 0x" VIRTIO_FEATURES_FMT " for backend\n", VIRTIO_FEATURES_PR(missing_features)); goto fail; @@ -587,7 +587,6 @@ VHostNetState *get_vhost_net(NetClientState *nc) int vhost_net_set_vring_enable(NetClientState *nc, int enable) { VHostNetState *net = get_vhost_net(nc); - const VhostOps *vhost_ops = net->dev.vhost_ops; /* * vhost-vdpa network devices need to enable dataplane virtqueues after @@ -601,11 +600,7 @@ int vhost_net_set_vring_enable(NetClientState *nc, int enable) nc->vring_enable = enable; - if (vhost_ops && vhost_ops->vhost_set_vring_enable) { - return vhost_ops->vhost_set_vring_enable(&net->dev, enable); - } - - return 0; + return vhost_dev_set_vring_enable(&net->dev, enable); } int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) diff --git a/hw/pci-host/pnv_phb3.c b/hw/pci-host/pnv_phb3.c index d6ab515337..4ffdb4ce31 100644 --- a/hw/pci-host/pnv_phb3.c +++ b/hw/pci-host/pnv_phb3.c @@ -475,6 +475,11 @@ void pnv_phb3_reg_write(void *opaque, hwaddr off, uint64_t val, unsigned size) /* Special case configuration data */ if ((off & 0xfffc) == PHB_CONFIG_DATA) { + if (size > 4) { + phb3_error(phb, "Invalid config access, offset: 0x%"PRIx64" size: %d", + off, size); + return; + } pnv_phb3_config_write(phb, off & 0x3, size, val); return; } @@ -597,6 +602,11 @@ uint64_t pnv_phb3_reg_read(void *opaque, hwaddr off, unsigned size) uint64_t val; if ((off & 0xfffc) == PHB_CONFIG_DATA) { + if (size > 4) { + phb3_error(phb, "Invalid config access, offset: 0x%"PRIx64" size: %d", + off, size); + return ~0ull; + } return pnv_phb3_config_read(phb, off & 0x3, size); } diff --git a/hw/pci-host/q35.c b/hw/pci-host/q35.c index 355e81bfa2..4784b8f59b 100644 --- a/hw/pci-host/q35.c +++ b/hw/pci-host/q35.c @@ -317,12 +317,12 @@ static void mch_update_pciexbar(MCHPCIState *mch) break; case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_128M: length = 128 * 1024 * 1024; - addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK | - MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK; + addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK; break; case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_64M: length = 64 * 1024 * 1024; - addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK; + addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK | + MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK; break; case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_RVD: qemu_log_mask(LOG_GUEST_ERROR, "Q35: Reserved PCIEXBAR LENGTH\n"); diff --git a/hw/pci/pci_host.c b/hw/pci/pci_host.c index 91e3885c7f..2a7fdfa563 100644 --- a/hw/pci/pci_host.c +++ b/hw/pci/pci_host.c @@ -81,7 +81,12 @@ void pci_host_config_write_common(PCIDevice *pci_dev, uint32_t addr, return; } - assert(len <= 4); + if (len > 4) { + PCI_DPRINTF("%s: invalid length access: addr " HWADDR_FMT_plx " \ + len %d val %"PRIx32"\n", __func__, addr, len, val); + return; + } + /* non-zero functions are only exposed when function 0 is present, * allowing direct removal of unexposed functions. */ @@ -106,7 +111,12 @@ uint32_t pci_host_config_read_common(PCIDevice *pci_dev, uint32_t addr, return ~0x0; } - assert(len <= 4); + if (len > 4) { + PCI_DPRINTF("%s: invalid length access: addr " HWADDR_FMT_plx " \ + len %d val %"PRIx32"\n", __func__, addr, len, val); + return ~0x0; + } + /* non-zero functions are only exposed when function 0 is present, * allowing direct removal of unexposed functions. */ diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c index ba55bf4fe9..cdad862ec2 100644 --- a/hw/s390x/virtio-ccw.c +++ b/hw/s390x/virtio-ccw.c @@ -1025,20 +1025,27 @@ static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n, VirtQueue *vq = virtio_get_queue(vdev, n); EventNotifier *notifier = virtio_queue_get_guest_notifier(vq); VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); + int r; + + if (!assign) { + if (k->guest_notifier_mask && vdev->use_guest_notifier_mask) { + k->guest_notifier_mask(vdev, n, true); + } + if (with_irqfd) { + virtio_ccw_remove_irqfd(dev, n); + } + } + + r = virtio_set_guest_notifier(vdev, n, assign, with_irqfd); + if (r < 0) { + return r; + } if (assign) { - int r = event_notifier_init(notifier, 0); - - if (r < 0) { - return r; - } - virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd); if (with_irqfd) { r = virtio_ccw_add_irqfd(dev, n); if (r) { - virtio_queue_set_guest_notifier_fd_handler(vq, false, - with_irqfd); - event_notifier_cleanup(notifier); + virtio_set_guest_notifier(vdev, n, false, with_irqfd); return r; } } @@ -1054,16 +1061,8 @@ static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n, k->guest_notifier_pending(vdev, n)) { event_notifier_set(notifier); } - } else { - if (k->guest_notifier_mask && vdev->use_guest_notifier_mask) { - k->guest_notifier_mask(vdev, n, true); - } - if (with_irqfd) { - virtio_ccw_remove_irqfd(dev, n); - } - virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd); - event_notifier_cleanup(notifier); } + return 0; } diff --git a/hw/scsi/vhost-scsi-common.c b/hw/scsi/vhost-scsi-common.c index 0bb4305de6..e19800a0bc 100644 --- a/hw/scsi/vhost-scsi-common.c +++ b/hw/scsi/vhost-scsi-common.c @@ -113,7 +113,7 @@ int vhost_scsi_common_stop(VHostSCSICommon *vsc) if (k->set_guest_notifiers) { int r = k->set_guest_notifiers(qbus->parent, vsc->dev.nvqs, false); if (r < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); + error_report("vhost guest notifier cleanup failed: %d", r); return r; } } diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c index 699863cc10..657403cad0 100644 --- a/hw/scsi/vhost-scsi.c +++ b/hw/scsi/vhost-scsi.c @@ -15,6 +15,7 @@ */ #include "qemu/osdep.h" +#include "standard-headers/linux/vhost_types.h" #include #include #include "qapi/error.h" @@ -275,7 +276,6 @@ static void vhost_scsi_realize(DeviceState *dev, Error **errp) vqs = g_new0(struct vhost_virtqueue, vsc->dev.nvqs); vsc->dev.vqs = vqs; vsc->dev.vq_index = 0; - vsc->dev.backend_features = 0; ret = vhost_dev_init(&vsc->dev, (void *)(uintptr_t)vhostfd, VHOST_BACKEND_TYPE_KERNEL, 0, errp); diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c index 3612897d4b..5070178dc2 100644 --- a/hw/scsi/vhost-user-scsi.c +++ b/hw/scsi/vhost-user-scsi.c @@ -159,7 +159,6 @@ static int vhost_user_scsi_connect(DeviceState *dev, Error **errp) vsc->dev.nvqs = VIRTIO_SCSI_VQ_NUM_FIXED + vs->conf.num_queues; vsc->dev.vqs = s->vhost_vqs; vsc->dev.vq_index = 0; - vsc->dev.backend_features = 0; ret = vhost_dev_init(&vsc->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0, errp); diff --git a/hw/vfio/device.c b/hw/vfio/device.c index ccf14743e9..1a7f8088aa 100644 --- a/hw/vfio/device.c +++ b/hw/vfio/device.c @@ -543,6 +543,17 @@ bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev) return false; } +bool vfio_device_get_viommu_flags_want_pasid_attach(VFIODevice *vbasedev) +{ + VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev); + + if (vdev) { + return !!(pci_device_get_viommu_flags(PCI_DEVICE(vdev)) & + VIOMMU_FLAG_WANT_PASID_ATTACH); + } + return false; +} + bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev, uint32_t type, void *caps, uint32_t size) diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c index df148a49a7..0718f029ca 100644 --- a/hw/vfio/iommufd.c +++ b/hw/vfio/iommufd.c @@ -20,6 +20,7 @@ #include "trace.h" #include "qapi/error.h" #include "system/iommufd.h" +#include "hw/core/iommu.h" #include "hw/core/qdev.h" #include "hw/vfio/vfio-cpr.h" #include "system/reset.h" @@ -305,43 +306,48 @@ out: return ret; } -static int iommufd_cdev_attach_ioas_hwpt(VFIODevice *vbasedev, uint32_t id, - Error **errp) +static int iommufd_cdev_pasid_attach_ioas_hwpt(VFIODevice *vbasedev, + uint32_t pasid, uint32_t id, + Error **errp) { int iommufd = vbasedev->iommufd->fd; struct vfio_device_attach_iommufd_pt attach_data = { .argsz = sizeof(attach_data), - .flags = 0, + .flags = pasid == IOMMU_NO_PASID ? 0 : VFIO_DEVICE_ATTACH_PASID, + .pasid = pasid, .pt_id = id, }; /* Attach device to an IOAS or hwpt within iommufd */ if (ioctl(vbasedev->fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data)) { error_setg_errno(errp, errno, - "[iommufd=%d] error attach %s (%d) to id=%d", - iommufd, vbasedev->name, vbasedev->fd, id); + "[iommufd=%d] error attach %s (%d) pasid %d to id=%d", + iommufd, vbasedev->name, vbasedev->fd, pasid, id); return -errno; } - trace_iommufd_cdev_attach_ioas_hwpt(iommufd, vbasedev->name, - vbasedev->fd, id); + trace_iommufd_cdev_pasid_attach_ioas_hwpt(iommufd, vbasedev->name, + vbasedev->fd, pasid, id); return 0; } -static bool iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp) +static bool iommufd_cdev_pasid_detach_ioas_hwpt(VFIODevice *vbasedev, + uint32_t pasid, Error **errp) { int iommufd = vbasedev->iommufd->fd; struct vfio_device_detach_iommufd_pt detach_data = { .argsz = sizeof(detach_data), - .flags = 0, + .flags = pasid == IOMMU_NO_PASID ? 0 : VFIO_DEVICE_DETACH_PASID, + .pasid = pasid, }; if (ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, &detach_data)) { - error_setg_errno(errp, errno, "detach %s failed", vbasedev->name); + error_setg_errno(errp, errno, "detach %s pasid %d failed", + vbasedev->name, pasid); return false; } - trace_iommufd_cdev_detach_ioas_hwpt(iommufd, vbasedev->name); + trace_iommufd_cdev_pasid_detach_ioas_hwpt(iommufd, vbasedev->name, pasid); return true; } @@ -358,12 +364,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, VendorCaps caps; VFIOIOASHwpt *hwpt; uint32_t hwpt_id; + uint8_t max_pasid_log2 = 0; int ret; /* Try to find a domain */ QLIST_FOREACH(hwpt, &container->hwpt_list, next) { if (!cpr_is_incoming()) { - ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt->hwpt_id, errp); + ret = iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, IOMMU_NO_PASID, + hwpt->hwpt_id, errp); } else if (vbasedev->cpr.hwpt_id == hwpt->hwpt_id) { ret = 0; } else { @@ -402,7 +410,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, */ if (!iommufd_backend_get_device_info(vbasedev->iommufd, vbasedev->devid, &type, &caps, sizeof(caps), &hw_caps, - NULL, errp)) { + &max_pasid_log2, errp)) { return false; } @@ -430,6 +438,11 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, } } + if (max_pasid_log2 && + vfio_device_get_viommu_flags_want_pasid_attach(vbasedev)) { + flags |= IOMMU_HWPT_ALLOC_PASID; + } + if (cpr_is_incoming()) { hwpt_id = vbasedev->cpr.hwpt_id; goto skip_alloc; @@ -442,7 +455,8 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, return false; } - ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt_id, errp); + ret = iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, IOMMU_NO_PASID, hwpt_id, + errp); if (ret) { iommufd_backend_free_id(container->be, hwpt_id); return false; @@ -495,7 +509,8 @@ static bool iommufd_cdev_attach_container(VFIODevice *vbasedev, /* If CPR, we are already attached to ioas_id. */ return cpr_is_incoming() || - !iommufd_cdev_attach_ioas_hwpt(vbasedev, container->ioas_id, errp); + !iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, IOMMU_NO_PASID, + container->ioas_id, errp); } static void iommufd_cdev_detach_container(VFIODevice *vbasedev, @@ -503,7 +518,7 @@ static void iommufd_cdev_detach_container(VFIODevice *vbasedev, { Error *err = NULL; - if (!iommufd_cdev_detach_ioas_hwpt(vbasedev, &err)) { + if (!iommufd_cdev_pasid_detach_ioas_hwpt(vbasedev, IOMMU_NO_PASID, &err)) { error_report_err(err); } @@ -925,20 +940,21 @@ static void vfio_iommu_iommufd_class_init(ObjectClass *klass, const void *data) static bool host_iommu_device_iommufd_vfio_attach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, - uint32_t hwpt_id, Error **errp) -{ - VFIODevice *vbasedev = HOST_IOMMU_DEVICE(hiodi)->agent; - - return !iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt_id, errp); -} - -static bool -host_iommu_device_iommufd_vfio_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, + uint32_t pasid, uint32_t hwpt_id, Error **errp) { VFIODevice *vbasedev = HOST_IOMMU_DEVICE(hiodi)->agent; - return iommufd_cdev_detach_ioas_hwpt(vbasedev, errp); + return !iommufd_cdev_pasid_attach_ioas_hwpt(vbasedev, pasid, hwpt_id, errp); +} + +static bool +host_iommu_device_iommufd_vfio_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, + uint32_t pasid, Error **errp) +{ + VFIODevice *vbasedev = HOST_IOMMU_DEVICE(hiodi)->agent; + + return iommufd_cdev_pasid_detach_ioas_hwpt(vbasedev, pasid, errp); } static bool hiod_iommufd_vfio_realize(HostIOMMUDevice *hiod, void *opaque, diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index 2049159015..8dbb477298 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -183,8 +183,8 @@ vfio_vmstate_change_prepare(const char *name, int running, const char *reason, c iommufd_cdev_connect_and_bind(int iommufd, const char *name, int devfd, int devid) " [iommufd=%d] Successfully bound device %s (fd=%d): output devid=%d" iommufd_cdev_getfd(const char *dev, int devfd) " %s (fd=%d)" -iommufd_cdev_attach_ioas_hwpt(int iommufd, const char *name, int devfd, int id) " [iommufd=%d] Successfully attached device %s (%d) to id=%d" -iommufd_cdev_detach_ioas_hwpt(int iommufd, const char *name) " [iommufd=%d] Successfully detached %s" +iommufd_cdev_pasid_attach_ioas_hwpt(int iommufd, const char *name, int devfd, uint32_t pasid, int id) " [iommufd=%d] Successfully attached device %s (%d) pasid %u to id=%d" +iommufd_cdev_pasid_detach_ioas_hwpt(int iommufd, const char *name, uint32_t pasid) " [iommufd=%d] Successfully detached %s pasid %u" iommufd_cdev_fail_attach_existing_container(const char *msg) " %s" iommufd_cdev_alloc_ioas(int iommufd, int ioas_id) " [iommufd=%d] new IOMMUFD container with ioasid=%d" iommufd_cdev_device_info(char *name, int devfd, int num_irqs, int num_regions, int flags) " %s (%d) num_irqs=%d num_regions=%d flags=%d" diff --git a/hw/virtio/Kconfig b/hw/virtio/Kconfig index 2ebdac16b3..91cf833407 100644 --- a/hw/virtio/Kconfig +++ b/hw/virtio/Kconfig @@ -76,6 +76,11 @@ config VIRTIO_MEM depends on VIRTIO_MEM_SUPPORTED select VIRTIO_MD +config VIRTIO_RTC + bool + default y + depends on VIRTIO + config VHOST_VSOCK_COMMON bool depends on VIRTIO diff --git a/hw/virtio/meson.build b/hw/virtio/meson.build index e0566a60de..0e02323516 100644 --- a/hw/virtio/meson.build +++ b/hw/virtio/meson.build @@ -14,7 +14,7 @@ system_virtio_ss.add(files('virtio-qmp.c')) if have_vhost system_virtio_ss.add(files('vhost.c')) - system_virtio_ss.add(files('vhost-backend.c', 'vhost-iova-tree.c')) + system_virtio_ss.add(files('vhost-kernel.c', 'vhost-iova-tree.c')) if have_vhost_user system_virtio_ss.add(files('vhost-user.c')) system_virtio_ss.add(files('vhost-user-base.c')) @@ -54,6 +54,7 @@ if have_vhost endif system_virtio_ss.add(when: 'CONFIG_VHOST_USER_VSOCK', if_true: files('vhost-user-vsock.c')) system_virtio_ss.add(when: 'CONFIG_VIRTIO_RNG', if_true: files('virtio-rng.c')) +system_virtio_ss.add(when: 'CONFIG_VIRTIO_RTC', if_true: files('virtio-rtc.c')) system_virtio_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon.c')) system_virtio_ss.add(when: 'CONFIG_VHOST_USER_FS', if_true: files('vhost-user-fs.c')) @@ -77,6 +78,7 @@ virtio_pci_ss.add(when: 'CONFIG_VIRTIO_CRYPTO', if_true: files('virtio-crypto-pc virtio_pci_ss.add(when: 'CONFIG_VIRTIO_INPUT_HOST', if_true: files('virtio-input-host-pci.c')) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_INPUT', if_true: files('virtio-input-pci.c')) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_RNG', if_true: files('virtio-rng-pci.c')) +virtio_pci_ss.add(when: 'CONFIG_VIRTIO_RTC', if_true: files('virtio-rtc-pci.c')) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_NSM', if_true: [files('virtio-nsm-pci.c', 'cbor-helpers.c'), libcbor]) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_BALLOON', if_true: files('virtio-balloon-pci.c')) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_9P', if_true: files('virtio-9p-pci.c')) @@ -94,6 +96,7 @@ system_virtio_ss.add_all(when: 'CONFIG_VIRTIO_PCI', if_true: virtio_pci_ss) system_ss.add_all(when: 'CONFIG_VIRTIO', if_true: system_virtio_ss) stub_ss.add(files('vhost-stub.c')) +stub_ss.add(files('vhost-user-stub.c')) stub_ss.add(files('virtio-stub.c')) stub_ss.add(files('virtio-md-stubs.c')) diff --git a/hw/virtio/trace-events b/hw/virtio/trace-events index 658cc365e7..2a57edc21e 100644 --- a/hw/virtio/trace-events +++ b/hw/virtio/trace-events @@ -9,8 +9,16 @@ vhost_section(const char *name) "%s" vhost_reject_section(const char *name, int d) "%s:%d" vhost_iotlb_miss(void *dev, int step) "%p step %d" vhost_dev_cleanup(void *dev) "%p" -vhost_dev_start(void *dev, const char *name, bool vrings) "%p:%s vrings:%d" -vhost_dev_stop(void *dev, const char *name, bool vrings) "%p:%s vrings:%d" +vhost_dev_start_in(void *dev, const char *name, bool vrings) "%p:%s vrings:%d" +vhost_dev_start_out(void *dev, const char *name) "%p:%s" +vhost_dev_stop_in(void *dev, const char *name, bool vrings) "%p:%s vrings:%d" +vhost_dev_stop_out(void *dev, const char *name) "%p:%s" +vhost_virtqueue_start_in(void *dev, const char *name, int idx) "%p:%s %d" +vhost_virtqueue_start_out(void *dev, const char *name, int idx) "%p:%s %d" +vhost_virtqueue_stop_in(void *dev, const char *name, int idx) "%p:%s %d" +vhost_virtqueue_stop_out(void *dev, const char *name, int idx) "%p:%s %d" +vhost_dev_init_in(void *dev) "%p" +vhost_dev_init_out(void *dev) "%p" # vhost-user.c @@ -25,8 +33,8 @@ vhost_user_set_mem_table_withfd(int index, const char *name, uint64_t memory_siz vhost_user_postcopy_waker(const char *rb, uint64_t rb_offset) "%s + 0x%"PRIx64 vhost_user_postcopy_waker_found(uint64_t client_addr) "0x%"PRIx64 vhost_user_postcopy_waker_nomatch(const char *rb, uint64_t rb_offset) "%s + 0x%"PRIx64 -vhost_user_read(uint32_t req, uint32_t flags) "req:%d flags:0x%"PRIx32"" -vhost_user_write(uint32_t req, uint32_t flags) "req:%d flags:0x%"PRIx32"" +vhost_user_read(uint32_t req, const char *req_name, uint32_t flags) "req:%d (%s) flags:0x%"PRIx32"" +vhost_user_write(uint32_t req, const char *req_name, uint32_t flags) "req:%d (%s) flags:0x%"PRIx32"" vhost_user_create_notifier(int idx, void *n) "idx:%d n:%p" # vhost-vdpa.c diff --git a/hw/virtio/vdpa-dev.c b/hw/virtio/vdpa-dev.c index 4532d63653..94188d37bb 100644 --- a/hw/virtio/vdpa-dev.c +++ b/hw/virtio/vdpa-dev.c @@ -104,7 +104,6 @@ static void vhost_vdpa_device_realize(DeviceState *dev, Error **errp) v->dev.vqs = vqs; v->dev.vq_index = 0; v->dev.vq_index_end = v->dev.nvqs; - v->dev.backend_features = 0; v->started = false; ret = vhost_vdpa_get_iova_range(v->vhostfd, &iova_range); @@ -225,7 +224,7 @@ static uint64_t vhost_vdpa_device_get_features(VirtIODevice *vdev, Error **errp) { VhostVdpaDevice *s = VHOST_VDPA_DEVICE(vdev); - uint64_t backend_features = s->dev.features; + uint64_t backend_features = vhost_dev_features(&s->dev); if (!virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM)) { virtio_clear_feature(&backend_features, VIRTIO_F_IOMMU_PLATFORM); diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-kernel.c similarity index 83% rename from hw/virtio/vhost-backend.c rename to hw/virtio/vhost-kernel.c index 4367db0d95..3390b48c6f 100644 --- a/hw/virtio/vhost-backend.c +++ b/hw/virtio/vhost-kernel.c @@ -1,5 +1,5 @@ /* - * vhost-backend + * vhost-kernel * * Copyright (c) 2013 Virtual Open Systems Sarl. * @@ -298,7 +298,7 @@ static void vhost_kernel_iotlb_read(void *opaque) break; } - vhost_backend_handle_iotlb_msg(dev, &msg.iotlb); + vhost_handle_iotlb_msg(dev, &msg.iotlb); } } else { struct vhost_msg msg; @@ -313,7 +313,7 @@ static void vhost_kernel_iotlb_read(void *opaque) break; } - vhost_backend_handle_iotlb_msg(dev, &msg.iotlb); + vhost_handle_iotlb_msg(dev, &msg.iotlb); } } } @@ -358,9 +358,9 @@ static void vhost_kernel_set_iotlb_callback(struct vhost_dev *dev, const VhostOps kernel_ops = { .backend_type = VHOST_BACKEND_TYPE_KERNEL, - .vhost_backend_init = vhost_kernel_init, - .vhost_backend_cleanup = vhost_kernel_cleanup, - .vhost_backend_memslots_limit = vhost_kernel_memslots_limit, + .vhost_init = vhost_kernel_init, + .vhost_cleanup = vhost_kernel_cleanup, + .vhost_memslots_limit = vhost_kernel_memslots_limit, .vhost_net_set_backend = vhost_kernel_net_set_backend, .vhost_scsi_set_endpoint = vhost_kernel_scsi_set_endpoint, .vhost_scsi_clear_endpoint = vhost_kernel_scsi_clear_endpoint, @@ -392,81 +392,3 @@ const VhostOps kernel_ops = { .vhost_send_device_iotlb_msg = vhost_kernel_send_device_iotlb_msg, }; #endif - -int vhost_backend_update_device_iotlb(struct vhost_dev *dev, - uint64_t iova, uint64_t uaddr, - uint64_t len, - IOMMUAccessFlags perm) -{ - struct vhost_iotlb_msg imsg; - - imsg.iova = iova; - imsg.uaddr = uaddr; - imsg.size = len; - imsg.type = VHOST_IOTLB_UPDATE; - - switch (perm) { - case IOMMU_RO: - imsg.perm = VHOST_ACCESS_RO; - break; - case IOMMU_WO: - imsg.perm = VHOST_ACCESS_WO; - break; - case IOMMU_RW: - imsg.perm = VHOST_ACCESS_RW; - break; - default: - return -EINVAL; - } - - if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg) - return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg); - - return -ENODEV; -} - -int vhost_backend_invalidate_device_iotlb(struct vhost_dev *dev, - uint64_t iova, uint64_t len) -{ - struct vhost_iotlb_msg imsg; - - imsg.iova = iova; - imsg.size = len; - imsg.type = VHOST_IOTLB_INVALIDATE; - - if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg) - return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg); - - return -ENODEV; -} - -int vhost_backend_handle_iotlb_msg(struct vhost_dev *dev, - struct vhost_iotlb_msg *imsg) -{ - int ret = 0; - - if (unlikely(!dev->vdev)) { - error_report("Unexpected IOTLB message when virtio device is stopped"); - return -EINVAL; - } - - switch (imsg->type) { - case VHOST_IOTLB_MISS: - ret = vhost_device_iotlb_miss(dev, imsg->iova, - imsg->perm != VHOST_ACCESS_RO); - break; - case VHOST_IOTLB_ACCESS_FAIL: - /* FIXME: report device iotlb error */ - error_report("Access failure IOTLB message type not supported"); - ret = -ENOTSUP; - break; - case VHOST_IOTLB_UPDATE: - case VHOST_IOTLB_INVALIDATE: - default: - error_report("Unexpected IOTLB message type"); - ret = -EINVAL; - break; - } - - return ret; -} diff --git a/hw/virtio/vhost-shadow-virtqueue.c b/hw/virtio/vhost-shadow-virtqueue.c index 6242aeb69c..bcb7f2ffc7 100644 --- a/hw/virtio/vhost-shadow-virtqueue.c +++ b/hw/virtio/vhost-shadow-virtqueue.c @@ -12,11 +12,14 @@ #include "qemu/error-report.h" #include "qapi/error.h" +#include "qemu/iov.h" #include "qemu/main-loop.h" #include "qemu/log.h" #include "qemu/memalign.h" #include "linux-headers/linux/vhost.h" +#define VIRTIO_RING_NOT_IN_BATCH UINT16_MAX + /** * Validate the transport device features that both guests can use with the SVQ * and SVQs can use with the device. @@ -35,6 +38,7 @@ bool vhost_svq_valid_features(uint64_t features, Error **errp) case VIRTIO_F_ANY_LAYOUT: case VIRTIO_RING_F_EVENT_IDX: case VIRTIO_RING_F_INDIRECT_DESC: + case VIRTIO_F_IN_ORDER: continue; case VIRTIO_F_ACCESS_PLATFORM: @@ -139,6 +143,46 @@ static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq, return true; } +/** + * Get the next descriptor in the chain in SVQ vring from a descriptor id + * + * @svq Shadow Virtqueue + * @id ID of the descriptor + * + * Return the id of the next descriptor. + */ +static uint16_t vhost_svq_next_desc(const VhostShadowVirtqueue *svq, + uint16_t id) +{ + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) { + return (id == svq->vring.num) ? 0 : ++id; + } else { + return svq->desc_state[id].next; + } +} + +/** + * Updates the SVQ free_head member after adding them to the SVQ avail ring. + * The new free_head is the next descriptor that SVQ will make available by + * forwarding a new guest descriptor. + * + * @svq Shadow Virtqueue + * @num Number of descriptors added + * @id ID of the last descriptor added to the SVQ avail ring. + */ +static void vhost_svq_update_free_head(VhostShadowVirtqueue *svq, + size_t num, uint16_t id) +{ + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) { + svq->free_head += num; + if (svq->free_head >= svq->vring.num) { + svq->free_head -= svq->vring.num; + } + } else { + svq->free_head = vhost_svq_next_desc(svq, id); + } +} + /** * Write descriptors to SVQ vring * @@ -173,9 +217,11 @@ static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg, } for (n = 0; n < num; n++) { + uint16_t next = vhost_svq_next_desc(svq, i); + if (more_descs || (n + 1 < num)) { descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT); - descs[i].next = cpu_to_le16(svq->desc_next[i]); + descs[i].next = cpu_to_le16(next); } else { descs[i].flags = flags; } @@ -183,10 +229,10 @@ static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg, descs[i].len = cpu_to_le32(iovec[n].iov_len); last = i; - i = svq->desc_next[i]; + i = next; } - svq->free_head = svq->desc_next[last]; + vhost_svq_update_free_head(svq, num, last); return true; } @@ -290,6 +336,9 @@ int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg, svq->num_free -= ndescs; svq->desc_state[qemu_head].elem = elem; svq->desc_state[qemu_head].ndescs = ndescs; + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) { + svq->desc_state[qemu_head].in_bytes = iov_size(in_sg, in_num); + } vhost_svq_kick(svq); return 0; } @@ -385,6 +434,12 @@ static void vhost_handle_guest_kick_notifier(EventNotifier *n) static bool vhost_svq_more_used(VhostShadowVirtqueue *svq) { uint16_t *used_idx = &svq->vring.used->idx; + + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER) && + svq->batch_last.id != VIRTIO_RING_NOT_IN_BATCH) { + return true; + } + if (svq->last_used_idx != svq->shadow_used_idx) { return true; } @@ -428,23 +483,121 @@ static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq) } } +/* + * Gets the next buffer id and moves forward the used idx, so the next time + * SVQ calls this function will get the next one. + * + * @svq: Shadow VirtQueue + * @len: Consumed length by the device. + * + * Return the next descriptor consumed by the device. + */ +static uint16_t vhost_svq_get_last_used_split(VhostShadowVirtqueue *svq, + uint32_t *len) +{ + const vring_used_t *used = svq->vring.used; + uint16_t last_used = svq->last_used_idx++ & (svq->vring.num - 1); + + *len = le32_to_cpu(used->ring[last_used].len); + return le32_to_cpu(used->ring[last_used].id); +} + +/* + * Gets the next buffer id and moves forward the used idx, so the next time + * SVQ calls this function will get the next one. IN_ORDER version + * + * @svq: Shadow VirtQueue + * @len: Consumed length by the device. + * + * Return the next descriptor consumed by the device. + */ +static int32_t vhost_svq_get_last_used_split_in_order( + VhostShadowVirtqueue *svq, + uint32_t *len) +{ + unsigned num = svq->vring.num; + const vring_used_t *used = svq->vring.used; + uint16_t last_used = svq->last_used & (num - 1); + uint16_t last_used_idx = svq->last_used_idx & (num - 1); + + if (svq->batch_last.id == VIRTIO_RING_NOT_IN_BATCH) { + svq->batch_last.id = le32_to_cpu(used->ring[last_used_idx].id); + svq->batch_last.len = le32_to_cpu(used->ring[last_used_idx].len); + } + + if (unlikely(last_used >= num)) { + qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used", + svq->vdev->name, last_used); + return -1; + } + + if (svq->batch_last.id == last_used) { + svq->batch_last.id = VIRTIO_RING_NOT_IN_BATCH; + *len = svq->batch_last.len; + } else { + *len = svq->desc_state[last_used].in_bytes; + } + + svq->last_used += svq->desc_state[last_used].ndescs; + svq->last_used_idx++; + return last_used; +} + static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq, uint16_t num, uint16_t i) { for (uint16_t j = 0; j < (num - 1); ++j) { - i = svq->desc_next[i]; + i = vhost_svq_next_desc(svq, i); } return i; } +G_GNUC_WARN_UNUSED_RESULT +static VirtQueueElement *vhost_svq_detach_buf_split(VhostShadowVirtqueue *svq, + uint16_t id) +{ + uint16_t num = svq->desc_state[id].ndescs; + uint16_t last_used_chain = vhost_svq_last_desc_of_chain(svq, num, id); + + svq->desc_state[last_used_chain].next = svq->free_head; + svq->free_head = id; + + return g_steal_pointer(&svq->desc_state[id].elem); +} + +G_GNUC_WARN_UNUSED_RESULT +static VirtQueueElement *vhost_svq_detach_buf_split_in_order( + VhostShadowVirtqueue *svq, + uint16_t id) +{ + return g_steal_pointer(&svq->desc_state[id].elem); +} + +/* + * Return the descriptor id (and the chain of ids) to the free list + * + * @svq: Shadow Virtqueue + * @id: Id of the buffer to return. + * + * Return the element associated to the buffer if any. + */ +G_GNUC_WARN_UNUSED_RESULT +static VirtQueueElement *vhost_svq_detach_buf(VhostShadowVirtqueue *svq, + uint16_t id) +{ + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) { + return vhost_svq_detach_buf_split_in_order(svq, id); + } else { + return vhost_svq_detach_buf_split(svq, id); + } +} + G_GNUC_WARN_UNUSED_RESULT static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq, uint32_t *len) { - const vring_used_t *used = svq->vring.used; - vring_used_elem_t used_elem; - uint16_t last_used, last_used_chain, num; + uint16_t last_used; if (!vhost_svq_more_used(svq)) { return NULL; @@ -452,33 +605,35 @@ static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq, /* Only get used array entries after they have been exposed by dev */ smp_rmb(); - last_used = svq->last_used_idx & (svq->vring.num - 1); - used_elem.id = le32_to_cpu(used->ring[last_used].id); - used_elem.len = le32_to_cpu(used->ring[last_used].len); - svq->last_used_idx++; - if (unlikely(used_elem.id >= svq->vring.num)) { + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) { + int32_t r; + r = vhost_svq_get_last_used_split_in_order(svq, len); + if (r < 0) { + return NULL; + } + + last_used = r; + } else { + last_used = vhost_svq_get_last_used_split(svq, len); + } + + if (unlikely(last_used >= svq->vring.num)) { qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used", - svq->vdev->name, used_elem.id); + svq->vdev->name, last_used); return NULL; } - if (unlikely(!svq->desc_state[used_elem.id].ndescs)) { + if (unlikely(!svq->desc_state[last_used].ndescs)) { qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used, but it was not available", - svq->vdev->name, used_elem.id); + svq->vdev->name, last_used); return NULL; } - num = svq->desc_state[used_elem.id].ndescs; - svq->desc_state[used_elem.id].ndescs = 0; - last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id); - svq->desc_next[last_used_chain] = svq->free_head; - svq->free_head = used_elem.id; - svq->num_free += num; - - *len = used_elem.len; - return g_steal_pointer(&svq->desc_state[used_elem.id].elem); + svq->num_free += svq->desc_state[last_used].ndescs; + svq->desc_state[last_used].ndescs = 0; + return vhost_svq_detach_buf(svq, last_used); } /** @@ -689,6 +844,8 @@ void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev, svq->next_guest_avail_elem = NULL; svq->shadow_avail_idx = 0; svq->shadow_used_idx = 0; + memset(&svq->batch_last, 0, sizeof(svq->batch_last)); + svq->last_used = 0; svq->last_used_idx = 0; svq->vdev = vdev; svq->vq = vq; @@ -705,9 +862,12 @@ void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); svq->desc_state = g_new0(SVQDescState, svq->vring.num); - svq->desc_next = g_new0(uint16_t, svq->vring.num); - for (unsigned i = 0; i < svq->vring.num - 1; i++) { - svq->desc_next[i] = i + 1; + if (virtio_vdev_has_feature(svq->vdev, VIRTIO_F_IN_ORDER)) { + svq->batch_last.id = VIRTIO_RING_NOT_IN_BATCH; + } else { + for (unsigned i = 0; i < svq->vring.num - 1; i++) { + svq->desc_state[i].next = i + 1; + } } } @@ -744,7 +904,6 @@ void vhost_svq_stop(VhostShadowVirtqueue *svq) virtqueue_unpop(svq->vq, next_avail_elem, 0); } svq->vq = NULL; - g_free(svq->desc_next); g_free(svq->desc_state); munmap(svq->vring.desc, vhost_svq_driver_area_size(svq)); munmap(svq->vring.used, vhost_svq_device_area_size(svq)); diff --git a/hw/virtio/vhost-shadow-virtqueue.h b/hw/virtio/vhost-shadow-virtqueue.h index 9c273739d6..ec16a1e838 100644 --- a/hw/virtio/vhost-shadow-virtqueue.h +++ b/hw/virtio/vhost-shadow-virtqueue.h @@ -23,6 +23,20 @@ typedef struct SVQDescState { * guest's */ unsigned int ndescs; + + union { + /* + * Total length of the available buffer that is writable by the device. + * Only used in packed vq. + */ + uint32_t in_bytes; + + /* + * Backup next field for each descriptor so we can recover securely, not + * needing to trust the device access. Only used in split vq. + */ + uint16_t next; + }; } SVQDescState; typedef struct VhostShadowVirtqueue VhostShadowVirtqueue; @@ -84,12 +98,6 @@ typedef struct VhostShadowVirtqueue { /* Next VirtQueue element that guest made available */ VirtQueueElement *next_guest_avail_elem; - /* - * Backup next field for each descriptor so we can recover securely, not - * needing to trust the device access. - */ - uint16_t *desc_next; - /* Caller callbacks */ const VhostShadowVirtqueueOps *ops; @@ -99,9 +107,25 @@ typedef struct VhostShadowVirtqueue { /* Next head to expose to the device */ uint16_t shadow_avail_idx; - /* Next free descriptor */ + /* + * Next free descriptor. + * + * Without IN_ORDER free_head is used as a linked list head, and + * desc_next[id] is the next element. + * With IN_ORDER free_head is the next available buffer index. + */ uint16_t free_head; + /* + * Last used element of the processing batch of used descriptors if + * IN_ORDER. + * If SVQ is not processing a batch of descriptors id is set to UINT_MAX. + */ + vring_used_elem_t batch_last; + + /* Last used id if IN_ORDER and split vq */ + uint16_t last_used; + /* Last seen used idx */ uint16_t shadow_used_idx; diff --git a/hw/virtio/vhost-user-base.c b/hw/virtio/vhost-user-base.c index 01ab9ca56b..39b5e637fc 100644 --- a/hw/virtio/vhost-user-base.c +++ b/hw/virtio/vhost-user-base.c @@ -16,6 +16,7 @@ #include "hw/virtio/virtio-bus.h" #include "hw/virtio/vhost-user-base.h" #include "qemu/error-report.h" +#include "migration/blocker.h" static void vub_start(VirtIODevice *vdev) { @@ -71,7 +72,7 @@ static int vub_stop(VirtIODevice *vdev) VHostUserBase *vub = VHOST_USER_BASE(vdev); BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int ret; + int ret, err; if (!k->set_guest_notifiers) { return 0; @@ -79,9 +80,10 @@ static int vub_stop(VirtIODevice *vdev) ret = vhost_dev_stop(&vub->vhost_dev, vdev, true); - if (k->set_guest_notifiers(qbus->parent, vub->vhost_dev.nvqs, false) < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); - return -1; + err = k->set_guest_notifiers(qbus->parent, vub->vhost_dev.nvqs, false); + if (err < 0) { + error_report("vhost guest notifier cleanup failed: %d", err); + return err; } vhost_dev_disable_notifiers(&vub->vhost_dev, vdev); @@ -118,9 +120,13 @@ static uint64_t vub_get_features(VirtIODevice *vdev, uint64_t requested_features, Error **errp) { VHostUserBase *vub = VHOST_USER_BASE(vdev); + uint64_t backend_features = vhost_dev_features(&vub->vhost_dev); + /* This should be set when the vhost connection initialises */ - g_assert(vub->vhost_dev.features); - return vub->vhost_dev.features & ~(1ULL << VHOST_USER_F_PROTOCOL_FEATURES); + g_assert(backend_features); + virtio_clear_feature(&backend_features, VHOST_USER_F_PROTOCOL_FEATURES); + + return backend_features; } /* @@ -276,7 +282,8 @@ static void vub_device_realize(DeviceState *dev, Error **errp) { VirtIODevice *vdev = VIRTIO_DEVICE(dev); VHostUserBase *vub = VHOST_USER_BASE(dev); - int ret; + uint64_t memory_sizes[VIRTIO_MAX_SHMEM_REGIONS]; + int i, ret, nregions, regions_processed = 0; if (!vub->chardev.chr) { error_setg(errp, "vhost-user-base: missing chardev"); @@ -319,7 +326,7 @@ static void vub_device_realize(DeviceState *dev, Error **errp) /* Allocate queues */ vub->vqs = g_ptr_array_sized_new(vub->num_vqs); - for (int i = 0; i < vub->num_vqs; i++) { + for (i = 0; i < vub->num_vqs; i++) { g_ptr_array_add(vub->vqs, virtio_add_queue(vdev, vub->vq_size, vub_handle_output)); @@ -333,11 +340,49 @@ static void vub_device_realize(DeviceState *dev, Error **errp) VHOST_BACKEND_TYPE_USER, 0, errp); if (ret < 0) { - do_vhost_user_cleanup(vdev, vub); + goto err; + } + + ret = vub->vhost_dev.vhost_ops->vhost_get_shmem_config(&vub->vhost_dev, + &nregions, + memory_sizes, + errp); + + if (ret < 0) { + goto err; + } + + for (i = 0; i < VIRTIO_MAX_SHMEM_REGIONS && regions_processed < nregions; i++) { + if (memory_sizes[i]) { + regions_processed++; + if (vub->vhost_dev.migration_blocker == NULL) { + error_setg(&vub->vhost_dev.migration_blocker, + "Migration disabled: devices with VIRTIO Shared Memory " + "Regions do not support migration yet."); + ret = migrate_add_blocker_normal( + &vub->vhost_dev.migration_blocker, + errp); + + if (ret < 0) { + goto err; + } + } + + if (memory_sizes[i] % qemu_real_host_page_size() != 0) { + error_setg(errp, "Shared memory %d size must be a multiple of " + "the host page size", i); + goto err; + } + + virtio_new_shmem_region(vdev, i, memory_sizes[i]); + } } qemu_chr_fe_set_handlers(&vub->chardev, NULL, NULL, vub_event, NULL, dev, NULL, true); + return; +err: + do_vhost_user_cleanup(vdev, vub); } static void vub_device_unrealize(DeviceState *dev) diff --git a/hw/virtio/vhost-user-fs.c b/hw/virtio/vhost-user-fs.c index ad6fcacf06..209993918a 100644 --- a/hw/virtio/vhost-user-fs.c +++ b/hw/virtio/vhost-user-fs.c @@ -105,7 +105,7 @@ static int vuf_stop(VirtIODevice *vdev) VHostUserFS *fs = VHOST_USER_FS(vdev); BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int ret; + int ret, err; if (!k->set_guest_notifiers) { return 0; @@ -113,9 +113,10 @@ static int vuf_stop(VirtIODevice *vdev) ret = vhost_dev_stop(&fs->vhost_dev, vdev, true); - if (k->set_guest_notifiers(qbus->parent, fs->vhost_dev.nvqs, false) < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); - return -1; + err = k->set_guest_notifiers(qbus->parent, fs->vhost_dev.nvqs, false); + if (err < 0) { + error_report("vhost guest notifier cleanup failed: %d", err); + return err; } vhost_dev_disable_notifiers(&fs->vhost_dev, vdev); diff --git a/hw/virtio/vhost-user-scmi.c b/hw/virtio/vhost-user-scmi.c index f9264c4374..9470f68c1f 100644 --- a/hw/virtio/vhost-user-scmi.c +++ b/hw/virtio/vhost-user-scmi.c @@ -89,7 +89,7 @@ static int vu_scmi_stop(VirtIODevice *vdev) BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); struct vhost_dev *vhost_dev = &scmi->vhost_dev; - int ret; + int ret, err; /* vhost_dev_is_started() check in the callers is not fully reliable. */ if (!scmi->started_vu) { @@ -103,9 +103,10 @@ static int vu_scmi_stop(VirtIODevice *vdev) ret = vhost_dev_stop(vhost_dev, vdev, true); - if (k->set_guest_notifiers(qbus->parent, vhost_dev->nvqs, false) < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); - return -1; + err = k->set_guest_notifiers(qbus->parent, vhost_dev->nvqs, false); + if (err < 0) { + error_report("vhost guest notifier cleanup failed: %d", err); + return err; } vhost_dev_disable_notifiers(vhost_dev, vdev); return ret; diff --git a/hw/virtio/vhost-user-stub.c b/hw/virtio/vhost-user-stub.c new file mode 100644 index 0000000000..45559788dd --- /dev/null +++ b/hw/virtio/vhost-user-stub.c @@ -0,0 +1,6 @@ +#include "qemu/osdep.h" +#include "hw/virtio/vhost-user.h" + +void vhost_user_qmp_status(struct vhost_dev *dev, VirtioStatus *status) +{ +} diff --git a/hw/virtio/vhost-user-test-device-pci.c b/hw/virtio/vhost-user-test-device-pci.c index 7f6d751690..2106557ed8 100644 --- a/hw/virtio/vhost-user-test-device-pci.c +++ b/hw/virtio/vhost-user-test-device-pci.c @@ -12,10 +12,13 @@ #include "hw/virtio/vhost-user-base.h" #include "hw/virtio/virtio-pci.h" +#define VIRTIO_DEVICE_PCI_SHMEM_BAR 4 + struct VHostUserDevicePCI { VirtIOPCIProxy parent_obj; VHostUserBase vub; + MemoryRegion shmembar; }; #define TYPE_VHOST_USER_TEST_DEVICE_PCI "vhost-user-test-device-pci-base" @@ -25,10 +28,45 @@ OBJECT_DECLARE_SIMPLE_TYPE(VHostUserDevicePCI, VHOST_USER_TEST_DEVICE_PCI) static void vhost_user_device_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp) { VHostUserDevicePCI *dev = VHOST_USER_TEST_DEVICE_PCI(vpci_dev); - DeviceState *vdev = DEVICE(&dev->vub); + DeviceState *dev_state = DEVICE(&dev->vub); + VirtIODevice *vdev = VIRTIO_DEVICE(dev_state); + VirtioSharedMemory *shmem; + uint64_t offset = 0, shmem_size = 0; + /* Keep modern 64-bit BARs (2 slots) away from the shared memory BAR. */ + vpci_dev->modern_mem_bar_idx = 2; vpci_dev->nvectors = 1; - qdev_realize(vdev, BUS(&vpci_dev->bus), errp); + if (!qdev_realize(dev_state, BUS(&vpci_dev->bus), errp)) { + return; + } + + QSIMPLEQ_FOREACH(shmem, &vdev->shmem_list, entry) { + if (memory_region_size(&shmem->mr) > UINT64_MAX - shmem_size) { + error_setg(errp, "Total shared memory required overflow"); + return; + } + shmem_size = shmem_size + memory_region_size(&shmem->mr); + } + if (shmem_size) { + if (vpci_dev->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY) { + error_setg(errp, "modern-pio-notify is not supported due to PCI BAR layout limitations"); + return; + } + memory_region_init(&dev->shmembar, OBJECT(vpci_dev), + "vhost-device-pci-shmembar", shmem_size); + QSIMPLEQ_FOREACH(shmem, &vdev->shmem_list, entry) { + memory_region_add_subregion(&dev->shmembar, offset, &shmem->mr); + virtio_pci_add_shm_cap(vpci_dev, VIRTIO_DEVICE_PCI_SHMEM_BAR, + offset, memory_region_size(&shmem->mr), + shmem->shmid); + offset = offset + memory_region_size(&shmem->mr); + } + pci_register_bar(&vpci_dev->pci_dev, VIRTIO_DEVICE_PCI_SHMEM_BAR, + PCI_BASE_ADDRESS_SPACE_MEMORY | + PCI_BASE_ADDRESS_MEM_PREFETCH | + PCI_BASE_ADDRESS_MEM_TYPE_64, + &dev->shmembar); + } } static void vhost_user_device_pci_class_init(ObjectClass *klass, diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c index a8907cca74..d627351f45 100644 --- a/hw/virtio/vhost-user.c +++ b/hw/virtio/vhost-user.c @@ -11,6 +11,7 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "hw/virtio/virtio-dmabuf.h" +#include "hw/virtio/virtio-qmp.h" #include "hw/virtio/vhost.h" #include "hw/virtio/virtio-crypto.h" #include "hw/virtio/vhost-user.h" @@ -99,6 +100,7 @@ typedef enum VhostUserRequest { VHOST_USER_GET_SHARED_OBJECT = 41, VHOST_USER_SET_DEVICE_STATE_FD = 42, VHOST_USER_CHECK_DEVICE_STATE = 43, + VHOST_USER_GET_SHMEM_CONFIG = 44, VHOST_USER_MAX } VhostUserRequest; @@ -110,9 +112,68 @@ typedef enum VhostUserBackendRequest { VHOST_USER_BACKEND_SHARED_OBJECT_ADD = 6, VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE = 7, VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP = 8, + VHOST_USER_BACKEND_SHMEM_MAP = 9, + VHOST_USER_BACKEND_SHMEM_UNMAP = 10, VHOST_USER_BACKEND_MAX } VhostUserBackendRequest; +#define VHOST_USER_CASE(name) \ + case VHOST_USER_##name: \ + return #name; + +static const char *vhost_req_name(VhostUserRequest req) +{ + switch (req) { + VHOST_USER_CASE(NONE) + VHOST_USER_CASE(GET_FEATURES) + VHOST_USER_CASE(SET_FEATURES) + VHOST_USER_CASE(SET_OWNER) + VHOST_USER_CASE(RESET_OWNER) + VHOST_USER_CASE(SET_MEM_TABLE) + VHOST_USER_CASE(SET_LOG_BASE) + VHOST_USER_CASE(SET_LOG_FD) + VHOST_USER_CASE(SET_VRING_NUM) + VHOST_USER_CASE(SET_VRING_ADDR) + VHOST_USER_CASE(SET_VRING_BASE) + VHOST_USER_CASE(GET_VRING_BASE) + VHOST_USER_CASE(SET_VRING_KICK) + VHOST_USER_CASE(SET_VRING_CALL) + VHOST_USER_CASE(SET_VRING_ERR) + VHOST_USER_CASE(GET_PROTOCOL_FEATURES) + VHOST_USER_CASE(SET_PROTOCOL_FEATURES) + VHOST_USER_CASE(GET_QUEUE_NUM) + VHOST_USER_CASE(SET_VRING_ENABLE) + VHOST_USER_CASE(SEND_RARP) + VHOST_USER_CASE(NET_SET_MTU) + VHOST_USER_CASE(SET_BACKEND_REQ_FD) + VHOST_USER_CASE(IOTLB_MSG) + VHOST_USER_CASE(SET_VRING_ENDIAN) + VHOST_USER_CASE(GET_CONFIG) + VHOST_USER_CASE(SET_CONFIG) + VHOST_USER_CASE(CREATE_CRYPTO_SESSION) + VHOST_USER_CASE(CLOSE_CRYPTO_SESSION) + VHOST_USER_CASE(POSTCOPY_ADVISE) + VHOST_USER_CASE(POSTCOPY_LISTEN) + VHOST_USER_CASE(POSTCOPY_END) + VHOST_USER_CASE(GET_INFLIGHT_FD) + VHOST_USER_CASE(SET_INFLIGHT_FD) + VHOST_USER_CASE(GPU_SET_SOCKET) + VHOST_USER_CASE(RESET_DEVICE) + VHOST_USER_CASE(GET_MAX_MEM_SLOTS) + VHOST_USER_CASE(ADD_MEM_REG) + VHOST_USER_CASE(REM_MEM_REG) + VHOST_USER_CASE(SET_STATUS) + VHOST_USER_CASE(GET_STATUS) + VHOST_USER_CASE(GET_SHARED_OBJECT) + VHOST_USER_CASE(SET_DEVICE_STATE_FD) + VHOST_USER_CASE(CHECK_DEVICE_STATE) + default: + return ""; + } +} + +#undef VHOST_USER_CASE + typedef struct VhostUserMemoryRegion { uint64_t guest_phys_addr; uint64_t memory_size; @@ -131,6 +192,12 @@ typedef struct VhostUserMemRegMsg { VhostUserMemoryRegion region; } VhostUserMemRegMsg; +typedef struct VhostUserShMemConfig { + uint32_t nregions; + uint32_t padding; + uint64_t memory_sizes[VIRTIO_MAX_SHMEM_REGIONS]; +} VhostUserShMemConfig; + typedef struct VhostUserLog { uint64_t mmap_size; uint64_t mmap_offset; @@ -187,6 +254,23 @@ typedef struct VhostUserShared { unsigned char uuid[16]; } VhostUserShared; +/* For the flags field of VhostUserMMap */ +#define VHOST_USER_FLAG_MAP_RW (1u << 0) + +typedef struct { + /* VIRTIO Shared Memory Region ID */ + uint8_t shmid; + uint8_t padding[7]; + /* File offset */ + uint64_t fd_offset; + /* Offset within the VIRTIO Shared Memory Region */ + uint64_t shm_offset; + /* Size of the mapping */ + uint64_t len; + /* Flags for the mmap operation, from VHOST_USER_FLAG_MAP_* */ + uint64_t flags; +} VhostUserMMap; + typedef struct { VhostUserRequest request; @@ -219,6 +303,8 @@ typedef union { VhostUserInflight inflight; VhostUserShared object; VhostUserTransferDeviceState transfer_state; + VhostUserMMap mmap; + VhostUserShMemConfig shmem; } VhostUserPayload; typedef struct VhostUserMsg { @@ -238,7 +324,7 @@ struct vhost_user { struct vhost_dev *dev; /* Shared between vhost devs of the same virtio device */ VhostUserState *user; - QIOChannel *backend_ioc; + QIOChannelSocket *backend_sioc; GSource *backend_src; NotifierWithReturn postcopy_notifier; struct PostCopyFD postcopy_fd; @@ -259,6 +345,14 @@ struct vhost_user { /* Our current regions */ int num_shadow_regions; struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS]; + + /** + * @protocol_features: the vhost-user protocol feature set by + * VHOST_USER_SET_PROTOCOL_FEATURES. Protocol features are only + * negotiated if VHOST_USER_F_PROTOCOL_FEATURES has been offered + * by the backend (see @features). + */ + uint64_t protocol_features; }; struct scrub_regions { @@ -267,6 +361,15 @@ struct scrub_regions { int fd_idx; }; +bool vhost_user_has_protocol_feature(struct vhost_dev *dev, uint64_t feature) +{ + struct vhost_user *u = dev->opaque; + + assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); + + return virtio_has_feature(u->protocol_features, feature); +} + static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg) { struct vhost_user *u = dev->opaque; @@ -290,7 +393,8 @@ static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg) return -EPROTO; } - trace_vhost_user_read(msg->hdr.request, msg->hdr.flags); + trace_vhost_user_read(msg->hdr.request, + vhost_req_name(msg->hdr.request), msg->hdr.flags); return 0; } @@ -410,7 +514,8 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg, return ret < 0 ? -saved_errno : -EIO; } - trace_vhost_user_write(msg->hdr.request, msg->hdr.flags); + trace_vhost_user_write(msg->hdr.request, vhost_req_name(msg->hdr.request), + msg->hdr.flags); return 0; } @@ -430,8 +535,8 @@ static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base, { int fds[VHOST_USER_MAX_RAM_SLOTS]; size_t fd_num = 0; - bool shmfd = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_LOG_SHMFD); + bool shmfd = + vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_LOG_SHMFD); int ret; VhostUserMsg msg = { .hdr.request = VHOST_USER_SET_LOG_BASE, @@ -486,12 +591,22 @@ static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset, return mr; } -static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst, +static bool vhost_user_gpa_addresses(struct vhost_dev *dev) +{ + return vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_GPA_ADDRESSES); +} + +static void vhost_user_fill_msg_region(struct vhost_dev *dev, + VhostUserMemoryRegion *dst, struct vhost_memory_region *src, uint64_t mmap_offset) { + bool use_phys = vhost_user_gpa_addresses(dev); + assert(src != NULL && dst != NULL); - dst->userspace_addr = src->userspace_addr; + + dst->userspace_addr = use_phys ? src->guest_phys_addr : src->userspace_addr; dst->memory_size = src->memory_size; dst->guest_phys_addr = src->guest_phys_addr; dst->mmap_offset = mmap_offset; @@ -529,7 +644,7 @@ static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u, error_report("Failed preparing vhost-user memory table msg"); return -ENOBUFS; } - vhost_user_fill_msg_region(®ion_buffer, reg, offset); + vhost_user_fill_msg_region(dev, ®ion_buffer, reg, offset); msg->payload.memory.regions[*fd_num] = region_buffer; fds[(*fd_num)++] = fd; } else if (track_ramblocks) { @@ -675,7 +790,7 @@ static int send_remove_regions(struct vhost_dev *dev, if (fd > 0) { msg->hdr.request = VHOST_USER_REM_MEM_REG; - vhost_user_fill_msg_region(®ion_buffer, shadow_reg, 0); + vhost_user_fill_msg_region(dev, ®ion_buffer, shadow_reg, 0); msg->payload.mem_reg.region = region_buffer; ret = vhost_user_write(dev, msg, NULL, 0); @@ -736,7 +851,7 @@ static int send_add_regions(struct vhost_dev *dev, u->region_rb[reg_idx] = mr->ram_block; } msg->hdr.request = VHOST_USER_ADD_MEM_REG; - vhost_user_fill_msg_region(®ion_buffer, reg, offset); + vhost_user_fill_msg_region(dev, ®ion_buffer, reg, offset); msg->payload.mem_reg.region = region_buffer; ret = vhost_user_write(dev, msg, &fd, 1); @@ -1001,11 +1116,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev, int fds[VHOST_MEMORY_BASELINE_NREGIONS]; size_t fd_num = 0; bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler; - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); + bool reply_supported = + vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK); bool config_mem_slots = - virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS); + vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS); int ret; if (do_postcopy) { @@ -1053,8 +1168,9 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev, static int vhost_user_set_vring_endian(struct vhost_dev *dev, struct vhost_vring_state *ring) { - bool cross_endian = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CROSS_ENDIAN); + bool cross_endian = + vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_CROSS_ENDIAN); VhostUserMsg msg = { .hdr.request = VHOST_USER_SET_VRING_ENDIAN, .hdr.flags = VHOST_USER_VERSION, @@ -1124,8 +1240,9 @@ static int vhost_user_write_sync(struct vhost_dev *dev, VhostUserMsg *msg, int ret; if (wait_for_reply) { - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); + bool reply_supported = + vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_REPLY_ACK); if (reply_supported) { msg->hdr.flags |= VHOST_USER_NEED_REPLY_MASK; } @@ -1229,8 +1346,13 @@ static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable) { int i; - if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) { - return -EINVAL; + if (!vhost_dev_has_feature(dev, VHOST_USER_F_PROTOCOL_FEATURES)) { + /* + * For vhost-user devices, if VHOST_USER_F_PROTOCOL_FEATURES has not + * been negotiated, the rings start directly in the enabled state, + * and can't be disabled. + */ + return 0; } for (i = 0; i < dev->nvqs; ++i) { @@ -1325,8 +1447,8 @@ static int vhost_set_vring_file(struct vhost_dev *dev, int ret; int fds[VHOST_USER_MAX_RAM_SLOTS]; size_t fd_num = 0; - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); + bool reply_supported = + vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK); VhostUserMsg msg = { .hdr.request = request, .hdr.flags = VHOST_USER_VERSION, @@ -1460,17 +1582,17 @@ static int vhost_user_set_features(struct vhost_dev *dev, int ret; /* - * We need to include any extra backend only feature bits that - * might be needed by our device. Currently this includes the - * VHOST_USER_F_PROTOCOL_FEATURES bit for enabling protocol - * features. + * Don't lose VHOST_USER_F_PROTOCOL_FEATURES, which is vhost-user + * specific. */ - ret = vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, - features | dev->backend_features, - log_enabled); + if (vhost_dev_has_feature(dev, VHOST_USER_F_PROTOCOL_FEATURES)) { + features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES; + } - if (virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_STATUS)) { + ret = vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features, + log_enabled); + + if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_STATUS)) { if (!ret) { return vhost_user_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); } @@ -1524,8 +1646,8 @@ static int vhost_user_reset_device(struct vhost_dev *dev) * Historically, reset was not implemented so only reset devices * that are expecting it. */ - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_RESET_DEVICE)) { + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_RESET_DEVICE)) { return -ENOSYS; } @@ -1582,8 +1704,8 @@ static int vhost_user_backend_handle_vring_host_notifier(struct vhost_dev *dev, void *addr; char *name; - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) || + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) || vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) { return -EINVAL; } @@ -1766,13 +1888,203 @@ vhost_user_backend_handle_shared_object_lookup(struct vhost_user *u, return 0; } +/** + * vhost_user_backend_handle_shmem_map() - Handle SHMEM_MAP backend request + * @dev: vhost device + * @ioc: QIOChannel for communication + * @hdr: vhost-user message header + * @payload: message payload containing mapping details + * @fd: file descriptor for the shared memory region + * + * Handles VHOST_USER_BACKEND_SHMEM_MAP requests from the backend. Creates + * a VhostUserShmemObject to manage the shared memory mapping and adds it + * to the appropriate VirtIO shared memory region. The VhostUserShmemObject + * serves as an intermediate parent for the MemoryRegion, ensuring proper + * lifecycle management with reference counting. + * + * Returns: 0 on success, negative errno on failure + */ +static int +vhost_user_backend_handle_shmem_map(struct vhost_dev *dev, + QIOChannel *ioc, + VhostUserHeader *hdr, + VhostUserPayload *payload, + int fd) +{ + VirtioSharedMemory *shmem; + VhostUserMMap *vu_mmap = &payload->mmap; + VirtioSharedMemoryMapping *existing; + Error *local_err = NULL; + int ret = 0; + + if (fd < 0) { + error_report("Bad fd for map"); + ret = -EBADF; + goto send_reply; + } + + if (QSIMPLEQ_EMPTY(&dev->vdev->shmem_list)) { + error_report("Device has no VIRTIO Shared Memory Regions. " + "Requested ID: %d", vu_mmap->shmid); + ret = -EFAULT; + goto send_reply; + } + + shmem = virtio_find_shmem_region(dev->vdev, vu_mmap->shmid); + if (!shmem) { + error_report("VIRTIO Shared Memory Region at " + "ID %d not found or uninitialized", vu_mmap->shmid); + ret = -EFAULT; + goto send_reply; + } + + if ((vu_mmap->shm_offset + vu_mmap->len) < vu_mmap->len || + (vu_mmap->shm_offset + vu_mmap->len) > memory_region_size(&shmem->mr)) { + error_report("Bad offset/len for mmap %" PRIx64 "+%" PRIx64, + vu_mmap->shm_offset, vu_mmap->len); + ret = -EFAULT; + goto send_reply; + } + + QTAILQ_FOREACH(existing, &shmem->mmaps, link) { + if (ranges_overlap(existing->offset, existing->len, + vu_mmap->shm_offset, vu_mmap->len)) { + error_report("VIRTIO Shared Memory mapping overlap"); + ret = -EFAULT; + goto send_reply; + } + } + + memory_region_transaction_begin(); + + /* Create VirtioSharedMemoryMapping object */ + VirtioSharedMemoryMapping *mapping = virtio_shared_memory_mapping_new( + vu_mmap->shmid, fd, vu_mmap->fd_offset, vu_mmap->shm_offset, + vu_mmap->len, vu_mmap->flags & VHOST_USER_FLAG_MAP_RW); + + if (!mapping) { + ret = -EFAULT; + goto send_reply_commit; + } + + /* Add the mapping to the shared memory region */ + if (virtio_add_shmem_map(shmem, mapping) != 0) { + error_report("Failed to add shared memory mapping"); + object_unref(OBJECT(mapping)); + ret = -EFAULT; + goto send_reply_commit; + } + +send_reply_commit: + /* Send reply and commit after transaction started */ + if (hdr->flags & VHOST_USER_NEED_REPLY_MASK) { + payload->u64 = !!ret; + hdr->size = sizeof(payload->u64); + if (!vhost_user_send_resp(ioc, hdr, payload, &local_err)) { + error_report_err(local_err); + memory_region_transaction_commit(); + return -EFAULT; + } + } + memory_region_transaction_commit(); + return 0; + +send_reply: + if (hdr->flags & VHOST_USER_NEED_REPLY_MASK) { + payload->u64 = !!ret; + hdr->size = sizeof(payload->u64); + if (!vhost_user_send_resp(ioc, hdr, payload, &local_err)) { + error_report_err(local_err); + return -EFAULT; + } + } + return 0; +} + +/** + * vhost_user_backend_handle_shmem_unmap() - Handle SHMEM_UNMAP backend request + * @dev: vhost device + * @ioc: QIOChannel for communication + * @hdr: vhost-user message header + * @payload: message payload containing unmapping details + * + * Handles VHOST_USER_BACKEND_SHMEM_UNMAP requests from the backend. Removes + * the specified memory mapping from the VirtIO shared memory region. This + * automatically unreferences the associated VhostUserShmemObject, which may + * trigger its finalization and cleanup (munmap, close fd) if no other + * references exist. + * + * Returns: 0 on success, negative errno on failure + */ +static int +vhost_user_backend_handle_shmem_unmap(struct vhost_dev *dev, + QIOChannel *ioc, + VhostUserHeader *hdr, + VhostUserPayload *payload) +{ + VirtioSharedMemory *shmem = NULL; + VirtioSharedMemoryMapping *mmap = NULL; + VhostUserMMap *vu_mmap = &payload->mmap; + Error *local_err = NULL; + int ret = 0; + + if (QSIMPLEQ_EMPTY(&dev->vdev->shmem_list)) { + error_report("Device has no VIRTIO Shared Memory Regions. " + "Requested ID: %d", vu_mmap->shmid); + ret = -EFAULT; + goto send_reply; + } + + shmem = virtio_find_shmem_region(dev->vdev, vu_mmap->shmid); + if (!shmem) { + error_report("VIRTIO Shared Memory Region at " + "ID %d not found or uninitialized", vu_mmap->shmid); + ret = -EFAULT; + goto send_reply; + } + + if ((vu_mmap->shm_offset + vu_mmap->len) < vu_mmap->len || + (vu_mmap->shm_offset + vu_mmap->len) > memory_region_size(&shmem->mr)) { + error_report("Bad offset/len for unmmap %" PRIx64 "+%" PRIx64, + vu_mmap->shm_offset, vu_mmap->len); + ret = -EFAULT; + goto send_reply; + } + + mmap = virtio_find_shmem_map(shmem, vu_mmap->shm_offset, vu_mmap->len); + if (!mmap) { + error_report("Shared memory mapping not found at offset %" PRIx64 + " with length %" PRIx64, + vu_mmap->shm_offset, vu_mmap->len); + ret = -EFAULT; + goto send_reply; + } + +send_reply: + if (hdr->flags & VHOST_USER_NEED_REPLY_MASK) { + payload->u64 = !!ret; + hdr->size = sizeof(payload->u64); + if (!vhost_user_send_resp(ioc, hdr, payload, &local_err)) { + error_report_err(local_err); + return -EFAULT; + } + } + + if (!ret && shmem && mmap) { + /* Free the MemoryRegion only after reply */ + virtio_del_shmem_map(shmem, vu_mmap->shm_offset, vu_mmap->len); + } + + return 0; +} + static void close_backend_channel(struct vhost_user *u) { g_source_destroy(u->backend_src); g_source_unref(u->backend_src); u->backend_src = NULL; - object_unref(OBJECT(u->backend_ioc)); - u->backend_ioc = NULL; + object_unref(OBJECT(u->backend_sioc)); + u->backend_sioc = NULL; } static gboolean backend_read(QIOChannel *ioc, GIOCondition condition, @@ -1817,7 +2129,7 @@ static gboolean backend_read(QIOChannel *ioc, GIOCondition condition, switch (hdr.request) { case VHOST_USER_BACKEND_IOTLB_MSG: - ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb); + ret = vhost_handle_iotlb_msg(dev, &payload.iotlb); break; case VHOST_USER_BACKEND_CONFIG_CHANGE_MSG: ret = vhost_user_backend_handle_config_change(dev); @@ -1839,6 +2151,21 @@ static gboolean backend_read(QIOChannel *ioc, GIOCondition condition, ret = vhost_user_backend_handle_shared_object_lookup(dev->opaque, &payload.object); break; + case VHOST_USER_BACKEND_SHMEM_MAP: + /* Handler manages its own response, check error and close connection */ + reply_ack = false; + if (vhost_user_backend_handle_shmem_map(dev, ioc, &hdr, &payload, + fd ? fd[0] : -1) < 0) { + goto err; + } + break; + case VHOST_USER_BACKEND_SHMEM_UNMAP: + /* Handler manages its own response, check error and close connection */ + reply_ack = false; + if (vhost_user_backend_handle_shmem_unmap(dev, ioc, &hdr, &payload) < 0) { + goto err; + } + break; default: error_report("Received unexpected msg type: %d.", hdr.request); ret = -EINVAL; @@ -1881,13 +2208,12 @@ static int vhost_setup_backend_channel(struct vhost_dev *dev) }; struct vhost_user *u = dev->opaque; int sv[2], ret = 0; - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); + bool reply_supported = + vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK); Error *local_err = NULL; - QIOChannel *ioc; - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_BACKEND_REQ)) { + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ)) { return 0; } @@ -1897,15 +2223,15 @@ static int vhost_setup_backend_channel(struct vhost_dev *dev) return -saved_errno; } - ioc = QIO_CHANNEL(qio_channel_socket_new_fd(sv[0], &local_err)); - if (!ioc) { + u->backend_sioc = qio_channel_socket_new_fd(sv[0], &local_err); + if (!u->backend_sioc) { error_report_err(local_err); return -ECONNREFUSED; } - u->backend_ioc = ioc; - u->backend_src = qio_channel_add_watch_source(u->backend_ioc, - G_IO_IN | G_IO_HUP, - backend_read, dev, NULL, NULL); + u->backend_src = qio_channel_add_watch_source(QIO_CHANNEL(u->backend_sioc), + G_IO_IN | G_IO_HUP, + backend_read, dev, + NULL, NULL); if (reply_supported) { msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; @@ -2135,8 +2461,8 @@ static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier, switch (pnd->reason) { case POSTCOPY_NOTIFY_PROBE: - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_PAGEFAULT)) { + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_PAGEFAULT)) { /* TODO: Get the device name into this error somehow */ error_setg(errp, "vhost-user backend not capable of postcopy"); @@ -2187,8 +2513,6 @@ static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque, (dev->config_ops && dev->config_ops->vhost_dev_config_notifier); uint64_t protocol_features; - dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES; - err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES, &protocol_features); if (err < 0) { @@ -2228,15 +2552,15 @@ static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque, } /* final set of protocol features */ - dev->protocol_features = protocol_features; - err = vhost_user_set_protocol_features(dev, dev->protocol_features); + u->protocol_features = protocol_features; + err = vhost_user_set_protocol_features(dev, u->protocol_features); if (err < 0) { error_setg_errno(errp, EPROTO, "vhost_backend_init failed"); return -EPROTO; } /* query the max queues we support if backend supports Multiple Queue */ - if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) { + if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_MQ)) { err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM, &dev->max_queues); if (err < 0) { @@ -2254,18 +2578,18 @@ static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque, } if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) && - !(virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_BACKEND_REQ) && - virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK))) { + !(vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ) && + vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_REPLY_ACK))) { error_setg(errp, "IOMMU support requires reply-ack and " "backend-req protocol features."); return -EINVAL; } /* get max memory regions if backend supports configurable RAM slots */ - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) { + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) { u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS; } else { err = vhost_user_get_max_memslots(dev, &ram_slots); @@ -2289,8 +2613,8 @@ static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque, } if (dev->migration_blocker == NULL && - !virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_LOG_SHMFD)) { + !vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_LOG_SHMFD)) { error_setg(&dev->migration_blocker, "Migration disabled: vhost-user backend lacks " "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature."); @@ -2327,7 +2651,7 @@ static int vhost_user_backend_cleanup(struct vhost_dev *dev) close(u->postcopy_fd.fd); u->postcopy_fd.handler = NULL; } - if (u->backend_ioc) { + if (u->backend_sioc) { close_backend_channel(u); } g_free(u->region_rb); @@ -2359,8 +2683,8 @@ static bool vhost_user_requires_shm_log(struct vhost_dev *dev) { assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); - return virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_LOG_SHMFD); + return vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_LOG_SHMFD); } static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr) @@ -2375,8 +2699,7 @@ static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr) } /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */ - if (virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_RARP)) { + if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_RARP)) { msg.hdr.request = VHOST_USER_SEND_RARP; msg.hdr.flags = VHOST_USER_VERSION; memcpy((char *)&msg.payload.u64, mac_addr, 6); @@ -2390,11 +2713,11 @@ static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr) static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu) { VhostUserMsg msg; - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); + bool reply_supported = + vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK); int ret; - if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) { + if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_NET_MTU)) { return 0; } @@ -2454,8 +2777,7 @@ static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config, .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len, }; - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CONFIG)) { + if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_CONFIG)) { error_setg(errp, "VHOST_USER_PROTOCOL_F_CONFIG not supported"); return -EINVAL; } @@ -2498,8 +2820,8 @@ static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data, { int ret; uint8_t *p; - bool reply_supported = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_REPLY_ACK); + bool reply_supported = + vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK); VhostUserMsg msg = { .hdr.request = VHOST_USER_SET_CONFIG, @@ -2507,8 +2829,7 @@ static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data, .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size, }; - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CONFIG)) { + if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_CONFIG)) { return -ENOTSUP; } @@ -2543,8 +2864,9 @@ static int vhost_user_crypto_create_session(struct vhost_dev *dev, uint64_t *session_id) { int ret; - bool crypto_session = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); + bool crypto_session = + vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); CryptoDevBackendSessionInfo *backend_info = session_info; VhostUserMsg msg = { .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION, @@ -2645,8 +2967,9 @@ static int vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id) { int ret; - bool crypto_session = virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); + bool crypto_session = + vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); VhostUserMsg msg = { .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION, .hdr.flags = VHOST_USER_VERSION, @@ -2691,8 +3014,8 @@ static int vhost_user_get_inflight_fd(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.inflight), }; - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { return 0; } @@ -2759,8 +3082,8 @@ static int vhost_user_set_inflight_fd(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.inflight), }; - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { + if (!vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { return 0; } @@ -2859,8 +3182,7 @@ void vhost_user_async_close(DeviceState *d, static int vhost_user_dev_start(struct vhost_dev *dev, bool started) { - if (!virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_STATUS)) { + if (!vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_STATUS)) { return 0; } @@ -2885,16 +3207,15 @@ static void vhost_user_reset_status(struct vhost_dev *dev) return; } - if (virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_STATUS)) { + if (vhost_user_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_STATUS)) { vhost_user_set_status(dev, 0); } } static bool vhost_user_supports_device_state(struct vhost_dev *dev) { - return virtio_has_feature(dev->protocol_features, - VHOST_USER_PROTOCOL_F_DEVICE_STATE); + return vhost_user_has_protocol_feature( + dev, VHOST_USER_PROTOCOL_F_DEVICE_STATE); } static int vhost_user_set_device_state_fd(struct vhost_dev *dev, @@ -3025,12 +3346,62 @@ static int vhost_user_check_device_state(struct vhost_dev *dev, Error **errp) return 0; } +void vhost_user_qmp_status(struct vhost_dev *dev, VirtioStatus *status) +{ + struct vhost_user *u = dev->opaque; + + assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); + + status->vhost_dev->protocol_features = + qmp_decode_protocols(u->protocol_features); +} + +static int vhost_user_get_shmem_config(struct vhost_dev *dev, + int *nregions, + uint64_t *memory_sizes, + Error **errp) +{ + int ret; + VhostUserMsg msg = { + .hdr.request = VHOST_USER_GET_SHMEM_CONFIG, + .hdr.flags = VHOST_USER_VERSION, + }; + + if (!vhost_user_has_protocol_feature(dev, + VHOST_USER_PROTOCOL_F_SHMEM)) { + *nregions = 0; + return 0; + } + + ret = vhost_user_write(dev, &msg, NULL, 0); + if (ret < 0) { + return ret; + } + + ret = vhost_user_read(dev, &msg); + if (ret < 0) { + return ret; + } + + if (msg.payload.shmem.nregions > VIRTIO_MAX_SHMEM_REGIONS) { + error_setg(errp, "Received too many shared memory regions: %d", + msg.payload.shmem.nregions); + return -EINVAL; + } + + *nregions = msg.payload.shmem.nregions; + memcpy(memory_sizes, + &msg.payload.shmem.memory_sizes, + sizeof(uint64_t) * VIRTIO_MAX_SHMEM_REGIONS); + return 0; +} + const VhostOps user_ops = { .backend_type = VHOST_BACKEND_TYPE_USER, - .vhost_backend_init = vhost_user_backend_init, - .vhost_backend_cleanup = vhost_user_backend_cleanup, - .vhost_backend_memslots_limit = vhost_user_memslots_limit, - .vhost_backend_no_private_memslots = vhost_user_no_private_memslots, + .vhost_init = vhost_user_backend_init, + .vhost_cleanup = vhost_user_backend_cleanup, + .vhost_memslots_limit = vhost_user_memslots_limit, + .vhost_no_private_memslots = vhost_user_no_private_memslots, .vhost_set_log_base = vhost_user_set_log_base, .vhost_set_mem_table = vhost_user_set_mem_table, .vhost_set_vring_addr = vhost_user_set_vring_addr, @@ -3063,4 +3434,7 @@ const VhostOps user_ops = { .vhost_supports_device_state = vhost_user_supports_device_state, .vhost_set_device_state_fd = vhost_user_set_device_state_fd, .vhost_check_device_state = vhost_user_check_device_state, + .vhost_phys_vring_addr = vhost_user_gpa_addresses, + .vhost_phys_iotlb_msg = vhost_user_gpa_addresses, + .vhost_get_shmem_config = vhost_user_get_shmem_config, }; diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index 9c7634e243..9e1aa4860a 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -1571,16 +1571,9 @@ static int vhost_vdpa_set_owner(struct vhost_dev *dev) return 0; } -static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev, - struct vhost_vring_addr *addr, struct vhost_virtqueue *vq) +static bool vhost_vdpa_phys_vring_addr(struct vhost_dev *dev) { - assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); - addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; - addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; - addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys; - trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr, - addr->avail_user_addr, addr->used_user_addr); - return 0; + return true; } static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) @@ -1590,8 +1583,8 @@ static bool vhost_vdpa_force_iommu(struct vhost_dev *dev) const VhostOps vdpa_ops = { .backend_type = VHOST_BACKEND_TYPE_VDPA, - .vhost_backend_init = vhost_vdpa_init, - .vhost_backend_cleanup = vhost_vdpa_cleanup, + .vhost_init = vhost_vdpa_init, + .vhost_cleanup = vhost_vdpa_cleanup, .vhost_set_log_base = vhost_vdpa_set_log_base, .vhost_set_vring_addr = vhost_vdpa_set_vring_addr, .vhost_set_vring_num = vhost_vdpa_set_vring_num, @@ -1602,7 +1595,7 @@ const VhostOps vdpa_ops = { .vhost_get_features = vhost_vdpa_get_features, .vhost_set_owner = vhost_vdpa_set_owner, .vhost_set_vring_endian = NULL, - .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit, + .vhost_memslots_limit = vhost_vdpa_memslots_limit, .vhost_set_mem_table = vhost_vdpa_set_mem_table, .vhost_set_features = vhost_vdpa_set_features, .vhost_reset_device = vhost_vdpa_reset_device, @@ -1617,7 +1610,7 @@ const VhostOps vdpa_ops = { .vhost_send_device_iotlb_msg = NULL, .vhost_dev_start = vhost_vdpa_dev_start, .vhost_get_device_id = vhost_vdpa_get_device_id, - .vhost_vq_get_addr = vhost_vdpa_vq_get_addr, + .vhost_phys_vring_addr = vhost_vdpa_phys_vring_addr, .vhost_force_iommu = vhost_vdpa_force_iommu, .vhost_set_config_call = vhost_vdpa_set_config_call, .vhost_reset_status = vhost_vdpa_reset_status, diff --git a/hw/virtio/vhost-vsock-common.c b/hw/virtio/vhost-vsock-common.c index b33def900a..b79f4c9ce6 100644 --- a/hw/virtio/vhost-vsock-common.c +++ b/hw/virtio/vhost-vsock-common.c @@ -100,7 +100,7 @@ int vhost_vsock_common_stop(VirtIODevice *vdev) VHostVSockCommon *vvc = VHOST_VSOCK_COMMON(vdev); BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); - int ret; + int ret, err; if (!k->set_guest_notifiers) { return 0; @@ -108,9 +108,10 @@ int vhost_vsock_common_stop(VirtIODevice *vdev) ret = vhost_dev_stop(&vvc->vhost_dev, vdev, true); - if (k->set_guest_notifiers(qbus->parent, vvc->vhost_dev.nvqs, false) < 0) { - error_report("vhost guest notifier cleanup failed: %d", ret); - return -1; + err = k->set_guest_notifiers(qbus->parent, vvc->vhost_dev.nvqs, false); + if (err < 0) { + error_report("vhost guest notifier cleanup failed: %d", err); + return err; } vhost_dev_disable_notifiers(&vvc->vhost_dev, vdev); diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index b9dc4ed13b..af41841b52 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -58,7 +58,7 @@ unsigned int vhost_get_max_memslots(void) struct vhost_dev *hdev; QLIST_FOREACH(hdev, &vhost_devices, entry) { - max = MIN(max, hdev->vhost_ops->vhost_backend_memslots_limit(hdev)); + max = MIN(max, hdev->vhost_ops->vhost_memslots_limit(hdev)); } return max; } @@ -69,7 +69,7 @@ unsigned int vhost_get_free_memslots(void) struct vhost_dev *hdev; QLIST_FOREACH(hdev, &vhost_devices, entry) { - unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev); + unsigned int r = hdev->vhost_ops->vhost_memslots_limit(hdev); unsigned int cur_free = r - hdev->mem->nregions; if (unlikely(r < hdev->mem->nregions)) { @@ -454,24 +454,92 @@ static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size) } static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr, - hwaddr *plen, bool is_write) + hwaddr len, bool is_write) { - if (!vhost_dev_has_iommu(dev)) { - return address_space_map(dev->vdev->dma_as, addr, plen, is_write, - MEMTXATTRS_UNSPECIFIED); - } else { - return (void *)(uintptr_t)addr; + hwaddr mapped_len = len; + void *res = address_space_map(dev->vdev->dma_as, addr, &mapped_len, + is_write, MEMTXATTRS_UNSPECIFIED); + if (!res) { + return NULL; } + if (len != mapped_len) { + address_space_unmap(dev->vdev->dma_as, res, mapped_len, 0, 0); + return NULL; + } + return res; } -static void vhost_memory_unmap(struct vhost_dev *dev, void *buffer, +static void vhost_memory_unmap(struct vhost_dev *dev, void **buffer, hwaddr len, int is_write, hwaddr access_len) { - if (!vhost_dev_has_iommu(dev)) { - address_space_unmap(dev->vdev->dma_as, buffer, len, is_write, - access_len); + if (!*buffer) { + return; } + + address_space_unmap(dev->vdev->dma_as, *buffer, len, is_write, + access_len); + *buffer = NULL; +} + +static void vhost_vrings_unmap(struct vhost_dev *dev, + struct vhost_virtqueue *vq, bool touched) +{ + if (vhost_dev_has_iommu(dev)) { + return; + } + + vhost_memory_unmap(dev, &vq->used_user, vq->used_size, touched, + touched ? vq->used_size : 0); + vhost_memory_unmap(dev, &vq->avail_user, vq->avail_size, 0, + touched ? vq->avail_size : 0); + vhost_memory_unmap(dev, &vq->desc_user, vq->desc_size, 0, + touched ? vq->desc_size : 0); +} + +static int vhost_vrings_map(struct vhost_dev *dev, + struct VirtIODevice *vdev, + struct vhost_virtqueue *vq, + unsigned idx) +{ + vq->desc_size = virtio_queue_get_desc_size(vdev, idx); + vq->desc_phys = virtio_queue_get_desc_addr(vdev, idx); + vq->desc_user = NULL; + vq->avail_size = virtio_queue_get_avail_size(vdev, idx); + vq->avail_phys = virtio_queue_get_avail_addr(vdev, idx); + vq->avail_user = NULL; + vq->used_size = virtio_queue_get_used_size(vdev, idx); + vq->used_phys = virtio_queue_get_used_addr(vdev, idx); + vq->used_user = NULL; + + if (vq->desc_phys == 0) { + /* Queue might not be ready for start */ + return 0; + } + + if (vhost_dev_has_iommu(dev)) { + return 1; + } + + vq->desc_user = vhost_memory_map(dev, vq->desc_phys, vq->desc_size, false); + if (!vq->desc_user) { + goto fail; + } + vq->avail_user = vhost_memory_map(dev, vq->avail_phys, vq->avail_size, + false); + if (!vq->avail_user) { + goto fail; + } + vq->used_user = vhost_memory_map(dev, vq->used_phys, vq->used_size, true); + if (!vq->used_user) { + goto fail; + } + + return 1; + +fail: + vhost_vrings_unmap(dev, vq, false); + return -ENOMEM; } static int vhost_verify_ring_part_mapping(void *ring_hva, @@ -527,7 +595,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev, j = 0; r = vhost_verify_ring_part_mapping( - vq->desc, vq->desc_phys, vq->desc_size, + vq->desc_user, vq->desc_phys, vq->desc_size, reg_hva, reg_gpa, reg_size); if (r) { break; @@ -535,7 +603,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev, j++; r = vhost_verify_ring_part_mapping( - vq->avail, vq->avail_phys, vq->avail_size, + vq->avail_user, vq->avail_phys, vq->avail_size, reg_hva, reg_gpa, reg_size); if (r) { break; @@ -543,7 +611,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev, j++; r = vhost_verify_ring_part_mapping( - vq->used, vq->used_phys, vq->used_size, + vq->used_user, vq->used_phys, vq->used_size, reg_hva, reg_gpa, reg_size); if (r) { break; @@ -599,8 +667,8 @@ static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section) */ if ((memory_region_get_fd(section->mr) < 0 || !qemu_ram_is_shared(section->mr->ram_block)) && - dev->vhost_ops->vhost_backend_no_private_memslots && - dev->vhost_ops->vhost_backend_no_private_memslots(dev)) { + dev->vhost_ops->vhost_no_private_memslots && + dev->vhost_ops->vhost_no_private_memslots(dev)) { trace_vhost_reject_section(mr->name, 2); return false; } @@ -851,14 +919,96 @@ static void vhost_region_addnop(MemoryListener *listener, vhost_region_add_section(dev, section); } +static int vhost_update_device_iotlb(struct vhost_dev *dev, + uint64_t iova, + uint64_t paddr, + uint64_t uaddr, + uint64_t len, + IOMMUAccessFlags perm) +{ + bool phys = dev->vhost_ops->vhost_phys_iotlb_msg && + dev->vhost_ops->vhost_phys_iotlb_msg(dev); + struct vhost_iotlb_msg imsg; + + imsg.iova = iova; + imsg.uaddr = phys ? paddr : uaddr; + imsg.size = len; + imsg.type = VHOST_IOTLB_UPDATE; + + switch (perm) { + case IOMMU_RO: + imsg.perm = VHOST_ACCESS_RO; + break; + case IOMMU_WO: + imsg.perm = VHOST_ACCESS_WO; + break; + case IOMMU_RW: + imsg.perm = VHOST_ACCESS_RW; + break; + default: + return -EINVAL; + } + + if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg) { + return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg); + } + + return -ENODEV; +} + +static int vhost_invalidate_device_iotlb(struct vhost_dev *dev, + uint64_t iova, uint64_t len) +{ + struct vhost_iotlb_msg imsg; + + imsg.iova = iova; + imsg.size = len; + imsg.type = VHOST_IOTLB_INVALIDATE; + + if (dev->vhost_ops && dev->vhost_ops->vhost_send_device_iotlb_msg) { + return dev->vhost_ops->vhost_send_device_iotlb_msg(dev, &imsg); + } + + return -ENODEV; +} + +int vhost_handle_iotlb_msg(struct vhost_dev *dev, struct vhost_iotlb_msg *imsg) +{ + int ret = 0; + + if (unlikely(!dev->vdev)) { + error_report("Unexpected IOTLB message when virtio device is stopped"); + return -EINVAL; + } + + switch (imsg->type) { + case VHOST_IOTLB_MISS: + ret = vhost_device_iotlb_miss(dev, imsg->iova, + imsg->perm != VHOST_ACCESS_RO); + break; + case VHOST_IOTLB_ACCESS_FAIL: + /* FIXME: report device iotlb error */ + error_report("Access failure IOTLB message type not supported"); + ret = -ENOTSUP; + break; + case VHOST_IOTLB_UPDATE: + case VHOST_IOTLB_INVALIDATE: + default: + error_report("Unexpected IOTLB message type"); + ret = -EINVAL; + break; + } + + return ret; +} + static void vhost_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) { struct vhost_iommu *iommu = container_of(n, struct vhost_iommu, n); struct vhost_dev *hdev = iommu->hdev; hwaddr iova = iotlb->iova + iommu->iommu_offset; - if (vhost_backend_invalidate_device_iotlb(hdev, iova, - iotlb->addr_mask + 1)) { + if (vhost_invalidate_device_iotlb(hdev, iova, iotlb->addr_mask + 1)) { error_report("Fail to invalidate device iotlb"); } } @@ -950,20 +1100,20 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev, struct vhost_virtqueue *vq, unsigned idx, bool enable_log) { + bool phys = dev->vhost_ops->vhost_phys_vring_addr && + dev->vhost_ops->vhost_phys_vring_addr(dev); struct vhost_vring_addr addr; int r; memset(&addr, 0, sizeof(struct vhost_vring_addr)); - if (dev->vhost_ops->vhost_vq_get_addr) { - r = dev->vhost_ops->vhost_vq_get_addr(dev, &addr, vq); - if (r < 0) { - VHOST_OPS_DEBUG(r, "vhost_vq_get_addr failed"); - return r; - } + if (phys || vhost_dev_has_iommu(dev)) { + addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys; + addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys; + addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys; } else { - addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc; - addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail; - addr.used_user_addr = (uint64_t)(unsigned long)vq->used; + addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_user; + addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_user; + addr.used_user_addr = (uint64_t)(unsigned long)vq->used_user; } addr.index = idx; addr.log_guest_addr = vq->used_phys; @@ -1239,8 +1389,8 @@ int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write) len = MIN(iotlb.addr_mask + 1, len); iova = iova & ~iotlb.addr_mask; - ret = vhost_backend_update_device_iotlb(dev, iova, uaddr, - len, iotlb.perm); + ret = vhost_update_device_iotlb(dev, iova, iotlb.translated_addr, uaddr, + len, iotlb.perm); if (ret) { trace_vhost_iotlb_miss(dev, 4); error_report("Fail to update device iotlb"); @@ -1262,7 +1412,6 @@ int vhost_virtqueue_start(struct vhost_dev *dev, BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusState *vbus = VIRTIO_BUS(qbus); VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus); - hwaddr l; int r; int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx); struct vhost_vring_file file = { @@ -1273,75 +1422,45 @@ int vhost_virtqueue_start(struct vhost_dev *dev, }; struct VirtQueue *vvq = virtio_get_queue(vdev, idx); - vq->desc_size = virtio_queue_get_desc_size(vdev, idx); - vq->desc_phys = virtio_queue_get_desc_addr(vdev, idx); - vq->desc = NULL; - vq->avail_size = virtio_queue_get_avail_size(vdev, idx); - vq->avail_phys = virtio_queue_get_avail_addr(vdev, idx); - vq->avail = NULL; - vq->used_size = virtio_queue_get_used_size(vdev, idx); - vq->used_phys = virtio_queue_get_used_addr(vdev, idx); - vq->used = NULL; + trace_vhost_virtqueue_start_in(dev, vdev->name, idx); - if (vq->desc_phys == 0) { - /* Queue might not be ready for start */ - return 0; + r = vhost_vrings_map(dev, vdev, vq, idx); + if (r <= 0) { + return r; } vq->num = state.num = virtio_queue_get_num(vdev, idx); r = dev->vhost_ops->vhost_set_vring_num(dev, &state); if (r) { VHOST_OPS_DEBUG(r, "vhost_set_vring_num failed"); - return r; + goto fail; } state.num = virtio_queue_get_last_avail_idx(vdev, idx); r = dev->vhost_ops->vhost_set_vring_base(dev, &state); if (r) { VHOST_OPS_DEBUG(r, "vhost_set_vring_base failed"); - return r; + goto fail; } if (vhost_needs_vring_endian(vdev)) { - r = vhost_virtqueue_set_vring_endian_legacy(dev, - virtio_vdev_is_big_endian(vdev), - vhost_vq_index); + r = vhost_virtqueue_set_vring_endian_legacy( + dev, virtio_vdev_is_big_endian(vdev), vhost_vq_index); if (r) { - return r; + goto fail; } } - l = vq->desc_size; - vq->desc = vhost_memory_map(dev, vq->desc_phys, &l, false); - if (!vq->desc || l != vq->desc_size) { - r = -ENOMEM; - goto fail_alloc_desc; - } - - l = vq->avail_size; - vq->avail = vhost_memory_map(dev, vq->avail_phys, &l, false); - if (!vq->avail || l != vq->avail_size) { - r = -ENOMEM; - goto fail_alloc_avail; - } - - l = vq->used_size; - vq->used = vhost_memory_map(dev, vq->used_phys, &l, true); - if (!vq->used || l != vq->used_size) { - r = -ENOMEM; - goto fail_alloc_used; - } - r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled); if (r < 0) { - goto fail_alloc; + goto fail; } file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq)); r = dev->vhost_ops->vhost_set_vring_kick(dev, &file); if (r) { VHOST_OPS_DEBUG(r, "vhost_set_vring_kick failed"); - goto fail_kick; + goto fail; } /* Clear and discard previous events if any. */ @@ -1361,24 +1480,16 @@ int vhost_virtqueue_start(struct vhost_dev *dev, file.fd = -1; r = dev->vhost_ops->vhost_set_vring_call(dev, &file); if (r) { - goto fail_vector; + goto fail; } } + trace_vhost_virtqueue_start_out(dev, vdev->name, idx); + return 0; -fail_vector: -fail_kick: -fail_alloc: - vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx), - 0, 0); -fail_alloc_used: - vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx), - 0, 0); -fail_alloc_avail: - vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx), - 0, 0); -fail_alloc_desc: +fail: + vhost_vrings_unmap(dev, vq, false); return r; } @@ -1393,6 +1504,8 @@ static int do_vhost_virtqueue_stop(struct vhost_dev *dev, }; int r = 0; + trace_vhost_virtqueue_stop_in(dev, vdev->name, idx); + if (virtio_queue_get_desc_addr(vdev, idx) == 0) { /* Don't stop the virtqueue which might have not been started */ return 0; @@ -1425,12 +1538,9 @@ static int do_vhost_virtqueue_stop(struct vhost_dev *dev, vhost_vq_index); } - vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx), - 1, virtio_queue_get_used_size(vdev, idx)); - vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx), - 0, virtio_queue_get_avail_size(vdev, idx)); - vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx), - 0, virtio_queue_get_desc_size(vdev, idx)); + vhost_vrings_unmap(dev, vq, true); + + trace_vhost_virtqueue_stop_out(dev, vdev->name, idx); return r; } @@ -1479,7 +1589,8 @@ static void vhost_virtqueue_error_notifier(EventNotifier *n) } static int vhost_virtqueue_init(struct vhost_dev *dev, - struct vhost_virtqueue *vq, int n) + struct vhost_virtqueue *vq, int n, + bool busyloop_timeout) { int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n); struct vhost_vring_file file = { @@ -1516,6 +1627,14 @@ static int vhost_virtqueue_init(struct vhost_dev *dev, vhost_virtqueue_error_notifier); } + if (busyloop_timeout) { + r = vhost_virtqueue_set_busyloop_timeout(dev, n, busyloop_timeout); + if (r < 0) { + VHOST_OPS_DEBUG(r, "Failed to set busyloop timeout"); + goto fail_err; + } + } + return 0; fail_err: @@ -1534,18 +1653,17 @@ static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq) } } -static int vhost_dev_get_features(struct vhost_dev *hdev, - uint64_t *features) +static int vhost_dev_init_features(struct vhost_dev *hdev) { uint64_t features64; int r; if (hdev->vhost_ops->vhost_get_features_ex) { - return hdev->vhost_ops->vhost_get_features_ex(hdev, features); + return hdev->vhost_ops->vhost_get_features_ex(hdev, hdev->_features_ex); } r = hdev->vhost_ops->vhost_get_features(hdev, &features64); - virtio_features_from_u64(features, features64); + virtio_features_from_u64(hdev->_features_ex, features64); return r; } @@ -1553,17 +1671,18 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, VhostBackendType backend_type, uint32_t busyloop_timeout, Error **errp) { - uint64_t features[VIRTIO_FEATURES_NU64S]; unsigned int used, reserved, limit; int i, r, n_initialized_vqs = 0; + trace_vhost_dev_init_in(hdev); + hdev->vdev = NULL; hdev->migration_blocker = NULL; r = vhost_set_backend_type(hdev, backend_type); assert(r >= 0); - r = hdev->vhost_ops->vhost_backend_init(hdev, opaque, errp); + r = hdev->vhost_ops->vhost_init(hdev, opaque, errp); if (r < 0) { goto fail; } @@ -1574,13 +1693,13 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, goto fail; } - r = vhost_dev_get_features(hdev, features); + r = vhost_dev_init_features(hdev); if (r < 0) { - error_setg_errno(errp, -r, "vhost_get_features failed"); + error_setg_errno(errp, -r, "vhost_init_features failed"); goto fail; } - limit = hdev->vhost_ops->vhost_backend_memslots_limit(hdev); + limit = hdev->vhost_ops->vhost_memslots_limit(hdev); if (limit < MEMORY_DEVICES_SAFE_MAX_MEMSLOTS && memory_devices_memslot_auto_decision_active()) { error_setg(errp, "some memory device (like virtio-mem)" @@ -1594,26 +1713,14 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, } for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) { - r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i); + r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i, + busyloop_timeout); if (r < 0) { error_setg_errno(errp, -r, "Failed to initialize virtqueue %d", i); goto fail; } } - if (busyloop_timeout) { - for (i = 0; i < hdev->nvqs; ++i) { - r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, - busyloop_timeout); - if (r < 0) { - error_setg_errno(errp, -r, "Failed to set busyloop timeout"); - goto fail_busyloop; - } - } - } - - virtio_features_copy(hdev->features_ex, features); - hdev->memory_listener = (MemoryListener) { .name = "vhost", .begin = vhost_begin, @@ -1635,7 +1742,7 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, }; if (hdev->migration_blocker == NULL) { - if (!virtio_has_feature_ex(hdev->features_ex, VHOST_F_LOG_ALL)) { + if (!vhost_dev_has_feature_ex(hdev, VHOST_F_LOG_ALL)) { error_setg(&hdev->migration_blocker, "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature."); } else if (vhost_dev_log_is_shared(hdev) && !qemu_memfd_alloc_check()) { @@ -1647,7 +1754,7 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, if (hdev->migration_blocker != NULL) { r = migrate_add_blocker_normal(&hdev->migration_blocker, errp); if (r < 0) { - goto fail_busyloop; + goto fail; } } @@ -1679,17 +1786,13 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, " than current number of used (%d) and reserved (%d)" " memory slots for memory devices.", limit, used, reserved); r = -EINVAL; - goto fail_busyloop; + goto fail; } + trace_vhost_dev_init_out(hdev); + return 0; -fail_busyloop: - if (busyloop_timeout) { - while (--i >= 0) { - vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0); - } - } fail: hdev->nvqs = n_initialized_vqs; vhost_dev_cleanup(hdev); @@ -1714,7 +1817,7 @@ void vhost_dev_cleanup(struct vhost_dev *hdev) g_free(hdev->mem); g_free(hdev->mem_sections); if (hdev->vhost_ops) { - hdev->vhost_ops->vhost_backend_cleanup(hdev); + hdev->vhost_ops->vhost_cleanup(hdev); } assert(!hdev->log); @@ -1909,7 +2012,7 @@ void vhost_get_features_ex(struct vhost_dev *hdev, const int *bit = feature_bits; while (*bit != VHOST_INVALID_FEATURE_BIT) { - if (!virtio_has_feature_ex(hdev->features_ex, *bit)) { + if (!vhost_dev_has_feature_ex(hdev, *bit)) { virtio_clear_feature_ex(features, *bit); } bit++; @@ -2078,27 +2181,6 @@ int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size, return 0; } -static int vhost_dev_set_vring_enable(struct vhost_dev *hdev, int enable) -{ - if (!hdev->vhost_ops->vhost_set_vring_enable) { - return 0; - } - - /* - * For vhost-user devices, if VHOST_USER_F_PROTOCOL_FEATURES has not - * been negotiated, the rings start directly in the enabled state, and - * .vhost_set_vring_enable callback will fail since - * VHOST_USER_SET_VRING_ENABLE is not supported. - */ - if (hdev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER && - !virtio_has_feature(hdev->backend_features, - VHOST_USER_F_PROTOCOL_FEATURES)) { - return 0; - } - - return hdev->vhost_ops->vhost_set_vring_enable(hdev, enable); -} - /* * Host notifiers must be enabled at this point. * @@ -2113,7 +2195,7 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings) /* should only be called after backend is connected */ assert(hdev->vhost_ops); - trace_vhost_dev_start(hdev, vdev->name, vrings); + trace_vhost_dev_start_in(hdev, vdev->name, vrings); vdev->vhost_started = true; hdev->started = true; @@ -2198,6 +2280,8 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev, bool vrings) } } vhost_start_config_intr(hdev); + + trace_vhost_dev_start_out(hdev, vdev->name); return 0; fail_iotlb: if (vhost_dev_has_iommu(hdev) && @@ -2247,7 +2331,7 @@ static int do_vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, event_notifier_cleanup( &hdev->vqs[VHOST_QUEUE_NUM_CONFIG_INR].masked_config_notifier); - trace_vhost_dev_stop(hdev, vdev->name, vrings); + trace_vhost_dev_stop_in(hdev, vdev->name, vrings); if (hdev->vhost_ops->vhost_dev_start) { hdev->vhost_ops->vhost_dev_start(hdev, false); @@ -2277,6 +2361,8 @@ static int do_vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev, hdev->started = false; vdev->vhost_started = false; hdev->vdev = NULL; + + trace_vhost_dev_stop_out(hdev, vdev->name); return rc; } diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c index cef944e015..9b545acda3 100644 --- a/hw/virtio/virtio-bus.c +++ b/hw/virtio/virtio-bus.c @@ -298,20 +298,18 @@ int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign) __func__, strerror(-r), r); return r; } - r = k->ioeventfd_assign(proxy, notifier, n, true); - if (r < 0) { - error_report("%s: unable to assign ioeventfd: %d", __func__, r); - virtio_bus_cleanup_host_notifier(bus, n); - } - } else { - k->ioeventfd_assign(proxy, notifier, n, false); } - if (r == 0) { - virtio_queue_set_host_notifier_enabled(vq, assign); + r = k->ioeventfd_assign(proxy, notifier, n, assign); + if (r < 0 && assign) { + error_report("%s: unable to assign ioeventfd: %d", __func__, r); + virtio_bus_cleanup_host_notifier(bus, n); + return r; } - return r; + virtio_queue_set_host_notifier_enabled(vq, assign); + + return 0; } void virtio_bus_cleanup_host_notifier(VirtioBusState *bus, int n) diff --git a/hw/virtio/virtio-hmp-cmds.c b/hw/virtio/virtio-hmp-cmds.c index 1daae482d3..a91bb7dbed 100644 --- a/hw/virtio/virtio-hmp-cmds.c +++ b/hw/virtio/virtio-hmp-cmds.c @@ -176,8 +176,6 @@ void hmp_virtio_status(Monitor *mon, const QDict *qdict) hmp_virtio_dump_features(mon, s->vhost_dev->features); monitor_printf(mon, " Acked features:\n"); hmp_virtio_dump_features(mon, s->vhost_dev->acked_features); - monitor_printf(mon, " Backend features:\n"); - hmp_virtio_dump_features(mon, s->vhost_dev->backend_features); monitor_printf(mon, " Protocol features:\n"); hmp_virtio_dump_protocols(mon, s->vhost_dev->protocol_features); } @@ -205,15 +203,12 @@ void hmp_vhost_queue_status(Monitor *mon, const QDict *qdict) monitor_printf(mon, " call: %"PRId64"\n", s->call); monitor_printf(mon, " VRing:\n"); monitor_printf(mon, " num: %"PRId64"\n", s->num); - monitor_printf(mon, " desc: 0x%016"PRIx64"\n", s->desc); monitor_printf(mon, " desc_phys: 0x%016"PRIx64"\n", s->desc_phys); monitor_printf(mon, " desc_size: %"PRId32"\n", s->desc_size); - monitor_printf(mon, " avail: 0x%016"PRIx64"\n", s->avail); monitor_printf(mon, " avail_phys: 0x%016"PRIx64"\n", s->avail_phys); monitor_printf(mon, " avail_size: %"PRId32"\n", s->avail_size); - monitor_printf(mon, " used: 0x%016"PRIx64"\n", s->used); monitor_printf(mon, " used_phys: 0x%016"PRIx64"\n", s->used_phys); monitor_printf(mon, " used_size: %"PRId32"\n", s->used_size); diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c index 742ca3d3e4..58c6d46aab 100644 --- a/hw/virtio/virtio-mmio.c +++ b/hw/virtio/virtio-mmio.c @@ -670,18 +670,11 @@ static int virtio_mmio_set_guest_notifier(DeviceState *d, int n, bool assign, VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d); VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); - VirtQueue *vq = virtio_get_queue(vdev, n); - EventNotifier *notifier = virtio_queue_get_guest_notifier(vq); + int r; - if (assign) { - int r = event_notifier_init(notifier, 0); - if (r < 0) { - return r; - } - virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd); - } else { - virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd); - event_notifier_cleanup(notifier); + r = virtio_set_guest_notifier(vdev, n, assign, with_irqfd); + if (r < 0) { + return r; } if (vdc->guest_notifier_mask && vdev->use_guest_notifier_mask) { @@ -690,30 +683,7 @@ static int virtio_mmio_set_guest_notifier(DeviceState *d, int n, bool assign, return 0; } -static int virtio_mmio_set_config_guest_notifier(DeviceState *d, bool assign, - bool with_irqfd) -{ - VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d); - VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); - VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); - EventNotifier *notifier = virtio_config_get_guest_notifier(vdev); - int r = 0; - if (assign) { - r = event_notifier_init(notifier, 0); - if (r < 0) { - return r; - } - virtio_config_set_guest_notifier_fd_handler(vdev, assign, with_irqfd); - } else { - virtio_config_set_guest_notifier_fd_handler(vdev, assign, with_irqfd); - event_notifier_cleanup(notifier); - } - if (vdc->guest_notifier_mask && vdev->use_guest_notifier_mask) { - vdc->guest_notifier_mask(vdev, VIRTIO_CONFIG_IRQ_IDX, !assign); - } - return r; -} static int virtio_mmio_set_guest_notifiers(DeviceState *d, int nvqs, bool assign) { @@ -735,7 +705,8 @@ static int virtio_mmio_set_guest_notifiers(DeviceState *d, int nvqs, goto assign_error; } } - r = virtio_mmio_set_config_guest_notifier(d, assign, with_irqfd); + r = virtio_mmio_set_guest_notifier(d, VIRTIO_CONFIG_IRQ_IDX, assign, + with_irqfd); if (r < 0) { goto assign_error; } diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index bcab2d18b8..fd9719547f 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -1208,43 +1208,17 @@ static void virtio_pci_vector_poll(PCIDevice *dev, } } -void virtio_pci_set_guest_notifier_fd_handler(VirtIODevice *vdev, VirtQueue *vq, - int n, bool assign, - bool with_irqfd) -{ - if (n == VIRTIO_CONFIG_IRQ_IDX) { - virtio_config_set_guest_notifier_fd_handler(vdev, assign, with_irqfd); - } else { - virtio_queue_set_guest_notifier_fd_handler(vq, assign, with_irqfd); - } -} - static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign, bool with_irqfd) { VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev); - VirtQueue *vq = NULL; - EventNotifier *notifier = NULL; + int r; - if (n == VIRTIO_CONFIG_IRQ_IDX) { - notifier = virtio_config_get_guest_notifier(vdev); - } else { - vq = virtio_get_queue(vdev, n); - notifier = virtio_queue_get_guest_notifier(vq); - } - - if (assign) { - int r = event_notifier_init(notifier, 0); - if (r < 0) { - return r; - } - virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, true, with_irqfd); - } else { - virtio_pci_set_guest_notifier_fd_handler(vdev, vq, n, false, - with_irqfd); - event_notifier_cleanup(notifier); + r = virtio_set_guest_notifier(vdev, n, assign, with_irqfd); + if (r < 0) { + return r; } if (!msix_enabled(&proxy->pci_dev) && diff --git a/hw/virtio/virtio-qmp.c b/hw/virtio/virtio-qmp.c index f9cdca50d9..57cc03828c 100644 --- a/hw/virtio/virtio-qmp.c +++ b/hw/virtio/virtio-qmp.c @@ -123,6 +123,9 @@ static const qmp_virtio_feature_map_t vhost_user_protocol_map[] = { FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_DEVICE_STATE, \ "VHOST_USER_PROTOCOL_F_DEVICE_STATE: Backend device state transfer " "supported"), + FEATURE_ENTRY(VHOST_USER_PROTOCOL_F_SHMEM, \ + "VHOST_USER_PROTOCOL_F_SHMEM: Backend shared memory mapping " + "supported"), { -1, "" } }; @@ -710,8 +713,6 @@ VirtioStatus *qmp_x_query_virtio_status(const char *path, Error **errp) vdev->guest_features_ex); status->host_features = qmp_decode_features(vdev->device_id, vdev->host_features_ex); - status->backend_features = qmp_decode_features(vdev->device_id, - vdev->backend_features_ex); switch (vdev->device_endian) { case VIRTIO_DEVICE_ENDIAN_LITTLE: @@ -749,18 +750,18 @@ VirtioStatus *qmp_x_query_virtio_status(const char *path, Error **errp) status->vhost_dev->nvqs = hdev->nvqs; status->vhost_dev->vq_index = hdev->vq_index; status->vhost_dev->features = - qmp_decode_features(vdev->device_id, hdev->features_ex); + qmp_decode_features(vdev->device_id, vhost_dev_features_ex(hdev)); status->vhost_dev->acked_features = qmp_decode_features(vdev->device_id, hdev->acked_features_ex); - status->vhost_dev->backend_features = - qmp_decode_features(vdev->device_id, hdev->backend_features_ex); - status->vhost_dev->protocol_features = - qmp_decode_protocols(hdev->protocol_features); status->vhost_dev->max_queues = hdev->max_queues; status->vhost_dev->backend_cap = hdev->backend_cap; status->vhost_dev->log_enabled = hdev->log_enabled; status->vhost_dev->log_size = hdev->log_size; + + if (hdev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER) { + vhost_user_qmp_status(hdev, status); + } } return status; @@ -796,9 +797,6 @@ VirtVhostQueueStatus *qmp_x_query_virtio_vhost_queue_status(const char *path, status->name = g_strdup(vdev->name); status->kick = hdev->vqs[queue].kick; status->call = hdev->vqs[queue].call; - status->desc = (uintptr_t)hdev->vqs[queue].desc; - status->avail = (uintptr_t)hdev->vqs[queue].avail; - status->used = (uintptr_t)hdev->vqs[queue].used; status->num = hdev->vqs[queue].num; status->desc_phys = hdev->vqs[queue].desc_phys; status->desc_size = hdev->vqs[queue].desc_size; diff --git a/hw/virtio/virtio-rtc-pci.c b/hw/virtio/virtio-rtc-pci.c new file mode 100644 index 0000000000..fe25f21fbc --- /dev/null +++ b/hw/virtio/virtio-rtc-pci.c @@ -0,0 +1,65 @@ +/* + * Virtio RTC PCI Bindings + * + * Copyright (c) 2026 Kuan-Wei Chiu + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "hw/virtio/virtio-pci.h" +#include "hw/virtio/virtio-rtc.h" +#include "standard-headers/linux/virtio_ids.h" + +typedef struct VirtIORtcPCI { + VirtIOPCIProxy parent_obj; + VirtIORtc vdev; +} VirtIORtcPCI; + +#define TYPE_VIRTIO_RTC_PCI "virtio-rtc-pci-base" +OBJECT_DECLARE_SIMPLE_TYPE(VirtIORtcPCI, VIRTIO_RTC_PCI) + +static void virtio_rtc_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp) +{ + VirtIORtcPCI *dev = VIRTIO_RTC_PCI(vpci_dev); + DeviceState *vdev = DEVICE(&dev->vdev); + + qdev_realize(vdev, BUS(&vpci_dev->bus), errp); +} + +static void virtio_rtc_pci_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass); + PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass); + + set_bit(DEVICE_CATEGORY_MISC, dc->categories); + k->realize = virtio_rtc_pci_realize; + + pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET; + pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_10_BASE + VIRTIO_ID_CLOCK; + pcidev_k->revision = 0x00; + pcidev_k->class_id = PCI_CLASS_SYSTEM_RTC; +} + +static void virtio_rtc_pci_instance_init(Object *obj) +{ + VirtIORtcPCI *dev = VIRTIO_RTC_PCI(obj); + + virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev), + TYPE_VIRTIO_RTC); +} + +static const VirtioPCIDeviceTypeInfo virtio_rtc_pci_info = { + .base_name = TYPE_VIRTIO_RTC_PCI, + .non_transitional_name = "virtio-rtc-pci", + .instance_size = sizeof(VirtIORtcPCI), + .instance_init = virtio_rtc_pci_instance_init, + .class_init = virtio_rtc_pci_class_init, +}; + +static void virtio_rtc_pci_register(void) +{ + virtio_pci_types_register(&virtio_rtc_pci_info); +} + +type_init(virtio_rtc_pci_register); diff --git a/hw/virtio/virtio-rtc.c b/hw/virtio/virtio-rtc.c new file mode 100644 index 0000000000..32de9c1650 --- /dev/null +++ b/hw/virtio/virtio-rtc.c @@ -0,0 +1,190 @@ +/* + * Virtio RTC device core + * + * Copyright (c) 2026 Kuan-Wei Chiu + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qemu/iov.h" +#include "qemu/timer.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-rtc.h" +#include "standard-headers/linux/virtio_ids.h" +#include "standard-headers/linux/virtio_rtc.h" + +static void virtio_rtc_handle_request(VirtIODevice *vdev, VirtQueue *vq) +{ + VirtQueueElement *elem; + struct virtio_rtc_req_head req_head; + size_t written; + + while ((elem = virtqueue_pop(vq, sizeof(VirtQueueElement)))) { + if (elem->out_num < 1 || elem->in_num < 1) { + virtio_error(vdev, "virtio-rtc: request missing in/out buffers"); + virtqueue_detach_element(vq, elem, 0); + g_free(elem); + break; + } + + if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req_head, + sizeof(req_head)) != sizeof(req_head)) { + virtio_error(vdev, "virtio-rtc: request header too short"); + virtqueue_detach_element(vq, elem, 0); + g_free(elem); + break; + } + + written = 0; + + switch (le16_to_cpu(req_head.msg_type)) { + case VIRTIO_RTC_REQ_CFG: { + struct virtio_rtc_resp_cfg resp = {0}; + resp.head.status = VIRTIO_RTC_S_OK; + resp.num_clocks = cpu_to_le16(1); + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp)); + break; + } + case VIRTIO_RTC_REQ_CLOCK_CAP: { + struct virtio_rtc_req_clock_cap req; + struct virtio_rtc_resp_clock_cap resp = {0}; + + if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, + sizeof(req)) != sizeof(req)) { + resp.head.status = VIRTIO_RTC_S_EINVAL; + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp.head)); + break; + } + + if (le16_to_cpu(req.clock_id) != 0) { + resp.head.status = VIRTIO_RTC_S_ENODEV; + } else { + resp.head.status = VIRTIO_RTC_S_OK; + resp.type = VIRTIO_RTC_CLOCK_UTC; + } + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp)); + break; + } + case VIRTIO_RTC_REQ_CROSS_CAP: { + struct virtio_rtc_req_cross_cap req; + struct virtio_rtc_resp_cross_cap resp = {0}; + + if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, + sizeof(req)) != sizeof(req)) { + resp.head.status = VIRTIO_RTC_S_EINVAL; + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp.head)); + break; + } + + if (le16_to_cpu(req.clock_id) != 0) { + resp.head.status = VIRTIO_RTC_S_ENODEV; + } else { + resp.head.status = VIRTIO_RTC_S_OK; + } + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp)); + break; + } + case VIRTIO_RTC_REQ_READ: { + struct virtio_rtc_req_read req; + struct virtio_rtc_resp_read resp = {0}; + + if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, + sizeof(req)) != sizeof(req)) { + resp.head.status = VIRTIO_RTC_S_EINVAL; + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp.head)); + break; + } + + if (le16_to_cpu(req.clock_id) != 0) { + resp.head.status = VIRTIO_RTC_S_ENODEV; + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp.head)); + } else { + resp.head.status = VIRTIO_RTC_S_OK; + resp.clock_reading = + cpu_to_le64(qemu_clock_get_ns(QEMU_CLOCK_HOST)); + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp)); + } + break; + } + default: { + struct virtio_rtc_resp_head resp = {0}; + resp.status = VIRTIO_RTC_S_EOPNOTSUPP; + written = iov_from_buf(elem->in_sg, elem->in_num, 0, &resp, + sizeof(resp)); + break; + } + } + + virtqueue_push(vq, elem, written); + virtio_notify(vdev, vq); + g_free(elem); + } +} + +static void virtio_rtc_device_realize(DeviceState *dev, Error **errp) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(dev); + VirtIORtc *vrtc = VIRTIO_RTC(dev); + + virtio_init(vdev, VIRTIO_ID_CLOCK, 0); + vrtc->vq = virtio_add_queue(vdev, 64, virtio_rtc_handle_request); +} + +static void virtio_rtc_device_unrealize(DeviceState *dev) +{ + VirtIODevice *vdev = VIRTIO_DEVICE(dev); + VirtIORtc *vrtc = VIRTIO_RTC(dev); + + virtio_delete_queue(vrtc->vq); + virtio_cleanup(vdev); +} + +static uint64_t virtio_rtc_get_features(VirtIODevice *vdev, uint64_t f, + Error **errp) +{ + return f; +} + +static const VMStateDescription vmstate_virtio_rtc = { + .name = "virtio-rtc", + .minimum_version_id = 1, + .version_id = 1, + .fields = (const VMStateField[]) { + VMSTATE_VIRTIO_DEVICE, + VMSTATE_END_OF_LIST() + }, +}; + +static void virtio_rtc_class_init(ObjectClass *klass, const void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); + + set_bit(DEVICE_CATEGORY_MISC, dc->categories); + vdc->realize = virtio_rtc_device_realize; + vdc->unrealize = virtio_rtc_device_unrealize; + vdc->get_features = virtio_rtc_get_features; + dc->vmsd = &vmstate_virtio_rtc; +} + +static const TypeInfo virtio_rtc_info = { + .name = TYPE_VIRTIO_RTC, + .parent = TYPE_VIRTIO_DEVICE, + .instance_size = sizeof(VirtIORtc), + .class_init = virtio_rtc_class_init, +}; + +static void virtio_rtc_register_types(void) +{ + type_register_static(&virtio_rtc_info); +} + +type_init(virtio_rtc_register_types) diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 63e2faee99..f4d86a3655 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -986,7 +986,7 @@ static void virtqueue_ordered_fill(VirtQueue *vq, const VirtQueueElement *elem, * We shouldn't need to increase 'i' by more than or equal to * the distance between used_idx and last_avail_idx (max_steps). */ - max_steps = (vq->last_avail_idx - vq->used_idx) % vq->vring.num; + max_steps = MIN(vq->last_avail_idx - vq->used_idx, vq->vring.num); /* Search for element in vq->used_elems */ while (steps < max_steps) { @@ -3119,6 +3119,173 @@ int virtio_save(VirtIODevice *vdev, QEMUFile *f) return ret; } +VirtioSharedMemory *virtio_new_shmem_region(VirtIODevice *vdev, uint8_t shmid, uint64_t size) +{ + VirtioSharedMemory *elem; + g_autofree char *name = NULL; + + elem = g_new0(VirtioSharedMemory, 1); + elem->shmid = shmid; + + /* Initialize embedded MemoryRegion as container for shmem mappings */ + name = g_strdup_printf("virtio-shmem-%d", shmid); + memory_region_init(&elem->mr, OBJECT(vdev), name, size); + QTAILQ_INIT(&elem->mmaps); + QSIMPLEQ_INSERT_TAIL(&vdev->shmem_list, elem, entry); + return elem; +} + +VirtioSharedMemory *virtio_find_shmem_region(VirtIODevice *vdev, uint8_t shmid) +{ + VirtioSharedMemory *shmem, *next; + QSIMPLEQ_FOREACH_SAFE(shmem, &vdev->shmem_list, entry, next) { + if (shmem->shmid == shmid) { + return shmem; + } + } + return NULL; +} + +static void virtio_shared_memory_mapping_instance_init(Object *obj) +{ + VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj); + + mapping->shmid = 0; + mapping->offset = 0; + mapping->len = 0; + mapping->mr = NULL; +} + +static void virtio_shared_memory_mapping_instance_finalize(Object *obj) +{ + VirtioSharedMemoryMapping *mapping = VIRTIO_SHARED_MEMORY_MAPPING(obj); + + /* Clean up MemoryRegion if it exists */ + if (mapping->mr) { + /* Unparent the MemoryRegion to trigger cleanup */ + object_unparent(OBJECT(mapping->mr)); + mapping->mr = NULL; + } +} + +VirtioSharedMemoryMapping *virtio_shared_memory_mapping_new(uint8_t shmid, + int fd, + uint64_t fd_offset, + uint64_t shm_offset, + uint64_t len, + bool allow_write) +{ + VirtioSharedMemoryMapping *mapping; + MemoryRegion *mr; + g_autoptr(GString) mr_name = g_string_new(NULL); + uint32_t ram_flags; + Error *local_err = NULL; + + if (len == 0) { + error_report("Shared memory mapping size cannot be zero"); + return NULL; + } + + fd = dup(fd); + if (fd < 0) { + error_report("Failed to duplicate fd: %s", strerror(errno)); + return NULL; + } + + /* Determine RAM flags */ + ram_flags = RAM_SHARED; + if (!allow_write) { + ram_flags |= RAM_READONLY_FD; + } + + /* Create the VirtioSharedMemoryMapping */ + mapping = VIRTIO_SHARED_MEMORY_MAPPING( + object_new(TYPE_VIRTIO_SHARED_MEMORY_MAPPING)); + + /* Set up object properties */ + mapping->shmid = shmid; + mapping->offset = shm_offset; + mapping->len = len; + + /* Create MemoryRegion as a child of this object */ + mr = g_new0(MemoryRegion, 1); + g_string_printf(mr_name, "virtio-shmem-%d-%" PRIx64, shmid, shm_offset); + + /* Initialize MemoryRegion with file descriptor */ + if (!memory_region_init_ram_from_fd(mr, OBJECT(mapping), mr_name->str, + len, ram_flags, fd, fd_offset, + &local_err)) { + error_report_err(local_err); + g_free(mr); + close(fd); + object_unref(OBJECT(mapping)); + return NULL; + } + + mapping->mr = mr; + return mapping; +} + +int virtio_add_shmem_map(VirtioSharedMemory *shmem, + VirtioSharedMemoryMapping *mapping) +{ + if (!mapping) { + error_report("VirtioSharedMemoryMapping cannot be NULL"); + return -1; + } + if (!mapping->mr) { + error_report("VirtioSharedMemoryMapping has no MemoryRegion"); + return -1; + } + + /* Validate boundaries against the VIRTIO shared memory region */ + if (mapping->offset + mapping->len > memory_region_size(&shmem->mr)) { + error_report("Memory exceeds the shared memory boundaries"); + return -1; + } + + /* Add as subregion to the VIRTIO shared memory */ + memory_region_add_subregion(&shmem->mr, mapping->offset, mapping->mr); + + /* Add to the mapped regions list */ + QTAILQ_INSERT_TAIL(&shmem->mmaps, mapping, link); + + return 0; +} + +VirtioSharedMemoryMapping *virtio_find_shmem_map(VirtioSharedMemory *shmem, + hwaddr offset, uint64_t size) +{ + VirtioSharedMemoryMapping *mapping; + QTAILQ_FOREACH(mapping, &shmem->mmaps, link) { + if (mapping->offset == offset && mapping->len == size) { + return mapping; + } + } + return NULL; +} + +void virtio_del_shmem_map(VirtioSharedMemory *shmem, hwaddr offset, + uint64_t size) +{ + VirtioSharedMemoryMapping *mapping = virtio_find_shmem_map(shmem, offset, size); + if (mapping == NULL) { + return; + } + + /* + * Remove from memory region first + */ + memory_region_del_subregion(&shmem->mr, mapping->mr); + + /* + * Remove from list and unref the mapping which will trigger automatic cleanup + * when the reference count reaches zero. + */ + QTAILQ_REMOVE(&shmem->mmaps, mapping, link); + object_unref(OBJECT(mapping)); +} + /* A wrapper for use as a VMState .put function */ static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, const VMStateField *field, JSONWriter *vmdesc) @@ -3244,6 +3411,7 @@ int virtio_set_features_ex(VirtIODevice *vdev, const uint64_t *features) void virtio_reset(VirtIODevice *vdev) { VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); + VirtioSharedMemory *shmem; uint64_t features[VIRTIO_FEATURES_NU64S]; int i; @@ -3283,6 +3451,15 @@ void virtio_reset(VirtIODevice *vdev) for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { __virtio_queue_reset(vdev, i); } + + /* Mappings are removed to prevent stale fds from remaining open. */ + QSIMPLEQ_FOREACH(shmem, &vdev->shmem_list, entry) { + while (!QTAILQ_EMPTY(&shmem->mmaps)) { + VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps); + virtio_del_shmem_map(shmem, mapping->offset, + memory_region_size(mapping->mr)); + } + } } static void virtio_device_check_notification_compatibility(VirtIODevice *vdev, @@ -3606,6 +3783,7 @@ void virtio_init(VirtIODevice *vdev, uint16_t device_id, size_t config_size) NULL, virtio_vmstate_change, vdev); vdev->device_endian = virtio_default_endian(); vdev->use_guest_notifier_mask = true; + QSIMPLEQ_INIT(&vdev->shmem_list); } /* @@ -3835,37 +4013,37 @@ static void virtio_config_guest_notifier_read(EventNotifier *n) virtio_notify_config(vdev); } } -void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, - bool with_irqfd) -{ - if (assign && !with_irqfd) { - event_notifier_set_handler(&vq->guest_notifier, - virtio_queue_guest_notifier_read); - } else { - event_notifier_set_handler(&vq->guest_notifier, NULL); - } - if (!assign) { - /* Test and clear notifier before closing it, - * in case poll callback didn't have time to run. */ - virtio_queue_guest_notifier_read(&vq->guest_notifier); - } -} -void virtio_config_set_guest_notifier_fd_handler(VirtIODevice *vdev, - bool assign, bool with_irqfd) +int virtio_set_guest_notifier(VirtIODevice *vdev, int n, bool assign, + bool with_irqfd) { - EventNotifier *n; - n = &vdev->config_notifier; - if (assign && !with_irqfd) { - event_notifier_set_handler(n, virtio_config_guest_notifier_read); - } else { - event_notifier_set_handler(n, NULL); + bool is_config = n == VIRTIO_CONFIG_IRQ_IDX; + VirtQueue *vq = is_config ? NULL : virtio_get_queue(vdev, n); + EventNotifier *notifier = is_config ? + virtio_config_get_guest_notifier(vdev) : + virtio_queue_get_guest_notifier(vq); + EventNotifierHandler *read_fn = is_config ? + virtio_config_guest_notifier_read : + virtio_queue_guest_notifier_read; + + if (assign) { + int r = event_notifier_init(notifier, 0); + if (r < 0) { + return r; + } } + + event_notifier_set_handler(notifier, + (assign && !with_irqfd) ? read_fn : NULL); + if (!assign) { /* Test and clear notifier before closing it,*/ /* in case poll callback didn't have time to run. */ - virtio_config_guest_notifier_read(n); + read_fn(notifier); + event_notifier_cleanup(notifier); } + + return 0; } EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) @@ -4117,11 +4295,25 @@ static void virtio_device_free_virtqueues(VirtIODevice *vdev) static void virtio_device_instance_finalize(Object *obj) { VirtIODevice *vdev = VIRTIO_DEVICE(obj); + VirtioSharedMemory *shmem; virtio_device_free_virtqueues(vdev); g_free(vdev->config); g_free(vdev->vector_queues); + while (!QSIMPLEQ_EMPTY(&vdev->shmem_list)) { + shmem = QSIMPLEQ_FIRST(&vdev->shmem_list); + while (!QTAILQ_EMPTY(&shmem->mmaps)) { + VirtioSharedMemoryMapping *mapping = QTAILQ_FIRST(&shmem->mmaps); + virtio_del_shmem_map(shmem, mapping->offset, + memory_region_size(mapping->mr)); + } + + /* Clean up the embedded MemoryRegion */ + object_unparent(OBJECT(&shmem->mr)); + QSIMPLEQ_REMOVE_HEAD(&vdev->shmem_list, entry); + g_free(shmem); + } } static const Property virtio_properties[] = { @@ -4487,9 +4679,18 @@ static const TypeInfo virtio_device_info = { .class_size = sizeof(VirtioDeviceClass), }; +static const TypeInfo virtio_shared_memory_mapping_info = { + .name = TYPE_VIRTIO_SHARED_MEMORY_MAPPING, + .parent = TYPE_OBJECT, + .instance_size = sizeof(VirtioSharedMemoryMapping), + .instance_init = virtio_shared_memory_mapping_instance_init, + .instance_finalize = virtio_shared_memory_mapping_instance_finalize, +}; + static void virtio_register_types(void) { type_register_static(&virtio_device_info); + type_register_static(&virtio_shared_memory_mapping_info); } type_init(virtio_register_types) diff --git a/include/hw/acpi/aml-build.h b/include/hw/acpi/aml-build.h index e70e0643b1..eaff025d26 100644 --- a/include/hw/acpi/aml-build.h +++ b/include/hw/acpi/aml-build.h @@ -343,6 +343,8 @@ Aml *aml_io(AmlIODecode dec, uint16_t min_base, uint16_t max_base, Aml *aml_operation_region(const char *name, AmlRegionSpace rs, Aml *offset, uint32_t len); Aml *aml_irq_no_flags(uint8_t irq); +Aml *aml_irq(uint8_t irq, AmlLevelAndEdge level_and_edge, + AmlActiveHighAndLow high_and_low, AmlShared shared); Aml *aml_named_field(const char *name, unsigned length); Aml *aml_reserved_field(unsigned length); Aml *aml_local(int num); diff --git a/include/hw/acpi/wdat-ich9.h b/include/hw/acpi/wdat-ich9.h new file mode 100644 index 0000000000..60a2a6a112 --- /dev/null +++ b/include/hw/acpi/wdat-ich9.h @@ -0,0 +1,15 @@ +/* + * Copyright Red Hat, Inc. 2026 + * Author(s): Igor Mammedov + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef QEMU_HW_ACPI_WDAT_ICH9_H +#define QEMU_HW_ACPI_WDAT_ICH9_H + +#include "hw/acpi/aml-build.h" + +void build_ich9_wdat(GArray *table_data, BIOSLinker *linker, const char *oem_id, + const char *oem_table_id, uint64_t tco_base); + +#endif /* QEMU_HW_ACPI_WDAT_ICH9_H */ diff --git a/include/hw/acpi/wdat.h b/include/hw/acpi/wdat.h new file mode 100644 index 0000000000..89803eef4b --- /dev/null +++ b/include/hw/acpi/wdat.h @@ -0,0 +1,118 @@ +/* + * Watchdog Action Table (WDAT) definitions + * + * Copyright Red Hat, Inc. 2026 + * Author(s): Igor Mammedov + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef QEMU_HW_ACPI_WDAT_H +#define QEMU_HW_ACPI_WDAT_H + +#include "hw/acpi/acpi-defs.h" + +/* + * Watchdog actions as described in + * "Hardware Watchdog Timers Design Specification" + * for link to spec see https://uefi.org/acpi + * 'Watchdog Action Table (WDAT)' + */ +typedef enum { + /* + * Restarts the watchdog timer's countdown. This action is + * required. + */ + WDAT_ACTION_RESET = 0x1, + /* + * Returns the current countdown value of the watchdog hardware + * (in count intervals). + */ + WDAT_ACTION_QUERY_CURRENT_COUNTDOWN_PERIOD = 0x4, + /* + * Returns the countdown value the watchdog hardware is + * configured to use when reset (in count intervals). + */ + WDAT_ACTION_QUERY_COUNTDOWN_PERIOD = 0x5, + /* + * Sets the countdown value (in count intervals) to be used when + * the watchdog timer is reset. This action is required if + * WDAT_ACTION_RESET does not explicitly write a new + * countdown value to a register during a reset. Otherwise, this + * action is optional. + */ + WDAT_ACTION_SET_COUNTDOWN_PERIOD = 0x6, + /* + * Determines if the watchdog hardware is currently in enabled/ + * running state. The same result must occur when performed from + * both from enabled/stopped state and enabled/running state. If + * the watchdog hardware is disabled, results are indeterminate. + * This action is required. + */ + WDAT_ACTION_QUERY_RUNNING_STATE = 0x8, + /* + * Starts the watchdog, if not already in running state. If the + * watchdog hardware is disabled, results are indeterminate. + * This action is required. + */ + WDAT_ACTION_SET_RUNNING_STATE = 0x9, + /* + * Determines if the watchdog hardware is currently in enabled/ + * stopped state. The same result must occur when performed from + * both the enabled/stopped state and enabled/running state. If + * the watchdog hardware is disabled, results are indeterminate. + * This action is required. + */ + WDAT_ACTION_QUERY_STOPPED_STATE = 0xA, + /* + * Stops the watchdog, if not already in stopped state. If the + * watchdog hardware is disabled, results are indeterminate. + * This action is required. + */ + WDAT_ACTION_SET_STOPPED_STATE = 0xB, + /* + * Determines if the watchdog hardware is configured to perform a + * reboot when the watchdog is fired. + */ + WDAT_ACTION_QUERY_REBOOT = 0x10, + /* + * Configures the watchdog hardware to perform a reboot when it + * is fired. + */ + WDAT_ACTION_SET_REBOOT = 0x11, + /* + * Determines if the watchdog hardware is configured to perform a + * system shutdown when fired. + */ + WDAT_ACTION_QUERY_SHUTDOWN = 0x12, + /* + * Configures the watchdog hardware to perform a system shutdown + * when fired. + */ + WDAT_ACTION_SET_SHUTDOWN = 0x13, + /* + * Determines if the current boot was caused by the watchdog + * firing. The boot status is required to be set if the watchdog + * fired and caused a reboot. It is recommended that the + * Watchdog Status be set if the watchdog fired and caused a + * shutdown. This action is required. + */ + WDAT_ACTION_QUERY_WATCHDOG_STATUS = 0x20, + /* + * Sets the watchdog's boot status to the default value. This + * action is required. + */ + WDAT_ACTION_SET_WATCHDOG_STATUS = 0x21, +} WDATAction; + +#define WDAT_INS_READ_VALUE 0x0 +#define WDAT_INS_READ_COUNTDOWN 0x1 +#define WDAT_INS_WRITE_VALUE 0x2 +#define WDAT_INS_WRITE_COUNTDOWN 0x3 +#define WDAT_INS_PRESERVE_REGISTER 0x80 + +void build_append_wdat_ins(GArray *table_data, + WDATAction action, uint8_t flags, + struct AcpiGenericAddress as, + uint32_t val, uint32_t mask); + +#endif /* QEMU_HW_ACPI_WDAT_H */ diff --git a/include/hw/core/iommu.h b/include/hw/core/iommu.h index cd59a367ce..fd513d6472 100644 --- a/include/hw/core/iommu.h +++ b/include/hw/core/iommu.h @@ -20,9 +20,20 @@ enum viommu_flags { /* vIOMMU needs nesting parent HWPT to create nested HWPT */ VIOMMU_FLAG_WANT_NESTING_PARENT = BIT_ULL(0), + /* + * vIOMMU supports PASID capability, VFIO checks this flag and synthesize + * a PASID capability. + */ VIOMMU_FLAG_PASID_SUPPORTED = BIT_ULL(1), /* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */ VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2), + /* + * vIOMMU requests other sub-system like VFIO to create a HWPT that can be + * used with PASID attachment. VIOMMU_FLAG_PASID_SUPPORTED can't be used + * for this purpose as PASID attachment is needed by VTD IOMMU but not ARM + * SMMU. + */ + VIOMMU_FLAG_WANT_PASID_ATTACH = BIT_ULL(3), }; /* Host IOMMU quirks. Extracted from host IOMMU capabilities */ @@ -30,4 +41,7 @@ enum host_iommu_quirks { HOST_IOMMU_QUIRK_NESTING_PARENT_BYPASS_RO = BIT_ULL(0), }; +/* ABI constant: IOMMU_NO_PASID must always be 0 (keep in sync with kernel) */ +#define IOMMU_NO_PASID 0 + #endif /* HW_IOMMU_H */ diff --git a/include/hw/cxl/cxl.h b/include/hw/cxl/cxl.h index 998f495a98..d8cd8359d2 100644 --- a/include/hw/cxl/cxl.h +++ b/include/hw/cxl/cxl.h @@ -71,4 +71,5 @@ CXLComponentState *cxl_usp_to_cstate(CXLUpstreamPort *usp); typedef struct CXLDownstreamPort CXLDownstreamPort; DECLARE_INSTANCE_CHECKER(CXLDownstreamPort, CXL_DSP, TYPE_CXL_DSP) +void cfmws_update_non_interleaved(bool commit); #endif diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h index 393f312217..ba551fa5f9 100644 --- a/include/hw/cxl/cxl_device.h +++ b/include/hw/cxl/cxl_device.h @@ -685,6 +685,8 @@ typedef struct CXLSetFeatureInfo { size_t data_size; } CXLSetFeatureInfo; +typedef struct CXLFixedWindow CXLFixedWindow; + struct CXLSanitizeInfo; typedef struct CXLAlertConfig { @@ -712,6 +714,8 @@ struct CXLType3Dev { uint64_t sn; /* State */ + MemoryRegion direct_mr[CXL_HDM_DECODER_COUNT]; + CXLFixedWindow *direct_mr_fw[CXL_HDM_DECODER_COUNT]; AddressSpace hostvmem_as; AddressSpace hostpmem_as; CXLComponentState cxl_cstate; diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h index e44ce31841..1842ba5840 100644 --- a/include/hw/i386/intel_iommu.h +++ b/include/hw/i386/intel_iommu.h @@ -154,8 +154,6 @@ struct VTDAddressSpace { * with the guest IOMMU pgtables for a device. */ IOVATree *iova_tree; - - uint32_t fs_hwpt_id; }; struct VTDIOTLBEntry { @@ -314,7 +312,7 @@ struct IntelIOMMUState { bool intr_eime; /* Extended interrupt mode enabled */ OnOffAuto intr_eim; /* Toggle for EIM cabability */ uint8_t aw_bits; /* Host/IOVA address width (in bits) */ - bool pasid; /* Whether to support PASID */ + uint8_t pasid; /* PASID supported in bits, 0 if not */ bool fs1gp; /* First Stage 1-GByte Page Support */ /* Transient Mapping, Reserved(0) since VTD spec revision 3.2 */ diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h index 85a74363b5..d4b6d3ed57 100644 --- a/include/hw/i386/pc.h +++ b/include/hw/i386/pc.h @@ -45,6 +45,7 @@ typedef struct PCMachineState { const char *south_bridge; bool acpi_build_enabled; + bool wdat_enabled; bool smbus_enabled; bool sata_enabled; bool hpet_enabled; diff --git a/include/hw/pci-host/q35.h b/include/hw/pci-host/q35.h index ddafc3f2e3..f31a71010b 100644 --- a/include/hw/pci-host/q35.h +++ b/include/hw/pci-host/q35.h @@ -100,8 +100,8 @@ struct Q35PCIHost { #define MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT 0xb0000000 #define MCH_HOST_BRIDGE_PCIEXBAR_MAX (0x10000000) /* 256M */ #define MCH_HOST_BRIDGE_PCIEXBAR_ADMSK Q35_MASK(64, 35, 28) -#define MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK ((uint64_t)(1 << 26)) -#define MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK ((uint64_t)(1 << 25)) +#define MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK ((uint64_t)(1 << 27)) +#define MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK ((uint64_t)(1 << 26)) #define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_MASK ((uint64_t)(0x3 << 1)) #define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_256M ((uint64_t)(0x0 << 1)) #define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_128M ((uint64_t)(0x1 << 1)) diff --git a/include/hw/vfio/vfio-device.h b/include/hw/vfio/vfio-device.h index 380a55d6e5..8472420d3f 100644 --- a/include/hw/vfio/vfio-device.h +++ b/include/hw/vfio/vfio-device.h @@ -282,6 +282,7 @@ void vfio_device_unprepare(VFIODevice *vbasedev); bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev); bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev); +bool vfio_device_get_viommu_flags_want_pasid_attach(VFIODevice *vbasedev); bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev, uint32_t type, void *caps, uint32_t size); diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h index ff94fa1734..d878d7b733 100644 --- a/include/hw/virtio/vhost-backend.h +++ b/include/hw/virtio/vhost-backend.h @@ -51,10 +51,9 @@ struct vhost_scsi_target; struct vhost_iotlb_msg; struct vhost_virtqueue; -typedef int (*vhost_backend_init)(struct vhost_dev *dev, void *opaque, - Error **errp); -typedef int (*vhost_backend_cleanup)(struct vhost_dev *dev); -typedef int (*vhost_backend_memslots_limit)(struct vhost_dev *dev); +typedef int (*vhost_init)(struct vhost_dev *dev, void *opaque, Error **errp); +typedef int (*vhost_cleanup)(struct vhost_dev *dev); +typedef int (*vhost_memslots_limit)(struct vhost_dev *dev); typedef int (*vhost_net_set_backend_op)(struct vhost_dev *dev, struct vhost_vring_file *file); @@ -131,7 +130,7 @@ typedef int (*vhost_crypto_create_session_op)(struct vhost_dev *dev, typedef int (*vhost_crypto_close_session_op)(struct vhost_dev *dev, uint64_t session_id); -typedef bool (*vhost_backend_no_private_memslots_op)(struct vhost_dev *dev); +typedef bool (*vhost_no_private_memslots_op)(struct vhost_dev *dev); typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev, uint16_t queue_size, @@ -142,9 +141,9 @@ typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev, typedef int (*vhost_dev_start_op)(struct vhost_dev *dev, bool started); -typedef int (*vhost_vq_get_addr_op)(struct vhost_dev *dev, - struct vhost_vring_addr *addr, - struct vhost_virtqueue *vq); +typedef bool (*vhost_phys_vring_addr_op)(struct vhost_dev *dev); + +typedef bool (*vhost_phys_iotlb_msg_op)(struct vhost_dev *dev); typedef int (*vhost_get_device_id_op)(struct vhost_dev *dev, uint32_t *dev_id); @@ -163,13 +162,22 @@ typedef int (*vhost_set_device_state_fd_op)(struct vhost_dev *dev, int *reply_fd, Error **errp); typedef int (*vhost_check_device_state_op)(struct vhost_dev *dev, Error **errp); +/* + * Max regions is VIRTIO_MAX_SHMEM_REGIONS, so that is the maximum + * number of memory_sizes that will be accepted. + */ +typedef int (*vhost_get_shmem_config_op)(struct vhost_dev *dev, + int *nregions, + uint64_t *memory_sizes, + Error **errp); + typedef struct VhostOps { VhostBackendType backend_type; - vhost_backend_init vhost_backend_init; - vhost_backend_cleanup vhost_backend_cleanup; - vhost_backend_memslots_limit vhost_backend_memslots_limit; - vhost_backend_no_private_memslots_op vhost_backend_no_private_memslots; + vhost_init vhost_init; + vhost_cleanup vhost_cleanup; + vhost_memslots_limit vhost_memslots_limit; + vhost_no_private_memslots_op vhost_no_private_memslots; vhost_net_set_backend_op vhost_net_set_backend; vhost_net_set_mtu_op vhost_net_set_mtu; vhost_scsi_set_endpoint_op vhost_scsi_set_endpoint; @@ -212,7 +220,8 @@ typedef struct VhostOps { vhost_get_inflight_fd_op vhost_get_inflight_fd; vhost_set_inflight_fd_op vhost_set_inflight_fd; vhost_dev_start_op vhost_dev_start; - vhost_vq_get_addr_op vhost_vq_get_addr; + vhost_phys_vring_addr_op vhost_phys_vring_addr; + vhost_phys_iotlb_msg_op vhost_phys_iotlb_msg; vhost_get_device_id_op vhost_get_device_id; vhost_force_iommu_op vhost_force_iommu; vhost_set_config_call_op vhost_set_config_call; @@ -220,19 +229,9 @@ typedef struct VhostOps { vhost_supports_device_state_op vhost_supports_device_state; vhost_set_device_state_fd_op vhost_set_device_state_fd; vhost_check_device_state_op vhost_check_device_state; + vhost_get_shmem_config_op vhost_get_shmem_config; } VhostOps; -int vhost_backend_update_device_iotlb(struct vhost_dev *dev, - uint64_t iova, uint64_t uaddr, - uint64_t len, - IOMMUAccessFlags perm); - -int vhost_backend_invalidate_device_iotlb(struct vhost_dev *dev, - uint64_t iova, uint64_t len); - -int vhost_backend_handle_iotlb_msg(struct vhost_dev *dev, - struct vhost_iotlb_msg *imsg); - int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd); int vhost_user_get_shared_object(struct vhost_dev *dev, unsigned char *uuid, diff --git a/include/hw/virtio/vhost-user.h b/include/hw/virtio/vhost-user.h index 53fe996686..06c360af18 100644 --- a/include/hw/virtio/vhost-user.h +++ b/include/hw/virtio/vhost-user.h @@ -10,6 +10,7 @@ #include "chardev/char-fe.h" #include "hw/virtio/virtio.h" +#include "qapi/qapi-types-virtio.h" enum VhostUserProtocolFeature { VHOST_USER_PROTOCOL_F_MQ = 0, @@ -33,6 +34,8 @@ enum VhostUserProtocolFeature { VHOST_USER_PROTOCOL_F_SHARED_OBJECT = 18, VHOST_USER_PROTOCOL_F_DEVICE_STATE = 19, VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT = 20, + VHOST_USER_PROTOCOL_F_GPA_ADDRESSES = 21, + VHOST_USER_PROTOCOL_F_SHMEM = 22, VHOST_USER_PROTOCOL_F_MAX }; @@ -113,4 +116,7 @@ void vhost_user_async_close(DeviceState *d, CharFrontend *chardev, struct vhost_dev *vhost, vu_async_close_fn cb); +void vhost_user_qmp_status(struct vhost_dev *dev, VirtioStatus *status); +bool vhost_user_has_protocol_feature(struct vhost_dev *dev, uint64_t feature); + #endif diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h index 89817bd848..684bafcaad 100644 --- a/include/hw/virtio/vhost.h +++ b/include/hw/virtio/vhost.h @@ -24,9 +24,9 @@ struct vhost_inflight { struct vhost_virtqueue { int kick; int call; - void *desc; - void *avail; - void *used; + void *desc_user; + void *avail_user; + void *used_user; int num; unsigned long long desc_phys; unsigned desc_size; @@ -98,26 +98,12 @@ struct vhost_dev { * offered by a backend which may be a subset of the total * features eventually offered to the guest. * - * @features: available features provided by the backend + * @_features: available features provided by the backend, private, + * direct access only in vhost.h/vhost.c * @acked_features: final negotiated features with front-end driver - * - * @backend_features: this is used in a couple of places to either - * store VHOST_USER_F_PROTOCOL_FEATURES to apply to - * VHOST_USER_SET_FEATURES or VHOST_NET_F_VIRTIO_NET_HDR. Its - * future use should be discouraged and the variable retired as - * its easy to confuse with the VirtIO backend_features. */ - VIRTIO_DECLARE_FEATURES(features); + VIRTIO_DECLARE_FEATURES(_features); VIRTIO_DECLARE_FEATURES(acked_features); - VIRTIO_DECLARE_FEATURES(backend_features); - - /** - * @protocol_features: is the vhost-user only feature set by - * VHOST_USER_SET_PROTOCOL_FEATURES. Protocol features are only - * negotiated if VHOST_USER_F_PROTOCOL_FEATURES has been offered - * by the backend (see @features). - */ - uint64_t protocol_features; uint64_t max_queues; uint64_t backend_cap; @@ -215,6 +201,15 @@ static inline bool vhost_dev_is_started(struct vhost_dev *hdev) return hdev->started; } +static inline int vhost_dev_set_vring_enable(struct vhost_dev *hdev, int enable) +{ + if (!hdev->vhost_ops->vhost_set_vring_enable) { + return 0; + } + + return hdev->vhost_ops->vhost_set_vring_enable(hdev, enable); +} + /** * vhost_dev_start() - start the vhost device * @hdev: common vhost_dev structure @@ -408,6 +403,42 @@ int vhost_dev_set_inflight(struct vhost_dev *dev, int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size, struct vhost_inflight *inflight); bool vhost_dev_has_iommu(struct vhost_dev *dev); +int vhost_handle_iotlb_msg(struct vhost_dev *dev, struct vhost_iotlb_msg *imsg); + + +static inline bool vhost_dev_has_feature(struct vhost_dev *dev, + uint64_t feature) +{ + return virtio_has_feature(dev->_features, feature); +} + +static inline bool vhost_dev_has_feature_ex(struct vhost_dev *dev, + uint64_t feature) +{ + return virtio_has_feature_ex(dev->_features_ex, feature); +} + +static inline uint64_t vhost_dev_features(struct vhost_dev *dev) +{ + return dev->_features; +} + +static inline const uint64_t *vhost_dev_features_ex(struct vhost_dev *dev) +{ + return dev->_features_ex; +} + +static inline void vhost_dev_clear_feature(struct vhost_dev *dev, + uint64_t feature) +{ + virtio_clear_feature(&dev->_features, feature); +} + +static inline void vhost_dev_clear_feature_ex(struct vhost_dev *dev, + uint64_t feature) +{ + virtio_clear_feature_ex(dev->_features_ex, feature); +} #ifdef CONFIG_VHOST int vhost_reset_device(struct vhost_dev *hdev); diff --git a/include/hw/virtio/virtio-pci.h b/include/hw/virtio/virtio-pci.h index 5cb20b73f6..ee50fb0d55 100644 --- a/include/hw/virtio/virtio-pci.h +++ b/include/hw/virtio/virtio-pci.h @@ -250,9 +250,6 @@ void virtio_pci_types_register(const VirtioPCIDeviceTypeInfo *t); * @fixed_queues. */ unsigned virtio_pci_optimal_num_queues(unsigned fixed_queues); -void virtio_pci_set_guest_notifier_fd_handler(VirtIODevice *vdev, VirtQueue *vq, - int n, bool assign, - bool with_irqfd); int virtio_pci_add_shm_cap(VirtIOPCIProxy *proxy, uint8_t bar, uint64_t offset, uint64_t length, uint8_t id); diff --git a/include/hw/virtio/virtio-rtc.h b/include/hw/virtio/virtio-rtc.h new file mode 100644 index 0000000000..51aa35201a --- /dev/null +++ b/include/hw/virtio/virtio-rtc.h @@ -0,0 +1,22 @@ +/* + * Virtio RTC device + * + * Copyright (c) 2026 Kuan-Wei Chiu + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef QEMU_VIRTIO_RTC_H +#define QEMU_VIRTIO_RTC_H + +#include "hw/virtio/virtio.h" +#include "qom/object.h" + +#define TYPE_VIRTIO_RTC "virtio-rtc-device" +OBJECT_DECLARE_SIMPLE_TYPE(VirtIORtc, VIRTIO_RTC) + +struct VirtIORtc { + VirtIODevice parent_obj; + VirtQueue *vq; +}; + +#endif diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index 6344bd7b68..c99cb19d88 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -82,6 +82,8 @@ typedef struct VirtQueueElement #define VIRTIO_NO_VECTOR 0xffff +#define VIRTIO_MAX_SHMEM_REGIONS 256 + /* special index value used internally for config irqs */ #define VIRTIO_CONFIG_IRQ_IDX -1 @@ -99,6 +101,44 @@ enum virtio_device_endian { VIRTIO_DEVICE_ENDIAN_BIG, }; +#define TYPE_VIRTIO_SHARED_MEMORY_MAPPING "virtio-shared-memory-mapping" +OBJECT_DECLARE_SIMPLE_TYPE(VirtioSharedMemoryMapping, VIRTIO_SHARED_MEMORY_MAPPING) + +/** + * VirtioSharedMemoryMapping: + * @parent: Parent QOM object + * @shmid: VIRTIO Shared Memory Region ID + * @offset: Offset within the VIRTIO Shared Memory Region + * @len: Size of the mapping + * @mr: MemoryRegion associated with this shared memory mapping + * @link: List entry for the shared memory region's mapping list + * + * A QOM object that represents an individual file descriptor-based shared + * memory mapping within a VIRTIO Shared Memory Region. It manages the + * MemoryRegion lifecycle and file descriptor cleanup through QOM reference + * counting. When the object is unreferenced and its reference count drops + * to zero, it automatically cleans up the MemoryRegion and closes the file + * descriptor. + */ +struct VirtioSharedMemoryMapping { + Object parent; + + uint8_t shmid; + hwaddr offset; + uint64_t len; + MemoryRegion *mr; + QTAILQ_ENTRY(VirtioSharedMemoryMapping) link; +}; + +struct VirtioSharedMemory { + uint8_t shmid; + MemoryRegion mr; + QTAILQ_HEAD(, VirtioSharedMemoryMapping) mmaps; + QSIMPLEQ_ENTRY(VirtioSharedMemory) entry; +}; + +typedef struct VirtioSharedMemory VirtioSharedMemory; + /** * struct VirtIODevice - common VirtIO structure * @name: name of the device @@ -168,6 +208,8 @@ struct VirtIODevice */ EventNotifier config_notifier; bool device_iotlb_enabled; + /* Shared memory region for mappings. */ + QSIMPLEQ_HEAD(, VirtioSharedMemory) shmem_list; }; struct VirtioDeviceClass { @@ -298,6 +340,100 @@ void virtio_notify(VirtIODevice *vdev, VirtQueue *vq); int virtio_save(VirtIODevice *vdev, QEMUFile *f); +/** + * virtio_new_shmem_region() - Create a new shared memory region + * @vdev: VirtIODevice + * @shmid: Shared memory ID + * @size: Size of the shared memory region + * + * Creates a new VirtioSharedMemory region for the given device and ID. + * The returned VirtioSharedMemory is owned by the VirtIODevice and will + * be automatically freed when the device is destroyed. The caller + * should not free the returned pointer. + * + * Returns: Pointer to the new VirtioSharedMemory region, or NULL on failure + */ +VirtioSharedMemory *virtio_new_shmem_region(VirtIODevice *vdev, uint8_t shmid, uint64_t size); + +/** + * virtio_find_shmem_region() - Find an existing shared memory region + * @vdev: VirtIODevice + * @shmid: Shared memory ID to find + * + * Finds an existing VirtioSharedMemory region by ID. The returned pointer + * is owned by the VirtIODevice and should not be freed by the caller. + * + * Returns: Pointer to the VirtioSharedMemory region, or NULL if not found + */ +VirtioSharedMemory *virtio_find_shmem_region(VirtIODevice *vdev, uint8_t shmid); + +/** + * virtio_shared_memory_mapping_new() - Create a new VirtioSharedMemoryMapping + * @shmid: VIRTIO Shared Memory Region ID + * @fd: File descriptor for the shared memory + * @fd_offset: Offset within the file descriptor + * @shm_offset: Offset within the VIRTIO Shared Memory Region + * @len: Size of the mapping + * @allow_write: Whether to allow write access to the mapping + * + * Creates a new VirtioSharedMemoryMapping that manages a shared memory mapping. + * The object will create a MemoryRegion using memory_region_init_ram_from_fd() + * as a child object. When the object is finalized, it will automatically + * clean up the MemoryRegion and close the file descriptor. + * + * Return: A new VirtioSharedMemoryMapping on success, NULL on error. + */ +VirtioSharedMemoryMapping *virtio_shared_memory_mapping_new(uint8_t shmid, + int fd, + uint64_t fd_offset, + uint64_t shm_offset, + uint64_t len, + bool allow_write); + +/** + * virtio_add_shmem_map() - Add a memory mapping to a shared region + * @shmem: VirtioSharedMemory region + * @mapping: VirtioSharedMemoryMapping to add (transfers ownership) + * + * Adds a memory mapping to the shared memory region. The VirtioSharedMemoryMapping + * ownership is transferred to the shared memory region and will be automatically + * cleaned up through QOM reference counting when virtio_del_shmem_map() is + * called or when the shared memory region is destroyed. + * + * Returns: 0 on success, negative errno on failure + */ +int virtio_add_shmem_map(VirtioSharedMemory *shmem, + VirtioSharedMemoryMapping *mapping); + +/** + * virtio_find_shmem_map() - Find a memory mapping in a shared region + * @shmem: VirtioSharedMemory region + * @offset: Offset within the shared memory region + * @size: Size of the mapping to find + * + * Finds an existing memory mapping that covers the specified range. + * The returned VirtioSharedMemoryMapping is owned by the VirtioSharedMemory + * region and should not be freed by the caller. + * + * Returns: Pointer to the VirtioSharedMemoryMapping, or NULL if not found + */ +VirtioSharedMemoryMapping *virtio_find_shmem_map(VirtioSharedMemory *shmem, + hwaddr offset, uint64_t size); + +/** + * virtio_del_shmem_map() - Remove a memory mapping from a shared region + * @shmem: VirtioSharedMemory region + * @offset: Offset of the mapping to remove + * @size: Size of the mapping to remove + * + * Removes a memory mapping from the shared memory region. This will + * automatically unref the associated VhostUserShmemObject, which may + * trigger its finalization and cleanup if no other references exist. + * The mapping's MemoryRegion will be properly unmapped and cleaned up. + */ +void virtio_del_shmem_map(VirtioSharedMemory *shmem, hwaddr offset, + uint64_t size); + extern const VMStateInfo virtio_vmstate_info; #define VMSTATE_VIRTIO_DEVICE \ @@ -420,8 +556,6 @@ void virtio_queue_update_used_idx(VirtIODevice *vdev, int n); VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n); uint16_t virtio_get_queue_index(VirtQueue *vq); EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq); -void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign, - bool with_irqfd); int virtio_device_start_ioeventfd(VirtIODevice *vdev); int virtio_device_grab_ioeventfd(VirtIODevice *vdev); void virtio_device_release_ioeventfd(VirtIODevice *vdev); @@ -435,8 +569,17 @@ void virtio_queue_aio_detach_host_notifier(VirtQueue *vq, AioContext *ctx); VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector); VirtQueue *virtio_vector_next_queue(VirtQueue *vq); EventNotifier *virtio_config_get_guest_notifier(VirtIODevice *vdev); -void virtio_config_set_guest_notifier_fd_handler(VirtIODevice *vdev, - bool assign, bool with_irqfd); + +/** + * virtio_set_guest_notifier - set/unset queue or config guest notifier + * + * @vdev: the VirtIO device + * @n: queue number, or VIRTIO_CONFIG_IRQ_IDX to set config notifer + * @assign: true to set notifier, false to unset + * @with_irqfd: irqfd enabled + */ +int virtio_set_guest_notifier(VirtIODevice *vdev, int n, bool assign, + bool with_irqfd); static inline void virtio_add_feature(uint64_t *features, unsigned int fbit) { diff --git a/include/system/iommufd.h b/include/system/iommufd.h index 2925d116ac..4257e50f62 100644 --- a/include/system/iommufd.h +++ b/include/system/iommufd.h @@ -138,14 +138,16 @@ struct HostIOMMUDeviceIOMMUFDClass { * * @hiodi: host IOMMU device backed by IOMMUFD backend. * + * @pasid: target pasid of the device to be attached. + * * @hwpt_id: ID of IOMMUFD hardware page table. * * @errp: pass an Error out when attachment fails. * * Returns: true on success, false on failure. */ - bool (*attach_hwpt)(HostIOMMUDeviceIOMMUFD *hiodi, uint32_t hwpt_id, - Error **errp); + bool (*attach_hwpt)(HostIOMMUDeviceIOMMUFD *hiodi, uint32_t pasid, + uint32_t hwpt_id, Error **errp); /** * @detach_hwpt: detach host IOMMU device from IOMMUFD hardware page table. * VFIO and VDPA device can have different implementation. @@ -154,15 +156,19 @@ struct HostIOMMUDeviceIOMMUFDClass { * * @hiodi: host IOMMU device backed by IOMMUFD backend. * - * @errp: pass an Error out when attachment fails. + * @pasid: target pasid of the device to be detached. + * + * @errp: pass an Error out when detachment fails. * * Returns: true on success, false on failure. */ - bool (*detach_hwpt)(HostIOMMUDeviceIOMMUFD *hiodi, Error **errp); + bool (*detach_hwpt)(HostIOMMUDeviceIOMMUFD *hiodi, uint32_t pasid, + Error **errp); }; bool host_iommu_device_iommufd_attach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, - uint32_t hwpt_id, Error **errp); -bool host_iommu_device_iommufd_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, + uint32_t pasid, uint32_t hwpt_id, Error **errp); +bool host_iommu_device_iommufd_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi, + uint32_t pasid, Error **errp); #endif diff --git a/include/system/memory.h b/include/system/memory.h index 1417132f6d..d5f8d8b349 100644 --- a/include/system/memory.h +++ b/include/system/memory.h @@ -150,7 +150,7 @@ struct IOMMUTLBEntry { hwaddr translated_addr; hwaddr addr_mask; /* 0xfff = 4k translation */ IOMMUAccessFlags perm; - uint32_t pasid; + uint32_t pasid; /* PCI pasid */ }; /* @@ -1456,6 +1456,7 @@ bool memory_region_init_ram_from_file(MemoryRegion *mr, const char *path, ram_addr_t offset, Error **errp); +#endif /** * memory_region_init_ram_from_fd: Initialize RAM memory region with a @@ -1485,7 +1486,6 @@ bool memory_region_init_ram_from_fd(MemoryRegion *mr, int fd, ram_addr_t offset, Error **errp); -#endif /** * memory_region_init_ram_ptr: Initialize RAM memory region from a diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c index f1523697e2..c526c2b2dc 100644 --- a/net/vhost-vdpa.c +++ b/net/vhost-vdpa.c @@ -257,15 +257,14 @@ static bool vhost_vdpa_get_vnet_hash_supported_types(NetClientState *nc, { assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); - uint64_t features = s->vhost_vdpa.dev->features; int fd = s->vhost_vdpa.shared->device_fd; struct { struct vhost_vdpa_config hdr; uint32_t supported_hash_types; } config; - if (!virtio_has_feature(features, VIRTIO_NET_F_HASH_REPORT) && - !virtio_has_feature(features, VIRTIO_NET_F_RSS)) { + if (!vhost_dev_has_feature(s->vhost_vdpa.dev, VIRTIO_NET_F_HASH_REPORT) && + !vhost_dev_has_feature(s->vhost_vdpa.dev, VIRTIO_NET_F_RSS)) { return false; } @@ -397,6 +396,7 @@ static int vhost_vdpa_net_data_start(NetClientState *nc) { VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); struct vhost_vdpa *v = &s->vhost_vdpa; + bool has_cvq = v->dev->vq_index_end % 2; assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); @@ -406,6 +406,15 @@ static int vhost_vdpa_net_data_start(NetClientState *nc) v->shadow_vqs_enabled = false; } + if (!has_cvq) { + for (int i = 0; i < v->dev->nvqs; ++i) { + int ret = vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index); + if (ret < 0) { + return ret; + } + } + } + if (v->index == 0) { v->shared->shadow_data = v->shadow_vqs_enabled; vhost_vdpa_net_data_start_first(s); @@ -415,25 +424,6 @@ static int vhost_vdpa_net_data_start(NetClientState *nc) return 0; } -static int vhost_vdpa_net_data_load(NetClientState *nc) -{ - VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); - struct vhost_vdpa *v = &s->vhost_vdpa; - bool has_cvq = v->dev->vq_index_end % 2; - - if (has_cvq) { - return 0; - } - - for (int i = 0; i < v->dev->nvqs; ++i) { - int ret = vhost_vdpa_set_vring_ready(v, i + v->dev->vq_index); - if (ret < 0) { - return ret; - } - } - return 0; -} - static void vhost_vdpa_net_client_stop(NetClientState *nc) { VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); @@ -450,7 +440,6 @@ static NetClientInfo net_vhost_vdpa_info = { .size = sizeof(VhostVDPAState), .receive = vhost_vdpa_receive, .start = vhost_vdpa_net_data_start, - .load = vhost_vdpa_net_data_load, .stop = vhost_vdpa_net_client_stop, .cleanup = vhost_vdpa_cleanup, .has_vnet_hdr = vhost_vdpa_has_vnet_hdr, @@ -586,7 +575,7 @@ static int vhost_vdpa_net_cvq_start(NetClientState *nc) * If we early return in these cases SVQ will not be enabled. The migration * will be blocked as long as vhost-vdpa backends will not offer _F_LOG. */ - if (!vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) { + if (!vhost_vdpa_net_valid_svq_features(vhost_dev_features(v->dev), NULL)) { return 0; } diff --git a/qapi/virtio.json b/qapi/virtio.json index 09dd0e6d05..1fc4e38a44 100644 --- a/qapi/virtio.json +++ b/qapi/virtio.json @@ -85,8 +85,6 @@ # # @acked-features: vhost_dev acked_features # -# @backend-features: vhost_dev backend_features -# # @protocol-features: vhost_dev protocol_features # # @max-queues: vhost_dev max_queues @@ -106,7 +104,6 @@ 'vq-index': 'int', 'features': 'VirtioDeviceFeatures', 'acked-features': 'VirtioDeviceFeatures', - 'backend-features': 'VirtioDeviceFeatures', 'protocol-features': 'VhostDeviceProtocols', 'max-queues': 'uint64', 'backend-cap': 'uint64', @@ -653,12 +650,6 @@ # # @call: vhost_virtqueue call # -# @desc: vhost_virtqueue desc -# -# @avail: vhost_virtqueue avail -# -# @used: vhost_virtqueue used -# # @num: vhost_virtqueue num # # @desc-phys: vhost_virtqueue desc_phys (descriptor area physical @@ -681,9 +672,6 @@ 'data': { 'name': 'str', 'kick': 'int', 'call': 'int', - 'desc': 'uint64', - 'avail': 'uint64', - 'used': 'uint64', 'num': 'int', 'desc-phys': 'uint64', 'desc-size': 'uint32', @@ -723,12 +711,9 @@ # "avail-size": 2054, # "desc-size": 16384, # "used-size": 8198, -# "desc": 140141447430144, # "num": 1024, # "call": 0, -# "avail": 140141447446528, # "desc-phys": 5216108544, -# "used": 140141447448640, # "kick": 0 # } # } @@ -747,12 +732,9 @@ # "avail-size": 262, # "desc-size": 2048, # "used-size": 1030, -# "desc": 140141413580800, # "num": 128, # "call": 0, -# "avail": 140141413582848, # "desc-phys": 5182259200, -# "used": 140141413583168, # "kick": 0 # } # } diff --git a/subprojects/libvduse/libvduse.c b/subprojects/libvduse/libvduse.c index 21ffbb5b8d..df9ca5e56f 100644 --- a/subprojects/libvduse/libvduse.c +++ b/subprojects/libvduse/libvduse.c @@ -465,8 +465,9 @@ static int vduse_queue_read_indirect_desc(VduseDev *dev, struct vring_desc *desc, uint64_t addr, size_t len) { - struct vring_desc *ori_desc; + char *dst_desc = (char *)desc; uint64_t read_len; + void *ori_desc; if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) { return -1; @@ -483,10 +484,10 @@ vduse_queue_read_indirect_desc(VduseDev *dev, struct vring_desc *desc, return -1; } - memcpy(desc, ori_desc, read_len); + memcpy(dst_desc, ori_desc, read_len); len -= read_len; addr += read_len; - desc += read_len; + dst_desc += read_len; } return 0; diff --git a/subprojects/libvhost-user/libvhost-user.c b/subprojects/libvhost-user/libvhost-user.c index 9c630c2170..d2df50e3d0 100644 --- a/subprojects/libvhost-user/libvhost-user.c +++ b/subprojects/libvhost-user/libvhost-user.c @@ -1592,6 +1592,76 @@ vu_rm_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN]) return vu_send_message(dev, &msg); } +bool +vu_shmem_map(VuDev *dev, uint8_t shmid, uint64_t fd_offset, + uint64_t shm_offset, uint64_t len, uint64_t flags, int fd) +{ + VhostUserMsg vmsg = { + .request = VHOST_USER_BACKEND_SHMEM_MAP, + .size = sizeof(vmsg.payload.mmap), + .flags = VHOST_USER_VERSION, + .payload.mmap = { + .shmid = shmid, + .fd_offset = fd_offset, + .shm_offset = shm_offset, + .len = len, + .flags = flags, + }, + .fd_num = 1, + .fds[0] = fd, + }; + + if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHMEM)) { + return false; + } + + if (vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK)) { + vmsg.flags |= VHOST_USER_NEED_REPLY_MASK; + } + + pthread_mutex_lock(&dev->backend_mutex); + if (!vu_message_write(dev, dev->backend_fd, &vmsg)) { + pthread_mutex_unlock(&dev->backend_mutex); + return false; + } + + /* Also unlocks the backend_mutex */ + return vu_process_message_reply(dev, &vmsg); +} + +bool +vu_shmem_unmap(VuDev *dev, uint8_t shmid, uint64_t shm_offset, uint64_t len) +{ + VhostUserMsg vmsg = { + .request = VHOST_USER_BACKEND_SHMEM_UNMAP, + .size = sizeof(vmsg.payload.mmap), + .flags = VHOST_USER_VERSION, + .payload.mmap = { + .shmid = shmid, + .fd_offset = 0, + .shm_offset = shm_offset, + .len = len, + }, + }; + + if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHMEM)) { + return false; + } + + if (vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK)) { + vmsg.flags |= VHOST_USER_NEED_REPLY_MASK; + } + + pthread_mutex_lock(&dev->backend_mutex); + if (!vu_message_write(dev, dev->backend_fd, &vmsg)) { + pthread_mutex_unlock(&dev->backend_mutex); + return false; + } + + /* Also unlocks the backend_mutex */ + return vu_process_message_reply(dev, &vmsg); +} + static bool vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg) { @@ -2391,8 +2461,9 @@ static int virtqueue_read_indirect_desc(VuDev *dev, struct vring_desc *desc, uint64_t addr, size_t len) { - struct vring_desc *ori_desc; + char *dst_desc = (char *)desc; uint64_t read_len; + void *ori_desc; if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) { return -1; @@ -2409,10 +2480,10 @@ virtqueue_read_indirect_desc(VuDev *dev, struct vring_desc *desc, return -1; } - memcpy(desc, ori_desc, read_len); + memcpy(dst_desc, ori_desc, read_len); len -= read_len; addr += read_len; - desc += read_len; + dst_desc += read_len; } return 0; diff --git a/subprojects/libvhost-user/libvhost-user.h b/subprojects/libvhost-user/libvhost-user.h index 2ffc58c11b..a534d70771 100644 --- a/subprojects/libvhost-user/libvhost-user.h +++ b/subprojects/libvhost-user/libvhost-user.h @@ -69,6 +69,10 @@ enum VhostUserProtocolFeature { /* Feature 16 is reserved for VHOST_USER_PROTOCOL_F_STATUS. */ /* Feature 17 reserved for VHOST_USER_PROTOCOL_F_XEN_MMAP. */ VHOST_USER_PROTOCOL_F_SHARED_OBJECT = 18, + /* Feature 19 is reserved for VHOST_USER_PROTOCOL_F_DEVICE_STATE */ + /* Feature 20 is reserved for VHOST_USER_PROTOCOL_F_GET_VRING_BASE_INFLIGHT */ + /* Feature 21 is reserved for VHOST_USER_PROTOCOL_F_GPA_ADDRESSES */ + VHOST_USER_PROTOCOL_F_SHMEM = 22, VHOST_USER_PROTOCOL_F_MAX }; @@ -127,6 +131,8 @@ typedef enum VhostUserBackendRequest { VHOST_USER_BACKEND_SHARED_OBJECT_ADD = 6, VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE = 7, VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP = 8, + VHOST_USER_BACKEND_SHMEM_MAP = 9, + VHOST_USER_BACKEND_SHMEM_UNMAP = 10, VHOST_USER_BACKEND_MAX } VhostUserBackendRequest; @@ -186,6 +192,23 @@ typedef struct VhostUserShared { unsigned char uuid[UUID_LEN]; } VhostUserShared; +/* For the flags field of VhostUserMMap */ +#define VHOST_USER_FLAG_MAP_RW (1u << 0) + +typedef struct { + /* VIRTIO Shared Memory Region ID */ + uint8_t shmid; + uint8_t padding[7]; + /* File offset */ + uint64_t fd_offset; + /* Offset within the VIRTIO Shared Memory Region */ + uint64_t shm_offset; + /* Size of the mapping */ + uint64_t len; + /* Flags for the mmap operation, from VHOST_USER_FLAG_MAP_* */ + uint64_t flags; +} VhostUserMMap; + #define VU_PACKED __attribute__((packed)) typedef struct VhostUserMsg { @@ -210,6 +233,7 @@ typedef struct VhostUserMsg { VhostUserVringArea area; VhostUserInflight inflight; VhostUserShared object; + VhostUserMMap mmap; } payload; int fds[VHOST_MEMORY_BASELINE_NREGIONS]; @@ -593,6 +617,38 @@ bool vu_add_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN]); */ bool vu_rm_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN]); +/** + * vu_shmem_map: + * @dev: a VuDev context + * @shmid: VIRTIO Shared Memory Region ID + * @fd_offset: File offset + * @shm_offset: Offset within the VIRTIO Shared Memory Region + * @len: Size of the mapping + * @flags: Flags for the mmap operation + * @fd: A file descriptor + * + * Advertises a new mapping to be made in a given VIRTIO Shared Memory Region. + * + * Returns: TRUE on success, FALSE on failure. + */ +bool vu_shmem_map(VuDev *dev, uint8_t shmid, uint64_t fd_offset, + uint64_t shm_offset, uint64_t len, uint64_t flags, int fd); + +/** + * vu_shmem_unmap: + * @dev: a VuDev context + * @shmid: VIRTIO Shared Memory Region ID + * @fd_offset: File offset + * @len: Size of the mapping + * + * The front-end un-mmaps a given range in the VIRTIO Shared Memory Region + * with the requested `shmid`. + * + * Returns: TRUE on success, FALSE on failure. + */ +bool vu_shmem_unmap(VuDev *dev, uint8_t shmid, uint64_t shm_offset, + uint64_t len); + /** * vu_queue_set_notification: * @dev: a VuDev context diff --git a/system/memory.c b/system/memory.c index 739ba11da6..96c13664fe 100644 --- a/system/memory.c +++ b/system/memory.c @@ -1652,6 +1652,20 @@ bool memory_region_init_ram_from_fd(MemoryRegion *mr, Object *owner, false, errp); return memory_region_set_ram_block(mr, rb); } +#else +bool memory_region_init_ram_from_fd(MemoryRegion *mr, + Object *owner, + const char *name, + uint64_t size, + uint32_t ram_flags, + int fd, + ram_addr_t offset, + Error **errp) +{ + error_setg(errp, + "memory_region_init_ram_from_fd is not supported on this platform"); + return false; +} #endif static void memory_region_set_ram_ptr(MemoryRegion *mr, uint64_t size, diff --git a/tests/data/acpi/x86/microvm/DSDT b/tests/data/acpi/x86/microvm/DSDT index f477668f2e..5d03103170 100644 Binary files a/tests/data/acpi/x86/microvm/DSDT and b/tests/data/acpi/x86/microvm/DSDT differ diff --git a/tests/data/acpi/x86/microvm/DSDT.ioapic2 b/tests/data/acpi/x86/microvm/DSDT.ioapic2 index e5924a4996..b48db019ff 100644 Binary files a/tests/data/acpi/x86/microvm/DSDT.ioapic2 and b/tests/data/acpi/x86/microvm/DSDT.ioapic2 differ diff --git a/tests/data/acpi/x86/microvm/DSDT.pcie b/tests/data/acpi/x86/microvm/DSDT.pcie index b646a05551..b92fb08532 100644 Binary files a/tests/data/acpi/x86/microvm/DSDT.pcie and b/tests/data/acpi/x86/microvm/DSDT.pcie differ diff --git a/tests/data/acpi/x86/microvm/DSDT.rtc b/tests/data/acpi/x86/microvm/DSDT.rtc index e375473b5f..c101332010 100644 Binary files a/tests/data/acpi/x86/microvm/DSDT.rtc and b/tests/data/acpi/x86/microvm/DSDT.rtc differ diff --git a/tests/data/acpi/x86/microvm/DSDT.usb b/tests/data/acpi/x86/microvm/DSDT.usb index d63fd84620..9261b67dc0 100644 Binary files a/tests/data/acpi/x86/microvm/DSDT.usb and b/tests/data/acpi/x86/microvm/DSDT.usb differ diff --git a/tests/data/acpi/x86/pc/DSDT b/tests/data/acpi/x86/pc/DSDT index 6ea2d36d13..e30e7091dd 100644 Binary files a/tests/data/acpi/x86/pc/DSDT and b/tests/data/acpi/x86/pc/DSDT differ diff --git a/tests/data/acpi/x86/pc/DSDT.acpierst b/tests/data/acpi/x86/pc/DSDT.acpierst index d8c173aa61..4e044bb4eb 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.acpierst and b/tests/data/acpi/x86/pc/DSDT.acpierst differ diff --git a/tests/data/acpi/x86/pc/DSDT.acpihmat b/tests/data/acpi/x86/pc/DSDT.acpihmat index ba363d6af7..54ce2df998 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.acpihmat and b/tests/data/acpi/x86/pc/DSDT.acpihmat differ diff --git a/tests/data/acpi/x86/pc/DSDT.bridge b/tests/data/acpi/x86/pc/DSDT.bridge index b68302363c..957e12c296 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.bridge and b/tests/data/acpi/x86/pc/DSDT.bridge differ diff --git a/tests/data/acpi/x86/pc/DSDT.cphp b/tests/data/acpi/x86/pc/DSDT.cphp index 20688edf2d..64e1b6693d 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.cphp and b/tests/data/acpi/x86/pc/DSDT.cphp differ diff --git a/tests/data/acpi/x86/pc/DSDT.dimmpxm b/tests/data/acpi/x86/pc/DSDT.dimmpxm index 8d4be05d2c..3c27fd0d1e 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.dimmpxm and b/tests/data/acpi/x86/pc/DSDT.dimmpxm differ diff --git a/tests/data/acpi/x86/pc/DSDT.hpbridge b/tests/data/acpi/x86/pc/DSDT.hpbridge index 2b5b885b86..89c4df716a 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.hpbridge and b/tests/data/acpi/x86/pc/DSDT.hpbridge differ diff --git a/tests/data/acpi/x86/pc/DSDT.hpbrroot b/tests/data/acpi/x86/pc/DSDT.hpbrroot index cc6f26a3f8..8cd0d2a4cf 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.hpbrroot and b/tests/data/acpi/x86/pc/DSDT.hpbrroot differ diff --git a/tests/data/acpi/x86/pc/DSDT.ipmikcs b/tests/data/acpi/x86/pc/DSDT.ipmikcs index 052a84e294..97bfbf74ef 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.ipmikcs and b/tests/data/acpi/x86/pc/DSDT.ipmikcs differ diff --git a/tests/data/acpi/x86/pc/DSDT.memhp b/tests/data/acpi/x86/pc/DSDT.memhp index 7efc12a46c..b2c4c70d9c 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.memhp and b/tests/data/acpi/x86/pc/DSDT.memhp differ diff --git a/tests/data/acpi/x86/pc/DSDT.nohpet b/tests/data/acpi/x86/pc/DSDT.nohpet index 7eedfcd64e..f762b83ee7 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.nohpet and b/tests/data/acpi/x86/pc/DSDT.nohpet differ diff --git a/tests/data/acpi/x86/pc/DSDT.numamem b/tests/data/acpi/x86/pc/DSDT.numamem index 910b4952a0..a215ea995a 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.numamem and b/tests/data/acpi/x86/pc/DSDT.numamem differ diff --git a/tests/data/acpi/x86/pc/DSDT.roothp b/tests/data/acpi/x86/pc/DSDT.roothp index 45d3dbe1b6..6a1557bfd8 100644 Binary files a/tests/data/acpi/x86/pc/DSDT.roothp and b/tests/data/acpi/x86/pc/DSDT.roothp differ diff --git a/tests/data/acpi/x86/q35/DSDT b/tests/data/acpi/x86/q35/DSDT index 377e880175..55b44cfa70 100644 Binary files a/tests/data/acpi/x86/q35/DSDT and b/tests/data/acpi/x86/q35/DSDT differ diff --git a/tests/data/acpi/x86/q35/DSDT.acpierst b/tests/data/acpi/x86/q35/DSDT.acpierst index 026bfdfebf..50691d93ed 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.acpierst and b/tests/data/acpi/x86/q35/DSDT.acpierst differ diff --git a/tests/data/acpi/x86/q35/DSDT.acpihmat b/tests/data/acpi/x86/q35/DSDT.acpihmat index f1b8483d8d..6b1b87699e 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.acpihmat and b/tests/data/acpi/x86/q35/DSDT.acpihmat differ diff --git a/tests/data/acpi/x86/q35/DSDT.acpihmat-generic-x b/tests/data/acpi/x86/q35/DSDT.acpihmat-generic-x index a7731403f4..833e55273c 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.acpihmat-generic-x and b/tests/data/acpi/x86/q35/DSDT.acpihmat-generic-x differ diff --git a/tests/data/acpi/x86/q35/DSDT.acpihmat-noinitiator b/tests/data/acpi/x86/q35/DSDT.acpihmat-noinitiator index cb4995de7e..4bbff971ec 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.acpihmat-noinitiator and b/tests/data/acpi/x86/q35/DSDT.acpihmat-noinitiator differ diff --git a/tests/data/acpi/x86/q35/DSDT.applesmc b/tests/data/acpi/x86/q35/DSDT.applesmc index 92c8fdb6cb..01834e412b 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.applesmc and b/tests/data/acpi/x86/q35/DSDT.applesmc differ diff --git a/tests/data/acpi/x86/q35/DSDT.bridge b/tests/data/acpi/x86/q35/DSDT.bridge index 957b3ad90c..e379a9be89 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.bridge and b/tests/data/acpi/x86/q35/DSDT.bridge differ diff --git a/tests/data/acpi/x86/q35/DSDT.core-count b/tests/data/acpi/x86/q35/DSDT.core-count index 50ca91b065..50d6eac8ba 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.core-count and b/tests/data/acpi/x86/q35/DSDT.core-count differ diff --git a/tests/data/acpi/x86/q35/DSDT.core-count2 b/tests/data/acpi/x86/q35/DSDT.core-count2 index f460be2bf7..69b31da613 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.core-count2 and b/tests/data/acpi/x86/q35/DSDT.core-count2 differ diff --git a/tests/data/acpi/x86/q35/DSDT.cphp b/tests/data/acpi/x86/q35/DSDT.cphp index 7c87d41d03..d46bac4e25 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.cphp and b/tests/data/acpi/x86/q35/DSDT.cphp differ diff --git a/tests/data/acpi/x86/q35/DSDT.cxl b/tests/data/acpi/x86/q35/DSDT.cxl index da86b25f51..9517048a57 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.cxl and b/tests/data/acpi/x86/q35/DSDT.cxl differ diff --git a/tests/data/acpi/x86/q35/DSDT.dimmpxm b/tests/data/acpi/x86/q35/DSDT.dimmpxm index a2d812e5a2..e817aa6d91 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.dimmpxm and b/tests/data/acpi/x86/q35/DSDT.dimmpxm differ diff --git a/tests/data/acpi/x86/q35/DSDT.ipmibt b/tests/data/acpi/x86/q35/DSDT.ipmibt index 43ac1bd693..e004d4e385 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.ipmibt and b/tests/data/acpi/x86/q35/DSDT.ipmibt differ diff --git a/tests/data/acpi/x86/q35/DSDT.ipmismbus b/tests/data/acpi/x86/q35/DSDT.ipmismbus index 1b998820d4..3cc9625a9b 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.ipmismbus and b/tests/data/acpi/x86/q35/DSDT.ipmismbus differ diff --git a/tests/data/acpi/x86/q35/DSDT.ivrs b/tests/data/acpi/x86/q35/DSDT.ivrs index 026bfdfebf..50691d93ed 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.ivrs and b/tests/data/acpi/x86/q35/DSDT.ivrs differ diff --git a/tests/data/acpi/x86/q35/DSDT.memhp b/tests/data/acpi/x86/q35/DSDT.memhp index 7346125d23..0dd16e8455 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.memhp and b/tests/data/acpi/x86/q35/DSDT.memhp differ diff --git a/tests/data/acpi/x86/q35/DSDT.mmio64 b/tests/data/acpi/x86/q35/DSDT.mmio64 index 15a291dbfb..0f436d68fc 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.mmio64 and b/tests/data/acpi/x86/q35/DSDT.mmio64 differ diff --git a/tests/data/acpi/x86/q35/DSDT.multi-bridge b/tests/data/acpi/x86/q35/DSDT.multi-bridge index 889a9040d9..1e3f17690f 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.multi-bridge and b/tests/data/acpi/x86/q35/DSDT.multi-bridge differ diff --git a/tests/data/acpi/x86/q35/DSDT.noacpihp b/tests/data/acpi/x86/q35/DSDT.noacpihp index 780616774f..2138ef2cad 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.noacpihp and b/tests/data/acpi/x86/q35/DSDT.noacpihp differ diff --git a/tests/data/acpi/x86/q35/DSDT.nohpet b/tests/data/acpi/x86/q35/DSDT.nohpet index 0f862ab293..20468b6b00 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.nohpet and b/tests/data/acpi/x86/q35/DSDT.nohpet differ diff --git a/tests/data/acpi/x86/q35/DSDT.numamem b/tests/data/acpi/x86/q35/DSDT.numamem index df8edc05b6..c24caffd8d 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.numamem and b/tests/data/acpi/x86/q35/DSDT.numamem differ diff --git a/tests/data/acpi/x86/q35/DSDT.pvpanic-isa b/tests/data/acpi/x86/q35/DSDT.pvpanic-isa index da3ce12787..9355f161aa 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.pvpanic-isa and b/tests/data/acpi/x86/q35/DSDT.pvpanic-isa differ diff --git a/tests/data/acpi/x86/q35/DSDT.thread-count b/tests/data/acpi/x86/q35/DSDT.thread-count index 50ca91b065..50d6eac8ba 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.thread-count and b/tests/data/acpi/x86/q35/DSDT.thread-count differ diff --git a/tests/data/acpi/x86/q35/DSDT.thread-count2 b/tests/data/acpi/x86/q35/DSDT.thread-count2 index f460be2bf7..69b31da613 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.thread-count2 and b/tests/data/acpi/x86/q35/DSDT.thread-count2 differ diff --git a/tests/data/acpi/x86/q35/DSDT.tis.tpm12 b/tests/data/acpi/x86/q35/DSDT.tis.tpm12 index 67ebd7c158..a16d232e61 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.tis.tpm12 and b/tests/data/acpi/x86/q35/DSDT.tis.tpm12 differ diff --git a/tests/data/acpi/x86/q35/DSDT.tis.tpm2 b/tests/data/acpi/x86/q35/DSDT.tis.tpm2 index c6b5847215..96e4b7b731 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.tis.tpm2 and b/tests/data/acpi/x86/q35/DSDT.tis.tpm2 differ diff --git a/tests/data/acpi/x86/q35/DSDT.type4-count b/tests/data/acpi/x86/q35/DSDT.type4-count index 17a64adb20..dd0a3a7581 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.type4-count and b/tests/data/acpi/x86/q35/DSDT.type4-count differ diff --git a/tests/data/acpi/x86/q35/DSDT.viot b/tests/data/acpi/x86/q35/DSDT.viot index 6eb30e8f4b..e00340c955 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.viot and b/tests/data/acpi/x86/q35/DSDT.viot differ diff --git a/tests/data/acpi/x86/q35/DSDT.xapic b/tests/data/acpi/x86/q35/DSDT.xapic index 111bb041dc..50780e37ee 100644 Binary files a/tests/data/acpi/x86/q35/DSDT.xapic and b/tests/data/acpi/x86/q35/DSDT.xapic differ diff --git a/tests/data/acpi/x86/q35/WDAT.wdat b/tests/data/acpi/x86/q35/WDAT.wdat new file mode 100644 index 0000000000..e1e056b390 Binary files /dev/null and b/tests/data/acpi/x86/q35/WDAT.wdat differ diff --git a/tests/qtest/amd-iommu-test.c b/tests/qtest/amd-iommu-test.c new file mode 100644 index 0000000000..fb28511588 --- /dev/null +++ b/tests/qtest/amd-iommu-test.c @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "libqtest.h" +#include "hw/i386/amd_iommu.h" + +#define CMDBUF_ADDR 0x200000 +#define CMDBUF_LEN_FIELD 8 +#define CMDBUF_ENTRIES (1U << CMDBUF_LEN_FIELD) + +static inline uint64_t amdvi_reg_readq(QTestState *s, uint64_t offset) +{ + return qtest_readq(s, AMDVI_BASE_ADDR + offset); +} + +static inline void amdvi_reg_writeq(QTestState *s, uint64_t offset, + uint64_t val) +{ + qtest_writeq(s, AMDVI_BASE_ADDR + offset, val); +} + +static void test_cmdbuf_head_wrap(void) +{ + QTestState *s; + uint64_t head; + int i; + /* 16 bytes per command */ + struct { + uint64_t qw0; + uint64_t qw1; + } cmdbuf[CMDBUF_ENTRIES]; + + if (!qtest_has_machine("q35")) { + g_test_skip("q35 machine not available"); + return; + } + + s = qtest_init("-M q35 -device amd-iommu"); + + /* fill the command buffer with COMPLETION_WAIT (no-op) commands */ + for (i = 0; i < CMDBUF_ENTRIES; i++) { + cmdbuf[i].qw0 = (uint64_t)AMDVI_CMD_COMPLETION_WAIT << 60; + cmdbuf[i].qw1 = 0; + } + qtest_memwrite(s, CMDBUF_ADDR, cmdbuf, sizeof(cmdbuf)); + + /* point the IOMMU at the command buffer and set its length */ + amdvi_reg_writeq(s, AMDVI_MMIO_COMMAND_BASE, + CMDBUF_ADDR | ((uint64_t)CMDBUF_LEN_FIELD << 56)); + + /* enable the IOMMU and its command buffer processor */ + amdvi_reg_writeq(s, AMDVI_MMIO_CONTROL, + AMDVI_MMIO_CONTROL_AMDVIEN | AMDVI_MMIO_CONTROL_CMDBUFLEN); + + /* advance tail to the last entry, consuming all but the final entry */ + amdvi_reg_writeq(s, AMDVI_MMIO_COMMAND_TAIL, + (CMDBUF_ENTRIES - 1) * AMDVI_COMMAND_SIZE); + + /* wrap tail to 0, consuming the final entry and completing the buffer */ + amdvi_reg_writeq(s, AMDVI_MMIO_COMMAND_TAIL, 0); + + /* after consuming all entries the IOMMU must wrap CmdHeadPtr to 0 */ + head = amdvi_reg_readq(s, AMDVI_MMIO_COMMAND_HEAD); + g_assert((head & AMDVI_MMIO_CMDBUF_HEAD_MASK) == 0); + + qtest_quit(s); +} + +int main(int argc, char **argv) +{ + g_test_init(&argc, &argv, NULL); + qtest_add_func("/q35/amd-iommu/cmdbuf-head-wrap", test_cmdbuf_head_wrap); + return g_test_run(); +} diff --git a/tests/qtest/bios-tables-test.c b/tests/qtest/bios-tables-test.c index 510751799e..af6d9b5136 100644 --- a/tests/qtest/bios-tables-test.c +++ b/tests/qtest/bios-tables-test.c @@ -2534,6 +2534,18 @@ static void test_acpi_isapc_smbios_legacy(void) free_test_data(&data); } +static void test_acpi_q35_wdat(void) +{ + test_data data = { + .machine = MACHINE_Q35, + .arch = "x86", + .variant = ".wdat", + }; + + test_acpi_one("-machine wdat=on", &data); + free_test_data(&data); +} + static void test_oem_fields(test_data *data) { int i; @@ -2828,6 +2840,7 @@ int main(int argc, char *argv[]) qtest_add_func("acpi/q35/cxl", test_acpi_q35_cxl); #endif qtest_add_func("acpi/q35/slic", test_acpi_q35_slic); + qtest_add_func("acpi/q35/wdat", test_acpi_q35_wdat); } if (qtest_has_machine("microvm")) { qtest_add_func("acpi/microvm", test_acpi_microvm_tcg); diff --git a/tests/qtest/intel-iommu-test.c b/tests/qtest/intel-iommu-test.c index e5cc6acaf0..ef6e6eb8b6 100644 --- a/tests/qtest/intel-iommu-test.c +++ b/tests/qtest/intel-iommu-test.c @@ -17,11 +17,39 @@ #define ECAP_STAGE_1_FIXED1 (VTD_ECAP_QI | VTD_ECAP_IR | VTD_ECAP_IRO | \ VTD_ECAP_MHMV | VTD_ECAP_SMTS | VTD_ECAP_FSTS) +static inline uint32_t vtd_reg_readl(QTestState *s, uint64_t offset) +{ + return qtest_readl(s, Q35_HOST_BRIDGE_IOMMU_ADDR + offset); +} + static inline uint64_t vtd_reg_readq(QTestState *s, uint64_t offset) { return qtest_readq(s, Q35_HOST_BRIDGE_IOMMU_ADDR + offset); } +static inline void vtd_reg_writeq(QTestState *s, uint64_t offset, + uint64_t value) +{ + qtest_writeq(s, Q35_HOST_BRIDGE_IOMMU_ADDR + offset, value); +} + +static void test_intel_iommu_8byte_access(void) +{ + QTestState *s; + uint64_t off; + + s = qtest_init("-M q35 -device intel-iommu"); + + for (off = 0; off < DMAR_REG_SIZE; off += 4) { + vtd_reg_readq(s, off); + vtd_reg_writeq(s, off, 0); + } + + g_assert_cmpuint(vtd_reg_readl(s, DMAR_VER_REG), !=, 0); + + qtest_quit(s); +} + static void test_intel_iommu_stage_1(void) { uint8_t init_csr[DMAR_REG_SIZE]; /* register values */ @@ -29,7 +57,7 @@ static void test_intel_iommu_stage_1(void) uint64_t cap, ecap, tmp; QTestState *s; - s = qtest_init("-M q35 -device intel-iommu,x-scalable-mode=on,x-flts=on"); + s = qtest_init("-M q35 -device intel-iommu,scalable-mode=on,fsts=on"); cap = vtd_reg_readq(s, DMAR_CAP_REG); g_assert((cap & CAP_STAGE_1_FIXED1) == CAP_STAGE_1_FIXED1); @@ -58,6 +86,8 @@ static void test_intel_iommu_stage_1(void) int main(int argc, char **argv) { g_test_init(&argc, &argv, NULL); + qtest_add_func("/q35/intel-iommu/8byte-access", + test_intel_iommu_8byte_access); qtest_add_func("/q35/intel-iommu/stage-1", test_intel_iommu_stage_1); return g_test_run(); diff --git a/tests/qtest/iommu-intel-test.c b/tests/qtest/iommu-intel-test.c index a52c45e298..703bbfef73 100644 --- a/tests/qtest/iommu-intel-test.c +++ b/tests/qtest/iommu-intel-test.c @@ -54,10 +54,10 @@ static const char *qvtd_iommu_args(QVTDTransMode mode) { switch (mode) { case QVTD_TM_SCALABLE_FLT: - return "-device intel-iommu,x-scalable-mode=on,x-flts=on "; + return "-device intel-iommu,scalable-mode=on,fsts=on "; case QVTD_TM_SCALABLE_PT: case QVTD_TM_SCALABLE_SLT: - return "-device intel-iommu,x-scalable-mode=on "; + return "-device intel-iommu,scalable-mode=on "; default: return "-device intel-iommu "; } diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index 45ea497fa5..2ae720156f 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -95,6 +95,7 @@ qtests_i386 = \ (config_all_devices.has_key('CONFIG_SB16') ? ['fuzz-sb16-test'] : []) + \ (config_all_devices.has_key('CONFIG_SDHCI_PCI') ? ['fuzz-sdcard-test'] : []) + \ (config_all_devices.has_key('CONFIG_ESP_PCI') ? ['am53c974-test'] : []) + \ + (config_all_devices.has_key('CONFIG_AMD_IOMMU') ? ['amd-iommu-test'] : []) + \ (config_all_devices.has_key('CONFIG_VTD') ? ['intel-iommu-test'] : []) + \ (config_all_devices.has_key('CONFIG_VTD') and config_all_devices.has_key('CONFIG_IOMMU_TESTDEV') ? ['iommu-intel-test'] : []) + \