vfio: Add VFIO I/O backend capability flags for feature support

Different VFIO I/O backends support different features. For example,
the kernel VFIO backend supports DMA-BUF creation, while vfio-user
does not. Currently, this is handled by attempting the operation and
checking for -ENOTTY, which can lead to misleading warnings when a
feature is simply not supported by a particular backend.

Introduce a capability flags mechanism in VFIODeviceIOOps that allows
backends to explicitly advertise which features they support. Callers
can check these capabilities before attempting operations, avoiding
spurious errors and warnings.

Cc: John Levon <john.levon@nutanix.com>
Reviewed-by: John Levon <john.levon@nutanix.com>
Link: https://lore.kernel.org/qemu-devel/20260409114312.1704062-1-clg@redhat.com
Signed-off-by: Cédric Le Goater <clg@redhat.com>

hw/vfio/device.c

@@ -648,6 +648,8 @@ static int vfio_device_io_region_write(VFIODevice *vbasedev, uint8_t index,
 }
 
 static VFIODeviceIOOps vfio_device_io_ops_ioctl = {
+    .capabilities = VFIO_IO_CAP_DMA_BUF,
+
     .device_feature = vfio_device_io_device_feature,
     .get_region_info = vfio_device_io_get_region_info,
     .get_irq_info = vfio_device_io_get_irq_info,

hw/vfio/region.c

@@ -293,6 +293,11 @@ static bool vfio_region_create_dma_buf(VFIORegion *region, Error **errp)
     size_t total_size;
     int i, ret;
 
+    /* Check if backend supports DMA-BUF creation */
+    if (!(vbasedev->io_ops->capabilities & VFIO_IO_CAP_DMA_BUF)) {
+        return true;
+    }
+
     total_size = sizeof(*feature) + sizeof(*dma_buf) +
                  sizeof(struct vfio_region_dma_range) * region->nr_mmaps;
     feature = g_malloc0(total_size);

include/hw/vfio/vfio-device.h

@@ -172,12 +172,25 @@ typedef QLIST_HEAD(VFIODeviceList, VFIODevice) VFIODeviceList;
 extern VFIODeviceList vfio_device_list;
 
 #ifdef CONFIG_LINUX
+/*
+ * VFIO backend I/O operation capabilities
+ */
+#define VFIO_IO_CAP_DMA_BUF        (1ULL << 0)
+
 /*
  * How devices communicate with the server.  The default option is through
  * ioctl() to the kernel VFIO driver, but vfio-user can use a socket to a remote
  * process.
  */
 struct VFIODeviceIOOps {
+    /**
+     * @capabilities
+     *
+     * Bitmask of VFIO_IO_CAP_* flags indicating which features this
+     * backend supports.
+     */
+    uint64_t capabilities;
+
     /**
      * @device_feature
      *