/*
 * Copyright (c) 2025 Huawei Technologies R & D (UK) Ltd
 * Copyright (C) 2025 NVIDIA
 * Written by Nicolin Chen, Shameer Kolothum
 *
 * SPDX-License-Identifier: GPL-2.0-or-later
 */

#include "qemu/osdep.h"
#include "qemu/error-report.h"
#include "trace.h"

#include "hw/arm/smmuv3.h"
#include "hw/core/iommu.h"
#include "hw/pci/pci_bridge.h"
#include "hw/pci-host/gpex.h"
#include "hw/vfio/pci.h"

#include "smmuv3-internal.h"
#include "smmuv3-accel.h"
#include "system/system.h"
#include "tegra241-cmdqv.h"

/*
 * The root region aliases the global system memory, and shared_as_sysmem
 * provides a shared Address Space referencing it. This Address Space is used
 * by all vfio-pci devices behind all accelerated SMMUv3 instances within a VM.
 */
static MemoryRegion root, sysmem;
static AddressSpace *shared_as_sysmem;

static int smmuv3_oas_bits(uint32_t oas)
{
    static const int map[] = { 32, 36, 40, 42, 44, 48, 52, 56 };

    g_assert(oas < ARRAY_SIZE(map));
    return map[oas];
}

static void smmuv3_accel_auto_finalise(SMMUv3State *s,
                                       struct iommu_hw_info_arm_smmuv3 *info)
{
    SMMUv3AccelState *accel = s->s_accel;

    /*
     * Return if 'auto' was not set for any accel SMMUv3 property, or
     * if property values were already resolved from a previous call
     * to this function (e.g. if this function was called again after
     * VM boot during device hot plug). We do not accept new property
     * values in this case where auto_finalised == true, and we re-use
     * the values determined from the initial cold plug.
     */
    if (!accel->auto_mode || accel->auto_finalised) {
        return;
    }

    if (s->ats == ON_OFF_AUTO_AUTO) {
        s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ATS,
                               FIELD_EX32(info->idr[0], IDR0, ATS));
    }

    if (s->ril == ON_OFF_AUTO_AUTO) {
        s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL,
                               FIELD_EX32(info->idr[3], IDR3, RIL));
    }

    if (s->ssidsize == SSID_SIZE_MODE_AUTO) {
        s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SSIDSIZE,
                               FIELD_EX32(info->idr[1], IDR1, SSIDSIZE));
    }

    if (s->oas == OAS_MODE_AUTO) {
        s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS,
                               FIELD_EX32(info->idr[5], IDR5, OAS));
    }

    accel->auto_finalised = true;
}

static bool
smmuv3_accel_check_hw_compatible(SMMUv3State *s,
                                 struct iommu_hw_info_arm_smmuv3 *info,
                                 Error **errp)
{
    smmuv3_accel_auto_finalise(s, info);

    /* QEMU SMMUv3 supports both linear and 2-level stream tables */
    if (FIELD_EX32(info->idr[0], IDR0, STLEVEL) !=
                FIELD_EX32(s->idr[0], IDR0, STLEVEL)) {
        error_setg(errp, "Host SMMUv3 Stream Table format mismatch "
                   "(host STLEVEL=%u, QEMU STLEVEL=%u)",
                   FIELD_EX32(info->idr[0], IDR0, STLEVEL),
                   FIELD_EX32(s->idr[0], IDR0, STLEVEL));
        return false;
    }

    /* QEMU SMMUv3 supports only little-endian translation table walks */
    if (FIELD_EX32(info->idr[0], IDR0, TTENDIAN) >
                FIELD_EX32(s->idr[0], IDR0, TTENDIAN)) {
        error_setg(errp, "Host SMMUv3 doesn't support Little-endian "
                   "translation table");
        return false;
    }

    /* QEMU SMMUv3 supports only AArch64 translation table format */
    if (FIELD_EX32(info->idr[0], IDR0, TTF) <
                FIELD_EX32(s->idr[0], IDR0, TTF)) {
        error_setg(errp, "Host SMMUv3 doesn't support AArch64 translation "
                   "table format");
        return false;
    }

    /* QEMU SMMUv3 supports SIDSIZE 16 */
    if (FIELD_EX32(info->idr[1], IDR1, SIDSIZE) <
                FIELD_EX32(s->idr[1], IDR1, SIDSIZE)) {
        error_setg(errp, "Host SMMUv3 SIDSIZE not compatible "
                   "(host=%u, QEMU=%u)",
                   FIELD_EX32(info->idr[1], IDR1, SIDSIZE),
                   FIELD_EX32(s->idr[1], IDR1, SIDSIZE));
        return false;
    }

    /* Check SSIDSIZE value opted-in is compatible with Host SMMUv3 SSIDSIZE */
    if (FIELD_EX32(info->idr[1], IDR1, SSIDSIZE) <
                FIELD_EX32(s->idr[1], IDR1, SSIDSIZE)) {
        error_setg(errp, "Host SMMUv3 SSIDSIZE not compatible "
                   "(host=%u, QEMU=%u)",
                   FIELD_EX32(info->idr[1], IDR1, SSIDSIZE),
                   FIELD_EX32(s->idr[1], IDR1, SSIDSIZE));
        return false;
    }

    /* User can disable QEMU SMMUv3 Range Invalidation support */
    if (FIELD_EX32(info->idr[3], IDR3, RIL) <
                FIELD_EX32(s->idr[3], IDR3, RIL)) {
        error_setg(errp, "Host SMMUv3 doesn't support Range Invalidation");
        return false;
    }
    /* Check OAS value opted is compatible with Host SMMUv3 IPA */
    if (FIELD_EX32(info->idr[5], IDR5, OAS) <
                FIELD_EX32(s->idr[5], IDR5, OAS)) {
        error_setg(errp, "Host SMMUv3 supports only %d-bit IPA, but the vSMMU "
                   "OAS implies %d-bit IPA",
                   smmuv3_oas_bits(FIELD_EX32(info->idr[5], IDR5, OAS)),
                   smmuv3_oas_bits(FIELD_EX32(s->idr[5], IDR5, OAS)));
        return false;
    }
    /* Check ATS value opted is compatible with Host SMMUv3 */
    if (FIELD_EX32(info->idr[0], IDR0, ATS) <
                FIELD_EX32(s->idr[0], IDR0, ATS)) {
        error_setg(errp, "Host SMMUv3 doesn't support Address Translation Services");
        return false;
    }

    /* QEMU SMMUv3 supports GRAN4K/GRAN16K/GRAN64K translation granules */
    if (FIELD_EX32(info->idr[5], IDR5, GRAN4K) !=
                FIELD_EX32(s->idr[5], IDR5, GRAN4K)) {
        error_setg(errp, "Host SMMUv3 doesn't support 4K translation granule");
        return false;
    }
    if (FIELD_EX32(info->idr[5], IDR5, GRAN16K) !=
                FIELD_EX32(s->idr[5], IDR5, GRAN16K)) {
        error_setg(errp, "Host SMMUv3 doesn't support 16K translation granule");
        return false;
    }
    if (FIELD_EX32(info->idr[5], IDR5, GRAN64K) !=
                FIELD_EX32(s->idr[5], IDR5, GRAN64K)) {
        error_setg(errp, "Host SMMUv3 doesn't support 64K translation granule");
        return false;
    }

    return true;
}

static bool
smmuv3_accel_hw_compatible(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *hiodi,
                           Error **errp)
{
    struct iommu_hw_info_arm_smmuv3 info;
    uint32_t data_type = IOMMU_HW_INFO_TYPE_DEFAULT;
    uint64_t caps;

    if (!iommufd_backend_get_device_info(hiodi->iommufd, hiodi->devid,
                                         &data_type, &info, sizeof(info), &caps,
                                         NULL, errp)) {
        return false;
    }

    if (data_type != IOMMU_HW_INFO_TYPE_ARM_SMMUV3) {
        error_setg(errp, "Wrong data type (%d) for Host SMMUv3 device info",
                     data_type);
        return false;
    }

    if (!smmuv3_accel_check_hw_compatible(s, &info, errp)) {
        return false;
    }
    return true;
}

static SMMUv3AccelDevice *smmuv3_accel_get_dev(SMMUState *bs, SMMUPciBus *sbus,
                                               PCIBus *bus, int devfn)
{
    SMMUDevice *sdev = sbus->pbdev[devfn];
    SMMUv3AccelDevice *accel_dev;

    if (sdev) {
        return container_of(sdev, SMMUv3AccelDevice, sdev);
    }

    accel_dev = g_new0(SMMUv3AccelDevice, 1);
    sdev = &accel_dev->sdev;

    sbus->pbdev[devfn] = sdev;
    smmu_init_sdev(bs, sdev, bus, devfn);
    return accel_dev;
}

static uint32_t smmuv3_accel_gbpa_hwpt(SMMUv3State *s, SMMUv3AccelState *accel)
{
    return FIELD_EX32(s->gbpa, GBPA, ABORT) ?
           accel->abort_hwpt_id : accel->bypass_hwpt_id;
}

static bool
smmuv3_accel_alloc_vdev(SMMUv3AccelDevice *accel_dev, int sid, Error **errp)
{
    SMMUv3AccelState *accel = accel_dev->s_accel;
    HostIOMMUDeviceIOMMUFD *hiodi = accel_dev->hiodi;
    IOMMUFDVdev *vdev = accel_dev->vdev;
    uint32_t vdevice_id;

    if (!hiodi || vdev) {
        return true;
    }

    if (!iommufd_backend_alloc_vdev(hiodi->iommufd, hiodi->devid,
                                    accel->viommu->viommu_id, sid,
                                    &vdevice_id, errp)) {
            return false;
    }

    vdev = g_new(IOMMUFDVdev, 1);
    vdev->vdevice_id = vdevice_id;
    vdev->virt_id = sid;
    accel_dev->vdev = vdev;
    return true;
}

static SMMUS1Hwpt *
smmuv3_accel_dev_alloc_translate(SMMUv3AccelDevice *accel_dev, STE *ste,
                                 Error **errp)
{
    uint64_t ste_0 = (uint64_t)ste->word[0] | (uint64_t)ste->word[1] << 32;
    uint64_t ste_1 = (uint64_t)ste->word[2] | (uint64_t)ste->word[3] << 32;
    HostIOMMUDeviceIOMMUFD *hiodi = accel_dev->hiodi;
    SMMUv3AccelState *accel = accel_dev->s_accel;
    struct iommu_hwpt_arm_smmuv3 nested_data = {
        .ste = {
            cpu_to_le64(ste_0 & STE0_MASK),
            cpu_to_le64(ste_1 & STE1_MASK),
        },
    };
    uint32_t hwpt_id = 0, flags = 0;
    SMMUS1Hwpt *s1_hwpt;

    if (!iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid,
                                    accel->viommu->viommu_id, flags,
                                    IOMMU_HWPT_DATA_ARM_SMMUV3,
                                    sizeof(nested_data), &nested_data,
                                    &hwpt_id, errp)) {
            return NULL;
    }

    s1_hwpt = g_new0(SMMUS1Hwpt, 1);
    s1_hwpt->hwpt_id = hwpt_id;
    trace_smmuv3_accel_translate_ste(accel_dev->vdev->virt_id, hwpt_id,
                                     nested_data.ste[1], nested_data.ste[0]);
    return s1_hwpt;
}

bool smmuv3_accel_install_ste(SMMUv3State *s, SMMUDevice *sdev, int sid,
                              Error **errp)
{
    SMMUEventInfo event = {.type = SMMU_EVT_NONE, .sid = sid,
                           .inval_ste_allowed = true};
    SMMUv3AccelState *accel = s->s_accel;
    SMMUv3AccelDevice *accel_dev;
    HostIOMMUDeviceIOMMUFD *hiodi;
    uint32_t config, hwpt_id = 0;
    SMMUS1Hwpt *s1_hwpt = NULL;
    const char *type;
    STE ste;

    if (!accel || !accel->viommu) {
        return true;
    }

    accel_dev = container_of(sdev, SMMUv3AccelDevice, sdev);
    if (!accel_dev->s_accel) {
        return true;
    }

    hiodi = accel_dev->hiodi;
    if (!smmuv3_accel_alloc_vdev(accel_dev, sid, errp)) {
        return false;
    }

    if (smmu_find_ste(sdev->smmu, sid, &ste, &event)) {
        /* No STE found, nothing to install */
        return true;
    }

    /*
     * Install the STE based on SMMU enabled/config:
     * - attach a pre-allocated HWPT for abort/bypass
     * - or a new HWPT for translate STE
     *
     * Note: The vdev remains associated with accel_dev even if HWPT
     * attach/alloc fails, since the Guest–Host SID mapping stays
     * valid as long as the device is behind the accelerated SMMUv3.
     */
    if (!smmu_enabled(s)) {
        hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel);
    } else {
        config = STE_CONFIG(&ste);

        if (!STE_VALID(&ste) || STE_CFG_ABORT(config)) {
            hwpt_id = accel->abort_hwpt_id;
        } else if (STE_CFG_BYPASS(config)) {
            hwpt_id = accel->bypass_hwpt_id;
        } else if (STE_CFG_S1_TRANSLATE(config)) {
            s1_hwpt = smmuv3_accel_dev_alloc_translate(accel_dev, &ste, errp);
            if (!s1_hwpt) {
                return false;
            }
            hwpt_id = s1_hwpt->hwpt_id;
       }
    }

    if (!hwpt_id) {
        error_setg(errp, "Invalid STE config for sid 0x%x",
                   smmu_get_sid(&accel_dev->sdev));
        return false;
    }

    if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, hwpt_id,
                                               errp)) {
        if (s1_hwpt) {
            iommufd_backend_free_id(hiodi->iommufd, s1_hwpt->hwpt_id);
            g_free(s1_hwpt);
        }
        return false;
    }

    /* Free the previous s1_hwpt */
    if (accel_dev->s1_hwpt) {
        iommufd_backend_free_id(hiodi->iommufd, accel_dev->s1_hwpt->hwpt_id);
        g_free(accel_dev->s1_hwpt);
    }

    accel_dev->s1_hwpt = s1_hwpt;
    if (hwpt_id == accel->abort_hwpt_id) {
        type = "abort";
    } else if (hwpt_id == accel->bypass_hwpt_id) {
        type = "bypass";
    } else {
        type = "translate";
    }

    trace_smmuv3_accel_install_ste(sid, type, hwpt_id);
    return true;
}

bool smmuv3_accel_install_ste_range(SMMUv3State *s, SMMUSIDRange *range,
                                    Error **errp)
{
    SMMUv3AccelState *accel = s->s_accel;
    SMMUv3AccelDevice *accel_dev;
    Error *local_err = NULL;
    bool all_ok = true;

    if (!accel || !accel->viommu) {
        return true;
    }

    QLIST_FOREACH(accel_dev, &accel->device_list, next) {
        uint32_t sid = smmu_get_sid(&accel_dev->sdev);

        if (sid >= range->start && sid <= range->end) {
            if (!smmuv3_accel_install_ste(s, &accel_dev->sdev,
                                          sid, &local_err)) {
                error_append_hint(&local_err, "Device 0x%x: Failed to install "
                                  "STE\n", sid);
                error_report_err(local_err);
                local_err = NULL;
                all_ok = false;
            }
        }
    }

    if (!all_ok) {
        error_setg(errp, "Failed to install all STEs properly");
    }
    return all_ok;
}

/*
 * This issues the invalidation cmd to the host SMMUv3.
 *
 * sdev is non-NULL for SID based invalidations (e.g. CFGI_CD), and NULL for
 * non SID invalidations such as SMMU_CMD_TLBI_NH_ASID and SMMU_CMD_TLBI_NH_VA.
 */
bool smmuv3_accel_issue_inv_cmd(SMMUv3State *bs, void *cmd, SMMUDevice *sdev,
                                Error **errp)
{
    SMMUv3State *s = ARM_SMMUV3(bs);
    SMMUv3AccelState *accel = s->s_accel;
    uint32_t entry_num = 1;

    /*
     * No accel or viommu means no VFIO/IOMMUFD devices, nothing to
     * invalidate.
     */
    if (!accel || !accel->viommu) {
        return true;
    }

    /*
     * SID based invalidations (e.g. CFGI_CD) apply only to vfio-pci endpoints
     * with a valid vIOMMU vdev.
     */
    if (sdev && !container_of(sdev, SMMUv3AccelDevice, sdev)->vdev) {
        return true;
    }

    /* Single command (entry_num = 1); no need to check returned entry_num */
    return iommufd_backend_invalidate_cache(
                   accel->viommu->iommufd, accel->viommu->viommu_id,
                   IOMMU_VIOMMU_INVALIDATE_DATA_ARM_SMMUV3,
                   sizeof(Cmd), &entry_num, cmd, errp);
}

/*
 * Returns  0 on success (buf is populated and valid).
 * Returns  1 if the read should be retried (EAGAIN/EINTR).
 * Returns -1 on error with @errp set.
 */
int smmuv3_accel_event_read_validate(IOMMUFDVeventq *veventq, uint32_t type,
                                     void *buf, size_t size, Error **errp)
{
    uint32_t last_seq = veventq->last_event_seq;
    uint32_t id = veventq->veventq_id;
    struct iommufd_vevent_header *hdr;
    ssize_t bytes;

    bytes = read(veventq->veventq_fd, buf, size);
    if (bytes <= 0) {
        if (errno == EAGAIN || errno == EINTR) {
            return 1;
        }
        error_setg(errp, "vEVENTQ(type %u id %u): read failed (%m)", type, id);
        return -1;
    }
    hdr = (struct iommufd_vevent_header *)buf;
    if (bytes == sizeof(*hdr) &&
        (hdr->flags & IOMMU_VEVENTQ_FLAG_LOST_EVENTS)) {
        error_setg(errp, "vEVENTQ(type %u id %u): overflowed", type, id);
        veventq->event_start = false;
        return -1;
    }
    if (bytes < size) {
        error_setg(errp, "vEVENTQ(type %u id %u): short read(%zd/%zd bytes)",
                          type, id, bytes, size);
        return -1;
    }
    /* Check sequence in hdr for lost events if any */
    if (veventq->event_start && (hdr->sequence - last_seq != 1)) {
        warn_report("vEVENTQ(type %u id %u): lost %u event(s)",
                    type, id, hdr->sequence - last_seq - 1);
    }
    veventq->last_event_seq = hdr->sequence;
    veventq->event_start = true;
    return 0;
}

static void smmuv3_accel_event_read(void *opaque)
{
    SMMUv3State *s = opaque;
    IOMMUFDVeventq *veventq = s->s_accel->veventq;
    struct {
        struct iommufd_vevent_header hdr;
        struct iommu_vevent_arm_smmuv3 vevent;
    } buf;
    Error *local_err = NULL;
    int ret;

    ret = smmuv3_accel_event_read_validate(veventq,
                                           IOMMU_VEVENTQ_TYPE_ARM_SMMUV3, &buf,
                                           sizeof(buf), &local_err);
    if (ret < 0) {
        warn_report_err_once(local_err);
        return;
    }
    if (ret > 0) {
        return; /* EAGAIN/EINTR */
    }
    smmuv3_propagate_event(s, (Evt *)&buf.vevent);
}

static void smmuv3_accel_free_veventq(SMMUv3AccelState *accel)
{
    IOMMUFDVeventq *veventq = accel->veventq;

    if (!veventq) {
        return;
    }
    qemu_set_fd_handler(veventq->veventq_fd, NULL, NULL, NULL);
    close(veventq->veventq_fd);
    iommufd_backend_free_id(accel->viommu->iommufd, veventq->veventq_id);
    g_free(veventq);
    accel->veventq = NULL;
}

static void smmuv3_accel_free_viommu(SMMUv3AccelState *accel)
{
    IOMMUFDViommu *viommu = accel->viommu;

    if (!viommu) {
        return;
    }
    smmuv3_accel_free_veventq(accel);
    iommufd_backend_free_id(viommu->iommufd, accel->bypass_hwpt_id);
    iommufd_backend_free_id(viommu->iommufd, accel->abort_hwpt_id);
    iommufd_backend_free_id(viommu->iommufd, accel->viommu->viommu_id);
    g_free(viommu);
    accel->viommu = NULL;
}

bool smmuv3_accel_alloc_veventq(SMMUv3State *s, Error **errp)
{
    SMMUv3AccelState *accel = s->s_accel;
    IOMMUFDVeventq *veventq;
    uint32_t veventq_id;
    uint32_t veventq_fd;
    int flags;

    if (!accel || !accel->viommu) {
        return true;
    }

    if (accel->veventq) {
        return true;
    }

    if (!smmuv3_eventq_enabled(s)) {
        return true;
    }

    if (!iommufd_backend_alloc_veventq(accel->viommu->iommufd,
                                       accel->viommu->viommu_id,
                                       IOMMU_VEVENTQ_TYPE_ARM_SMMUV3,
                                       1 << s->eventq.log2size, &veventq_id,
                                       &veventq_fd, errp)) {
        return false;
    }

    flags = fcntl(veventq_fd, F_GETFL);
    if (flags < 0) {
        error_setg_errno(errp, errno, "Failed to get flags for vEVENTQ fd");
        goto free_veventq;
    }
    if (fcntl(veventq_fd, F_SETFL, flags | O_NONBLOCK) < 0) {
        error_setg_errno(errp, errno, "Failed to set O_NONBLOCK on vEVENTQ fd");
        goto free_veventq;
    }

    veventq = g_new0(IOMMUFDVeventq, 1);
    veventq->veventq_id = veventq_id;
    veventq->veventq_fd = veventq_fd;
    accel->veventq = veventq;

    /* Set up event handler for veventq fd */
    qemu_set_fd_handler(veventq_fd, smmuv3_accel_event_read, NULL, s);
    return true;

free_veventq:
    close(veventq_fd);
    iommufd_backend_free_id(accel->viommu->iommufd, veventq_id);
    return false;
}

static bool
smmuv3_accel_alloc_viommu(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *hiodi,
                          Error **errp)
{
    SMMUv3AccelState *accel = s->s_accel;
    const SMMUv3AccelCmdqvOps *cmdqv_ops = accel->cmdqv_ops;
    struct iommu_hwpt_arm_smmuv3 bypass_data = {
        .ste = { SMMU_STE_CFG_BYPASS | SMMU_STE_VALID, 0x0ULL },
    };
    struct iommu_hwpt_arm_smmuv3 abort_data = {
        .ste = { SMMU_STE_VALID, 0x0ULL },
    };
    uint32_t s2_hwpt_id = hiodi->hwpt_id;
    uint32_t viommu_id, hwpt_id;
    IOMMUFDViommu *viommu;

    if (cmdqv_ops) {
        if (!cmdqv_ops->alloc_viommu(s, hiodi, &viommu_id, errp)) {
            return false;
        }
    } else {
        if (!iommufd_backend_alloc_viommu(hiodi->iommufd, hiodi->devid,
                                          IOMMU_VIOMMU_TYPE_ARM_SMMUV3,
                                          s2_hwpt_id, NULL, 0, &viommu_id,
                                          errp)) {
            return false;
        }
    }

    viommu = g_new0(IOMMUFDViommu, 1);
    viommu->viommu_id = viommu_id;
    viommu->s2_hwpt_id = s2_hwpt_id;
    viommu->iommufd = hiodi->iommufd;
    accel->viommu = viommu;

    /*
     * Pre-allocate HWPTs for S1 bypass and abort cases. These will be attached
     * later for guest STEs or GBPAs that require bypass or abort configuration.
     */
    if (!iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid, viommu_id,
                                    0, IOMMU_HWPT_DATA_ARM_SMMUV3,
                                    sizeof(abort_data), &abort_data,
                                    &accel->abort_hwpt_id, errp)) {
        goto free_viommu;
    }

    if (!iommufd_backend_alloc_hwpt(hiodi->iommufd, hiodi->devid, viommu_id,
                                    0, IOMMU_HWPT_DATA_ARM_SMMUV3,
                                    sizeof(bypass_data), &bypass_data,
                                    &accel->bypass_hwpt_id, errp)) {
        goto free_abort_hwpt;
    }

    /* Allocate a vEVENTQ if guest has enabled event queue */
    if (!smmuv3_accel_alloc_veventq(s, errp)) {
        goto free_bypass_hwpt;
    }

    /* Attach a HWPT based on SMMUv3 GBPA.ABORT value */
    hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel);
    if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID, hwpt_id,
                                               errp)) {
        goto free_veventq;
    }
    return true;

free_veventq:
    smmuv3_accel_free_veventq(accel);
free_bypass_hwpt:
    iommufd_backend_free_id(hiodi->iommufd, accel->bypass_hwpt_id);
free_abort_hwpt:
    iommufd_backend_free_id(hiodi->iommufd, accel->abort_hwpt_id);
free_viommu:
    if (cmdqv_ops && cmdqv_ops->free_viommu) {
        cmdqv_ops->free_viommu(s);
    } else {
        iommufd_backend_free_id(hiodi->iommufd, viommu->viommu_id);
    }
    g_free(viommu);
    accel->viommu = NULL;
    return false;
}

static const SMMUv3AccelCmdqvOps *
smmuv3_accel_probe_cmdqv(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *idev,
                         Error **errp)
{
    const SMMUv3AccelCmdqvOps *ops = tegra241_cmdqv_get_ops();

    if (!ops) {
        error_setg(errp, "No CMDQV ops found");
        return NULL;
    }
    g_assert(ops->probe);
    g_assert(ops->alloc_viommu);

    if (!ops->probe(s, idev, errp)) {
        return NULL;
    }
    return ops;
}

static bool
smmuv3_accel_select_cmdqv(SMMUv3State *s, HostIOMMUDeviceIOMMUFD *idev,
                          Error **errp)
{
    const SMMUv3AccelCmdqvOps *ops = NULL;

    if (s->s_accel->cmdqv_ops) {
        return true;
    }

    switch (s->cmdqv) {
    case ON_OFF_AUTO_OFF:
        s->s_accel->cmdqv_ops = NULL;
        return true;
    case ON_OFF_AUTO_AUTO:
        ops = smmuv3_accel_probe_cmdqv(s, idev, NULL);
        break;
    case ON_OFF_AUTO_ON:
        ops = smmuv3_accel_probe_cmdqv(s, idev, errp);
        if (!ops) {
            error_append_hint(errp, "CMDQV requested but not supported");
            return false;
        }
        break;
    default:
        g_assert_not_reached();
    }

    if (ops && ops->init && !ops->init(s, errp)) {
        return false;
    }
    s->s_accel->cmdqv_ops = ops;
    return true;
}

static bool smmuv3_accel_set_iommu_device(PCIBus *bus, void *opaque, int devfn,
                                          HostIOMMUDevice *hiod, Error **errp)
{
    HostIOMMUDeviceIOMMUFD *hiodi = HOST_IOMMU_DEVICE_IOMMUFD(hiod);
    SMMUState *bs = opaque;
    SMMUv3State *s = ARM_SMMUV3(bs);
    SMMUPciBus *sbus = smmu_get_sbus(bs, bus);
    SMMUv3AccelDevice *accel_dev = smmuv3_accel_get_dev(bs, sbus, bus, devfn);

    if (!hiodi) {
        return true;
    }

    if (accel_dev->hiodi) {
        if (accel_dev->hiodi != hiodi) {
            error_setg(errp, "Device already has an associated hiodi 0x%x",
                       hiodi->devid);
            return false;
        }
        return true;
    }

    /*
     * Check the host SMMUv3 associated with the dev is compatible with the
     * QEMU SMMUv3 accel.
     */
    if (!smmuv3_accel_hw_compatible(s, hiodi, errp)) {
        return false;
    }

    if (s->s_accel->viommu) {
        goto done;
    }

    if (!smmuv3_accel_select_cmdqv(s, hiodi, errp)) {
        return false;
    }

    if (!smmuv3_accel_alloc_viommu(s, hiodi, errp)) {
        error_append_hint(errp, "Unable to alloc vIOMMU: hiodi devid 0x%x: ",
                          hiodi->devid);
        return false;
    }

    /*
     * CMDQV is active: block hot-unplug of the device that established the
     * viommu association. Removing it would cause the vIOMMU to host SMMUv3
     * association be changed via device hot-plug.
     */
    if (s->s_accel->cmdqv_ops) {
        PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn);
        error_setg(&accel_dev->unplug_blocker,
                   "CMDQV is active: removing the device that established the "
                   "viommu association would break the guest CMDQV");
        qdev_add_unplug_blocker(DEVICE(pdev), accel_dev->unplug_blocker);
    }
done:
    accel_dev->hiodi = hiodi;
    accel_dev->s_accel = s->s_accel;
    QLIST_INSERT_HEAD(&s->s_accel->device_list, accel_dev, next);
    trace_smmuv3_accel_set_iommu_device(devfn, hiodi->devid);
    return true;
}

static void smmuv3_accel_unset_iommu_device(PCIBus *bus, void *opaque,
                                            int devfn)
{
    SMMUState *bs = opaque;
    SMMUPciBus *sbus = g_hash_table_lookup(bs->smmu_pcibus_by_busptr, bus);
    HostIOMMUDeviceIOMMUFD *hiodi;
    SMMUv3AccelDevice *accel_dev;
    SMMUv3AccelState *accel;
    IOMMUFDVdev *vdev;
    SMMUDevice *sdev;

    if (!sbus) {
        return;
    }

    sdev = sbus->pbdev[devfn];
    if (!sdev) {
        return;
    }

    accel_dev = container_of(sdev, SMMUv3AccelDevice, sdev);
    hiodi = accel_dev->hiodi;
    accel = accel_dev->s_accel;
    /* Re-attach the default s2 hwpt id */
    if (!host_iommu_device_iommufd_attach_hwpt(hiodi, IOMMU_NO_PASID,
                                               hiodi->hwpt_id, NULL)) {
        error_report("Unable to attach the default HW pagetable: hiodi devid "
                     "0x%x", hiodi->devid);
    }

    if (accel_dev->s1_hwpt) {
        iommufd_backend_free_id(accel_dev->hiodi->iommufd,
                                accel_dev->s1_hwpt->hwpt_id);
        g_free(accel_dev->s1_hwpt);
        accel_dev->s1_hwpt = NULL;
    }

    vdev = accel_dev->vdev;
    if (vdev) {
        iommufd_backend_free_id(accel->viommu->iommufd, vdev->vdevice_id);
        g_free(vdev);
        accel_dev->vdev = NULL;
    }

    accel_dev->hiodi = NULL;
    accel_dev->s_accel = NULL;
    QLIST_REMOVE(accel_dev, next);
    trace_smmuv3_accel_unset_iommu_device(devfn, hiodi->devid);

    if (QLIST_EMPTY(&accel->device_list)) {
        smmuv3_accel_free_viommu(accel);
    }
}

static uint64_t smmuv3_accel_get_msi_gpa(PCIBus *bus, void *opaque, int devfn)
{
    SMMUState *bs = opaque;
    SMMUv3State *s = ARM_SMMUV3(bs);

    g_assert(s->msi_gpa);
    return s->msi_gpa;
}

/*
 * Only allow PCIe bridges, pxb-pcie roots, and GPEX roots so vfio-pci
 * endpoints can sit downstream. Accelerated SMMUv3 requires a vfio-pci
 * endpoint using the iommufd backend; all other device types are rejected.
 * This avoids supporting emulated endpoints, which would complicate IOTLB
 * invalidation and hurt performance.
 */
static bool smmuv3_accel_pdev_allowed(PCIDevice *pdev, bool *vfio_pci)
{

    if (object_dynamic_cast(OBJECT(pdev), TYPE_PCI_BRIDGE) ||
        object_dynamic_cast(OBJECT(pdev), TYPE_PXB_PCIE_DEV) ||
        object_dynamic_cast(OBJECT(pdev), TYPE_GPEX_ROOT_DEVICE)) {
        return true;
    } else if ((object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI))) {
        *vfio_pci = true;
        if (object_property_get_link(OBJECT(pdev), "iommufd", NULL)) {
            return true;
        }
    }
    return false;
}

static bool smmuv3_accel_supports_as(PCIBus *bus, void *opaque, int devfn,
                                     Error **errp)
{
    PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn);
    bool vfio_pci = false;

    if (pdev && !smmuv3_accel_pdev_allowed(pdev, &vfio_pci)) {
        if (vfio_pci) {
            error_setg(errp, "vfio-pci endpoint devices without an iommufd "
                       "backend not allowed when using arm-smmuv3,accel=on");

        } else {
            error_setg(errp, "Emulated endpoint devices are not allowed when "
                       "using arm-smmuv3,accel=on");
        }
        return false;
    }
    return true;
}
/*
 * Find or add an address space for the given PCI device.
 *
 * If a device matching @bus and @devfn already exists, return its
 * corresponding address space. Otherwise, create a new device entry
 * and initialize address space for it.
 */
static AddressSpace *smmuv3_accel_find_add_as(PCIBus *bus, void *opaque,
                                              int devfn)
{
    PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn);
    SMMUState *bs = opaque;
    SMMUPciBus *sbus = smmu_get_sbus(bs, bus);
    SMMUv3AccelDevice *accel_dev = smmuv3_accel_get_dev(bs, sbus, bus, devfn);
    SMMUDevice *sdev = &accel_dev->sdev;
    bool vfio_pci = false;

    if (pdev && !smmuv3_accel_pdev_allowed(pdev, &vfio_pci)) {
        /* Should never be here: supports_address_space() filters these out */
        g_assert_not_reached();
    }

    /*
     * In the accelerated mode, a vfio-pci device attached via the iommufd
     * backend must remain in the system address space. Such a device is
     * always translated by its physical SMMU (using either a stage-2-only
     * STE or a nested STE), where the parent stage-2 page table is allocated
     * by the VFIO core to back the system address space.
     *
     * Return the shared_as_sysmem aliased to the global system memory in this
     * case. Sharing address_space_memory also allows devices under different
     * vSMMU instances in the same VM to reuse a single nesting parent HWPT in
     * the VFIO core.
     *
     * For non-endpoint emulated devices such as PCIe root ports and bridges,
     * which may use the normal emulated translation path and software IOTLBs,
     * return the SMMU's IOMMU address space.
     */
    if (vfio_pci) {
        return shared_as_sysmem;
    } else {
        return &sdev->as;
    }
}

static inline bool smmuv3_pasid_supported(SMMUv3State *s)
{
    return s->ssidsize > SSID_SIZE_MODE_0 ||
           (s->ssidsize == SSID_SIZE_MODE_AUTO &&
            FIELD_EX32(s->idr[1], IDR1, SSIDSIZE));
}

static uint64_t smmuv3_accel_get_viommu_flags(void *opaque)
{
    /*
     * We return VIOMMU_FLAG_WANT_NESTING_PARENT to inform VFIO core to create a
     * nesting parent which is required for accelerated SMMUv3 support.
     * The real HW nested support should be reported from host SMMUv3 and if
     * it doesn't, the nesting parent allocation will fail anyway in VFIO core.
     */
    uint64_t flags = VIOMMU_FLAG_WANT_NESTING_PARENT;
    SMMUState *bs = opaque;
    SMMUv3State *s = ARM_SMMUV3(bs);

    if (smmuv3_pasid_supported(s)) {
        flags |= VIOMMU_FLAG_PASID_SUPPORTED;
    }
    return flags;
}

static const PCIIOMMUOps smmuv3_accel_ops = {
    .supports_address_space = smmuv3_accel_supports_as,
    .get_address_space = smmuv3_accel_find_add_as,
    .get_viommu_flags = smmuv3_accel_get_viommu_flags,
    .set_iommu_device = smmuv3_accel_set_iommu_device,
    .unset_iommu_device = smmuv3_accel_unset_iommu_device,
    .get_msi_direct_gpa = smmuv3_accel_get_msi_gpa,
};

/*
 * This returns the value of a SsidSizeMode value offset by 1 to
 * account for the enum values offset by 1 from actual values.
 *
 * SSID_SIZE_MODE_0 = 1, SSID_SIZE_MODE_1 = 2, etc. so return 0
 * if SSID_SIZE_MODE_0 is passed as input, return 1 if
 * SSID_SIZE_MODE_1 is passed as input, etc.
 */
static uint8_t ssidsize_mode_to_value(SsidSizeMode mode)
{
    if (mode == SSID_SIZE_MODE_AUTO) {
        return 0;
    }
    return mode - 1;
}

void smmuv3_accel_idr_override(SMMUv3State *s)
{
    if (!s->accel) {
        return;
    }

    /* Only override RIL if user explicitly set OFF */
    if (s->ril == ON_OFF_AUTO_OFF) {
        s->idr[3] = FIELD_DP32(s->idr[3], IDR3, RIL, 0);
    }

    /* QEMU SMMUv3 has no ATS. Advertise ATS if opt-in by property */
    if (s->ats == ON_OFF_AUTO_ON) {
        s->idr[0] = FIELD_DP32(s->idr[0], IDR0, ATS, 1);
    }

    /* Advertise 48-bit OAS in IDR5 when requested (default is 44 bits). */
    if (s->oas == OAS_MODE_48) {
        s->idr[5] = FIELD_DP32(s->idr[5], IDR5, OAS, SMMU_IDR5_OAS_48);
    }

    /*
     * By default QEMU SMMUv3 has no SubstreamID support. Update IDR1 if user
     * has enabled it.
     */
    if (s->ssidsize > SSID_SIZE_MODE_0) {
        s->idr[1] = FIELD_DP32(s->idr[1], IDR1, SSIDSIZE,
                               ssidsize_mode_to_value(s->ssidsize));
    }
}

/* Based on SMUUv3 GPBA.ABORT configuration, attach a corresponding HWPT */
bool smmuv3_accel_attach_gbpa_hwpt(SMMUv3State *s, Error **errp)
{
    SMMUv3AccelState *accel = s->s_accel;
    SMMUv3AccelDevice *accel_dev;
    Error *local_err = NULL;
    bool all_ok = true;
    uint32_t hwpt_id;

    if (!accel || !accel->viommu) {
        return true;
    }

    hwpt_id = smmuv3_accel_gbpa_hwpt(s, accel);
    QLIST_FOREACH(accel_dev, &accel->device_list, next) {
        if (!host_iommu_device_iommufd_attach_hwpt(accel_dev->hiodi,
                                                   IOMMU_NO_PASID, hwpt_id,
                                                   &local_err)) {
            error_append_hint(&local_err, "Failed to attach GBPA hwpt %u for "
                              "hiodi devid %u", hwpt_id,
                               accel_dev->hiodi->devid);
            error_report_err(local_err);
            local_err = NULL;
            all_ok = false;
        }
    }
    if (!all_ok) {
        error_setg(errp, "Failed to attach all GBPA based HWPTs properly");
    }
    return all_ok;
}

void smmuv3_accel_reset(SMMUv3State *s)
{
    SMMUv3AccelState *accel = s->s_accel;

    if (!accel) {
        return;
    }
    /* Attach a HWPT based on GBPA reset value */
    smmuv3_accel_attach_gbpa_hwpt(s, NULL);

    if (accel->cmdqv_ops && accel->cmdqv_ops->reset) {
        accel->cmdqv_ops->reset(s);
    }
}

static void smmuv3_accel_as_init(SMMUv3State *s)
{

    if (shared_as_sysmem) {
        return;
    }

    memory_region_init(&root, OBJECT(s), "root", UINT64_MAX);
    memory_region_init_alias(&sysmem, OBJECT(s), "smmuv3-accel-sysmem",
                             get_system_memory(), 0,
                             memory_region_size(get_system_memory()));
    memory_region_add_subregion(&root, 0, &sysmem);

    shared_as_sysmem = g_new0(AddressSpace, 1);
    address_space_init(shared_as_sysmem, &root, "smmuv3-accel-as-sysmem");
}

SMMUv3AccelCmdqvType smmuv3_accel_cmdqv_type(Object *obj)
{
    SMMUv3State *s = ARM_SMMUV3(obj);
    SMMUv3AccelState *accel = s->s_accel;

    if (!accel || !accel->cmdqv_ops || !accel->cmdqv_ops->get_type) {
        return SMMUV3_CMDQV_NONE;
    }

    return accel->cmdqv_ops->get_type();
}

static void smmuv3_accel_machine_done(Notifier *notifier, void *data)
{
    SMMUv3State *s = container_of(notifier, SMMUv3State, machine_done);
    SMMUv3AccelState *accel = s->s_accel;

    if (accel->auto_mode && !accel->auto_finalised) {
        error_report("arm-smmuv3 accel=on with 'auto' properties requires "
                     "at least one cold-plugged VFIO device");
        exit(1);
    }

    if (s->cmdqv == ON_OFF_AUTO_ON && !accel->cmdqv) {
        error_report("arm-smmuv3 cmdqv=on requires at least one cold-plugged "
                     "VFIO device");
        exit(1);
    }
}

bool smmuv3_accel_init(SMMUv3State *s, Error **errp)
{
    SMMUState *bs = ARM_SMMU(s);

    s->s_accel = g_new0(SMMUv3AccelState, 1);
    bs->iommu_ops = &smmuv3_accel_ops;
    smmuv3_accel_as_init(s);

    if (s->ats == ON_OFF_AUTO_AUTO ||
        s->ril == ON_OFF_AUTO_AUTO ||
        s->ssidsize == SSID_SIZE_MODE_AUTO ||
        s->oas == OAS_MODE_AUTO) {
        s->s_accel->auto_mode = true;
    }

    if (s->s_accel->auto_mode) {
        s->machine_done.notify = smmuv3_accel_machine_done;
        qemu_add_machine_init_done_notifier(&s->machine_done);
    }

    return true;
}
