qemu/util/aio-posix.c

/*
 * QEMU aio implementation
 *
 * Copyright IBM, Corp. 2008
 *
 * Authors:
 *  Anthony Liguori   <aliguori@us.ibm.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
 */

#include "qemu/osdep.h"
#include "block/block.h"
#include "block/thread-pool.h"
#include "qapi/error.h"
#include "qemu/main-loop.h"
#include "qemu/lockcnt.h"
#include "qemu/rcu.h"
#include "qemu/rcu_queue.h"
#include "qemu/sockets.h"
#include "qemu/cutils.h"
#include "system/iothread.h"
#include "trace.h"
#include "aio-posix.h"

/* Stop userspace polling on a handler if it isn't active for some time */
#define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND)
#define POLL_WEIGHT_SHIFT   (3)

static void update_handler_poll_times(AioContext *ctx, int64_t block_ns,
                                      int64_t dispatch_time);
static void adjust_polling_time(AioContext *ctx, int64_t block_ns);

bool aio_poll_disabled(AioContext *ctx)
{
    return qatomic_read(&ctx->poll_disable_cnt);
}

void aio_add_ready_handler(AioHandlerList *ready_list,
                           AioHandler *node,
                           int revents)
{
    QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
    node->pfd.revents = revents;
    QLIST_INSERT_HEAD(ready_list, node, node_ready);
}

static void aio_add_poll_ready_handler(AioHandlerList *ready_list,
                                       AioHandler *node)
{
    QLIST_SAFE_REMOVE(node, node_ready); /* remove from nested parent's list */
    node->poll_ready = true;
    QLIST_INSERT_HEAD(ready_list, node, node_ready);
}

static AioHandler *find_aio_handler(AioContext *ctx, int fd)
{
    AioHandler *node;

    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
        if (node->pfd.fd == fd) {
            if (!QLIST_IS_INSERTED(node, node_deleted)) {
                return node;
            }
        }
    }

    return NULL;
}

static bool aio_remove_fd_handler(AioContext *ctx, AioHandler *node)
{
    node->pfd.revents = 0;
    node->poll_ready = false;

    /* If the fd monitor has already marked it deleted, leave it alone */
    if (QLIST_IS_INSERTED(node, node_deleted)) {
        return false;
    }

    /* If a read is in progress, just mark the node as deleted */
    if (qemu_lockcnt_count(&ctx->list_lock)) {
        QLIST_INSERT_HEAD_RCU(&ctx->deleted_aio_handlers, node, node_deleted);
        return false;
    }
    /* Otherwise, delete it for real.  We can't just mark it as
     * deleted because deleted nodes are only cleaned up while
     * no one is walking the handlers list.
     */
    QLIST_SAFE_REMOVE(node, node_poll);
    QLIST_REMOVE(node, node);
    return true;
}

void aio_set_fd_handler(AioContext *ctx,
                        int fd,
                        IOHandler *io_read,
                        IOHandler *io_write,
                        AioPollFn *io_poll,
                        IOHandler *io_poll_ready,
                        void *opaque)
{
    AioHandler *node;
    AioHandler *new_node = NULL;
    bool is_new = false;
    bool deleted = false;
    int poll_disable_change;

    if (io_poll && !io_poll_ready) {
        io_poll = NULL; /* polling only makes sense if there is a handler */
    }

    qemu_lockcnt_lock(&ctx->list_lock);

    node = find_aio_handler(ctx, fd);

    /* Are we deleting the fd handler? */
    if (!io_read && !io_write && !io_poll) {
        if (node == NULL) {
            qemu_lockcnt_unlock(&ctx->list_lock);
            return;
        }
        /* Clean events in order to unregister fd from the ctx epoll. */
        node->pfd.events = 0;

        poll_disable_change = -!node->io_poll;
    } else {
        poll_disable_change = !io_poll - (node && !node->io_poll);
        if (node == NULL) {
            is_new = true;
        }
        /* Alloc and insert if it's not already there */
        new_node = g_new0(AioHandler, 1);

        /* Update handler with latest information */
        new_node->io_read = io_read;
        new_node->io_write = io_write;
        new_node->io_poll = io_poll;
        new_node->io_poll_ready = io_poll_ready;
        new_node->opaque = opaque;

        if (is_new) {
            new_node->pfd.fd = fd;
        } else {
            new_node->pfd = node->pfd;
        }

        new_node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
        new_node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);

        QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, new_node, node);
    }

    /* No need to order poll_disable_cnt writes against other updates;
     * the counter is only used to avoid wasting time and latency on
     * iterated polling when the system call will be ultimately necessary.
     * Changing handlers is a rare event, and a little wasted polling until
     * the aio_notify below is not an issue.
     */
    qatomic_set(&ctx->poll_disable_cnt,
               qatomic_read(&ctx->poll_disable_cnt) + poll_disable_change);

    ctx->fdmon_ops->update(ctx, node, new_node);
    if (node) {
        deleted = aio_remove_fd_handler(ctx, node);
    }
    qemu_lockcnt_unlock(&ctx->list_lock);
    aio_notify(ctx);

    if (deleted) {
        g_free(node);
    }
}

static void aio_set_fd_poll(AioContext *ctx, int fd,
                            IOHandler *io_poll_begin,
                            IOHandler *io_poll_end)
{
    AioHandler *node = find_aio_handler(ctx, fd);

    if (!node) {
        return;
    }

    node->io_poll_begin = io_poll_begin;
    node->io_poll_end = io_poll_end;
}

void aio_set_event_notifier(AioContext *ctx,
                            EventNotifier *notifier,
                            EventNotifierHandler *io_read,
                            AioPollFn *io_poll,
                            EventNotifierHandler *io_poll_ready)
{
    aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
                       (IOHandler *)io_read, NULL, io_poll,
                       (IOHandler *)io_poll_ready, notifier);
}

void aio_set_event_notifier_poll(AioContext *ctx,
                                 EventNotifier *notifier,
                                 EventNotifierHandler *io_poll_begin,
                                 EventNotifierHandler *io_poll_end)
{
    aio_set_fd_poll(ctx, event_notifier_get_fd(notifier),
                    (IOHandler *)io_poll_begin,
                    (IOHandler *)io_poll_end);
}

static bool poll_set_started(AioContext *ctx, AioHandlerList *ready_list,
                             bool started)
{
    AioHandler *node;
    bool progress = false;

    if (started == ctx->poll_started) {
        return false;
    }

    ctx->poll_started = started;

    qemu_lockcnt_inc(&ctx->list_lock);
    QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) {
        IOHandler *fn;

        if (QLIST_IS_INSERTED(node, node_deleted)) {
            continue;
        }

        if (started) {
            fn = node->io_poll_begin;
        } else {
            fn = node->io_poll_end;
        }

        if (fn) {
            fn(node->opaque);
        }

        /* Poll one last time in case ->io_poll_end() raced with the event */
        if (!started && node->io_poll(node->opaque)) {
            aio_add_poll_ready_handler(ready_list, node);
            progress = true;
        }
    }
    qemu_lockcnt_dec(&ctx->list_lock);

    return progress;
}


bool aio_prepare(AioContext *ctx)
{
    AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);

    /* Poll mode cannot be used with glib's event loop, disable it. */
    poll_set_started(ctx, &ready_list, false);
    /* TODO what to do with this list? */

    ctx->fdmon_ops->gsource_prepare(ctx);
    return false;
}

bool aio_pending(AioContext *ctx)
{
    return ctx->fdmon_ops->gsource_check(ctx);
}

static void aio_free_deleted_handlers(AioContext *ctx)
{
    AioHandler *node;

    if (QLIST_EMPTY_RCU(&ctx->deleted_aio_handlers)) {
        return;
    }
    if (!qemu_lockcnt_dec_if_lock(&ctx->list_lock)) {
        return; /* we are nested, let the parent do the freeing */
    }

    while ((node = QLIST_FIRST_RCU(&ctx->deleted_aio_handlers))) {
        QLIST_REMOVE(node, node);
        QLIST_REMOVE(node, node_deleted);
        QLIST_SAFE_REMOVE(node, node_poll);
        g_free(node);
    }

    qemu_lockcnt_inc_and_unlock(&ctx->list_lock);
}

static bool aio_dispatch_handler(AioContext *ctx, AioHandler *node)
{
    bool progress = false;
    bool poll_ready;
    int revents;

    revents = node->pfd.revents & node->pfd.events;
    node->pfd.revents = 0;

    poll_ready = node->poll_ready;
    node->poll_ready = false;

    /*
     * Start polling AioHandlers when they become ready because activity is
     * likely to continue.  Note that starvation is theoretically possible when
     * fdmon_supports_polling(), but only until the fd fires for the first
     * time.
     */
    if (ctx->poll_max_ns && !QLIST_IS_INSERTED(node, node_deleted) &&
        !QLIST_IS_INSERTED(node, node_poll) && node->io_poll) {
        trace_poll_add(ctx, node, node->pfd.fd, revents);
        if (ctx->poll_started && node->io_poll_begin) {
            node->io_poll_begin(node->opaque);
        }
        QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
    }
    if (!QLIST_IS_INSERTED(node, node_deleted) &&
        poll_ready && revents == 0 && node->io_poll_ready) {
        /*
         * Remove temporarily to avoid infinite loops when ->io_poll_ready()
         * calls aio_poll() before clearing the condition that made the poll
         * handler become ready.
         */
        QLIST_SAFE_REMOVE(node, node_poll);

        node->io_poll_ready(node->opaque);

        if (!QLIST_IS_INSERTED(node, node_poll)) {
            QLIST_INSERT_HEAD(&ctx->poll_aio_handlers, node, node_poll);
        }

        /*
         * Return early since revents was zero. aio_notify() does not count as
         * progress.
         */
        return node->opaque != &ctx->notifier;
    }

    if (!QLIST_IS_INSERTED(node, node_deleted) &&
        (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
        node->io_read) {
        node->io_read(node->opaque);

        /* aio_notify() does not count as progress */
        if (node->opaque != &ctx->notifier) {
            progress = true;
        }
    }
    if (!QLIST_IS_INSERTED(node, node_deleted) &&
        (revents & (G_IO_OUT | G_IO_ERR)) &&
        node->io_write) {
        node->io_write(node->opaque);
        progress = true;
    }

    return progress;
}

static bool aio_dispatch_ready_handlers(AioContext *ctx,
                                        AioHandlerList *ready_list,
                                        int64_t dispatch_time)
{
    bool progress = false;
    AioHandler *node;

    while ((node = QLIST_FIRST(ready_list))) {
        QLIST_REMOVE(node, node_ready);
        progress = aio_dispatch_handler(ctx, node) || progress;

        /*
         * Update last_dispatch_timestamp to mark this as an active
         * handler for polling time adjustment and prevent idle removal.
         */
        if (ctx->poll_max_ns && QLIST_IS_INSERTED(node, node_poll)) {
            node->last_dispatch_timestamp = dispatch_time;
        }
    }

    return progress;
}

void aio_dispatch(AioContext *ctx)
{
    AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);

    qemu_lockcnt_inc(&ctx->list_lock);

    aio_bh_poll(ctx);

    ctx->fdmon_ops->gsource_dispatch(ctx, &ready_list);

    if (ctx->fdmon_ops->dispatch) {
        ctx->fdmon_ops->dispatch(ctx);
    }

    /* Set now to 0 as polling is disabled in the glib event loop */
    aio_dispatch_ready_handlers(ctx, &ready_list, 0);

    aio_free_deleted_handlers(ctx);
    qemu_lockcnt_dec(&ctx->list_lock);

    timerlistgroup_run_timers(&ctx->tlg);
}

static bool run_poll_handlers_once(AioContext *ctx,
                                   AioHandlerList *ready_list,
                                   int64_t now,
                                   int64_t *timeout)
{
    bool progress = false;
    AioHandler *node;
    AioHandler *tmp;

    QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
        if (node->io_poll(node->opaque)) {
            aio_add_poll_ready_handler(ready_list, node);
            /*
             * Polling was successful, exit try_poll_mode immediately
             * to adjust the next polling time.
             */
            *timeout = 0;
            if (node->opaque != &ctx->notifier) {
                progress = true;
            }
        }

        /* Caller handles freeing deleted nodes.  Don't do it here. */
    }

    return progress;
}

static bool fdmon_supports_polling(AioContext *ctx)
{
    return ctx->fdmon_ops->need_wait != aio_poll_disabled;
}

static bool remove_idle_poll_handlers(AioContext *ctx,
                                      AioHandlerList *ready_list,
                                      int64_t now)
{
    AioHandler *node;
    AioHandler *tmp;
    bool progress = false;

    /*
     * File descriptor monitoring implementations without userspace polling
     * support suffer from starvation when a subset of handlers is polled
     * because fds will not be processed in a timely fashion.  Don't remove
     * idle poll handlers.
     */
    if (!fdmon_supports_polling(ctx)) {
        return false;
    }

    QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) {
        if (node->poll_ready == false &&
            now >= node->last_dispatch_timestamp + POLL_IDLE_INTERVAL_NS) {
            trace_poll_remove(ctx, node, node->pfd.fd);
            node->last_dispatch_timestamp = 0LL;
            QLIST_SAFE_REMOVE(node, node_poll);
            if (ctx->poll_started && node->io_poll_end) {
                node->io_poll_end(node->opaque);

                /*
                 * Final poll in case ->io_poll_end() races with an event.
                 * Nevermind about re-adding the handler in the rare case where
                 * this causes progress.
                 */
                if (node->io_poll(node->opaque)) {
                    aio_add_poll_ready_handler(ready_list, node);
                    progress = true;
                }
            }
        }
    }

    return progress;
}

/* run_poll_handlers:
 * @ctx: the AioContext
 * @ready_list: the list to place ready handlers on
 * @max_ns: maximum time to poll for, in nanoseconds
 *
 * Polls for a given time.
 *
 * Note that the caller must have incremented ctx->list_lock.
 *
 * Returns: true if progress was made, false otherwise
 */
static bool run_poll_handlers(AioContext *ctx, AioHandlerList *ready_list,
                              int64_t max_ns, int64_t *timeout)
{
    bool progress;
    int64_t start_time, elapsed_time;

    assert(qemu_lockcnt_count(&ctx->list_lock) > 0);

    trace_run_poll_handlers_begin(ctx, max_ns, *timeout);

    /*
     * Optimization: ->io_poll() handlers often contain RCU read critical
     * sections and we therefore see many rcu_read_lock() -> rcu_read_unlock()
     * -> rcu_read_lock() -> ... sequences with expensive memory
     * synchronization primitives.  Make the entire polling loop an RCU
     * critical section because nested rcu_read_lock()/rcu_read_unlock() calls
     * are cheap.
     */
    RCU_READ_LOCK_GUARD();

    start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
    do {
        progress = run_poll_handlers_once(ctx, ready_list,
                                          start_time, timeout);
        elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time;
        max_ns = qemu_soonest_timeout(*timeout, max_ns);
        assert(!(max_ns && progress));

        if (ctx->fdmon_ops->need_wait(ctx)) {
            if (fdmon_supports_polling(ctx)) {
                *timeout = 0; /* stay in polling mode */
            }
            break;
        }
    } while (elapsed_time < max_ns);

    if (remove_idle_poll_handlers(ctx, ready_list,
                                  start_time + elapsed_time)) {
        *timeout = 0;
        progress = true;
    }

    /* If time has passed with no successful polling, adjust *timeout to
     * keep the same ending time.
     */
    if (*timeout != -1) {
        *timeout -= MIN(*timeout, elapsed_time);
    }

    trace_run_poll_handlers_end(ctx, progress, *timeout);
    return progress;
}

/* try_poll_mode:
 * @ctx: the AioContext
 * @ready_list: list to add handlers that need to be run
 * @timeout: timeout for blocking wait, computed by the caller and updated if
 *    polling succeeds.
 *
 * Note that the caller must have incremented ctx->list_lock.
 *
 * Returns: true if progress was made, false otherwise
 */
static bool try_poll_mode(AioContext *ctx, AioHandlerList *ready_list,
                          int64_t *timeout)
{
    int64_t max_ns;

    if (QLIST_EMPTY_RCU(&ctx->poll_aio_handlers)) {
        return false;
    }

    max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns);

    if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) {
        /*
         * Enable poll mode. It pairs with the poll_set_started() in
         * aio_poll() which disables poll mode.
         */
        poll_set_started(ctx, ready_list, true);

        if (run_poll_handlers(ctx, ready_list, max_ns, timeout)) {
            return true;
        }
    }
    return false;
}

static void adjust_polling_time(AioContext *ctx, int64_t block_ns)
{
    if (block_ns < ctx->poll_ns) {
        int64_t old = ctx->poll_ns;
        int64_t shrink = ctx->poll_shrink;

        if (shrink == 0) {
            shrink = 2;
        }

        if (block_ns < (ctx->poll_ns / shrink)) {
            ctx->poll_ns /= shrink;
        }

        trace_poll_shrink(ctx, old, ctx->poll_ns);
    } else if (block_ns > ctx->poll_ns) {
        /* There is room to grow, poll longer */
        int64_t old = ctx->poll_ns;
        int64_t grow = ctx->poll_grow;

        if (grow == 0) {
            grow = 2;
        }

        if (block_ns > ctx->poll_ns * grow) {
            ctx->poll_ns = block_ns;
        } else {
            ctx->poll_ns *= grow;
        }

        if (ctx->poll_ns > ctx->poll_max_ns) {
            ctx->poll_ns = ctx->poll_max_ns;
        }

        trace_poll_grow(ctx, old, ctx->poll_ns);
    }
}

static void update_handler_poll_times(AioContext *ctx, int64_t block_ns,
                                      int64_t dispatch_time)
{
    AioHandler *node;
    int64_t max_poll_ns = -1;

    QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) {
        if (node->last_dispatch_timestamp == dispatch_time) {
            /*
             * Active handler: had an event in this aio_poll() call.
             * Update poll.ns using a weighted average of the current
             * block_ns and previous poll.ns to smooth adjustments.
             */
            node->poll.ns = node->poll.ns
                ? (node->poll.ns - (node->poll.ns >> POLL_WEIGHT_SHIFT))
                + (block_ns >> POLL_WEIGHT_SHIFT) : block_ns;

            if (node->poll.ns > ctx->poll_max_ns) {
                node->poll.ns = 0;
            }
            /*
             * Track the maximum poll.ns among active handlers to
             * calculate the next polling time.
             */
            max_poll_ns = MAX(max_poll_ns, node->poll.ns);
        } else {
            /*
             * Inactive handler: no event in this aio_poll() call but
             * was active before. Increase poll.ns by block_ns. If it
             * exceeds poll_max_ns, reset to 0 until next event.
             */
            if (node->poll.ns != 0) {
                node->poll.ns += block_ns;
                if (node->poll.ns > ctx->poll_max_ns) {
                    node->poll.ns = 0;
                }
            }
        }
    }
    if (max_poll_ns >= 0) {
        adjust_polling_time(ctx, max_poll_ns);
    }
}

bool aio_poll(AioContext *ctx, bool blocking)
{
    AioHandlerList ready_list = QLIST_HEAD_INITIALIZER(ready_list);
    bool progress = false;
    bool use_notify_me;
    int64_t timeout;
    int64_t start = 0;
    int64_t block_ns = 0;
    int64_t dispatch_ns = 0;

    /*
     * There cannot be two concurrent aio_poll calls for the same AioContext (or
     * an aio_poll concurrent with a GSource prepare/check/dispatch callback).
     * We rely on this below to avoid slow locked accesses to ctx->notify_me.
     *
     * aio_poll() may only be called in the AioContext's thread. iohandler_ctx
     * is special in that it runs in the main thread, but that thread's context
     * is qemu_aio_context.
     */
    assert(in_aio_context_home_thread(ctx == iohandler_get_aio_context() ?
                                      qemu_get_aio_context() : ctx));

    qemu_lockcnt_inc(&ctx->list_lock);

    if (ctx->poll_max_ns) {
        start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
    }

    timeout = blocking ? aio_compute_timeout(ctx) : 0;
    if (ctx->poll_max_ns != 0) {
        progress = try_poll_mode(ctx, &ready_list, &timeout);
    }
    assert(!(timeout && progress));

    /*
     * aio_notify can avoid the expensive event_notifier_set if
     * everything (file descriptors, bottom halves, timers) will
     * be re-evaluated before the next blocking poll().  This is
     * already true when aio_poll is called with blocking == false;
     * if blocking == true, it is only true after poll() returns,
     * so disable the optimization now.
     */
    use_notify_me = timeout != 0;
    if (use_notify_me) {
        qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) + 2);
        /*
         * Write ctx->notify_me before reading ctx->notified.  Pairs with
         * smp_mb in aio_notify().
         */
        smp_mb();

        /* Don't block if aio_notify() was called */
        if (qatomic_read(&ctx->notified)) {
            timeout = 0;
        }
    }

    /* If polling is allowed, non-blocking aio_poll does not need the
     * system call---a single round of run_poll_handlers_once suffices.
     */
    if (timeout || ctx->fdmon_ops->need_wait(ctx)) {
        /*
         * Disable poll mode. poll mode should be disabled before the call
         * of ctx->fdmon_ops->wait() so that guest's notification can wake
         * up IO threads when some work becomes pending. It is essential to
         * avoid hangs or unnecessary latency.
         */
        if (timeout && poll_set_started(ctx, &ready_list, false)) {
            timeout = 0;
            progress = true;
        }

        ctx->fdmon_ops->wait(ctx, &ready_list, timeout);
    }

    if (use_notify_me) {
        /* Finish the poll before clearing the flag.  */
        qatomic_store_release(&ctx->notify_me,
                             qatomic_read(&ctx->notify_me) - 2);
    }

    aio_notify_accept(ctx);

    /* Calculate blocked time for adaptive polling */
    if (ctx->poll_max_ns) {
        dispatch_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
        block_ns = dispatch_ns - start;
    }

    if (ctx->fdmon_ops->dispatch) {
        progress |= ctx->fdmon_ops->dispatch(ctx);
    }

    progress |= aio_bh_poll(ctx);
    progress |= aio_dispatch_ready_handlers(ctx, &ready_list, dispatch_ns);

    aio_free_deleted_handlers(ctx);

    if (ctx->poll_max_ns) {
        update_handler_poll_times(ctx, block_ns, dispatch_ns);
    }

    qemu_lockcnt_dec(&ctx->list_lock);

    progress |= timerlistgroup_run_timers(&ctx->tlg);

    return progress;
}

bool aio_context_setup(AioContext *ctx, Error **errp)
{
    ctx->fdmon_ops = &fdmon_poll_ops;
    ctx->epollfd = -1;
    ctx->epollfd_tag = NULL;

#ifdef CONFIG_LINUX_IO_URING
    {
        static bool need_io_uring;
        Error *local_err = NULL; /* ERRP_GUARD() doesn't handle error_abort */

        /* io_uring takes precedence because it provides aio_add_sqe() support */
        if (fdmon_io_uring_setup(ctx, &local_err)) {
            /*
             * If one AioContext gets io_uring, then all AioContexts need io_uring
             * so that aio_add_sqe() support is available across all threads.
             */
            need_io_uring = true;
            return true;
        }
        if (need_io_uring) {
            error_propagate(errp, local_err);
            return false;
        }

        /* Silently fall back on systems where io_uring is unavailable */
        error_free(local_err);
    }
#endif /* CONFIG_LINUX_IO_URING */

    fdmon_epoll_setup(ctx);
    return true;
}

void aio_context_destroy(AioContext *ctx)
{
#ifdef CONFIG_LINUX_IO_URING
    fdmon_io_uring_destroy(ctx);
#endif

    qemu_lockcnt_lock(&ctx->list_lock);
    fdmon_epoll_disable(ctx);
    qemu_lockcnt_unlock(&ctx->list_lock);

    aio_free_deleted_handlers(ctx);
}

void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns,
                                 int64_t grow, int64_t shrink, Error **errp)
{
    AioHandler *node;

    qemu_lockcnt_inc(&ctx->list_lock);
    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
        node->poll.ns = 0;
    }
    qemu_lockcnt_dec(&ctx->list_lock);

    /* No thread synchronization here, it doesn't matter if an incorrect value
     * is used once.
     */
    ctx->poll_max_ns = max_ns;
    ctx->poll_grow = grow;
    ctx->poll_shrink = shrink;
    ctx->poll_ns = 0;

    aio_notify(ctx);
}

void aio_context_set_aio_params(AioContext *ctx, int64_t max_batch)
{
    /*
     * No thread synchronization here, it doesn't matter if an incorrect value
     * is used once.
     */
    ctx->aio_max_batch = max_batch;

    aio_notify(ctx);
}

#ifdef CONFIG_LINUX_IO_URING
void aio_add_sqe(void (*prep_sqe)(struct io_uring_sqe *sqe, void *opaque),
                 void *opaque, CqeHandler *cqe_handler)
{
    AioContext *ctx = qemu_get_current_aio_context();
    ctx->fdmon_ops->add_sqe(ctx, prep_sqe, opaque, cqe_handler);

    /*
     * Wake the main loop if it is sleeping in ppoll().  When a vCPU thread
     * queues SQEs, the actual io_uring_submit() only happens in
     * gsource_prepare() in the main loop thread.  Without this notify, the
     * main loop thread's ppoll() can sleep up to 499ms before submitting.
     */
    aio_notify(ctx);
}
#endif /* CONFIG_LINUX_IO_URING */