diff --git a/include/qemu/aio.h b/include/qemu/aio.h index 8cca2360d1..6c22064a28 100644 --- a/include/qemu/aio.h +++ b/include/qemu/aio.h @@ -195,7 +195,7 @@ struct BHListSlice { typedef QSLIST_HEAD(, AioHandler) AioHandlerSList; typedef struct AioPolledEvent { - int64_t ns; /* current polling time in nanoseconds */ + int64_t ns; /* estimated block time in nanoseconds */ } AioPolledEvent; struct AioContext { @@ -306,6 +306,7 @@ struct AioContext { int poll_disable_cnt; /* Polling mode parameters */ + int64_t poll_ns; /* current polling time in nanoseconds */ int64_t poll_max_ns; /* maximum polling time in nanoseconds */ int64_t poll_grow; /* polling time growth factor */ int64_t poll_shrink; /* polling time shrink factor */ diff --git a/util/aio-posix.c b/util/aio-posix.c index 351847c6fb..8e9e9e5d8f 100644 --- a/util/aio-posix.c +++ b/util/aio-posix.c @@ -29,9 +29,11 @@ /* Stop userspace polling on a handler if it isn't active for some time */ #define POLL_IDLE_INTERVAL_NS (7 * NANOSECONDS_PER_SECOND) +#define POLL_WEIGHT_SHIFT (3) -static void adjust_polling_time(AioContext *ctx, AioPolledEvent *poll, - int64_t block_ns); +static void update_handler_poll_times(AioContext *ctx, int64_t block_ns, + int64_t dispatch_time); +static void adjust_polling_time(AioContext *ctx, int64_t block_ns); bool aio_poll_disabled(AioContext *ctx) { @@ -359,7 +361,7 @@ static bool aio_dispatch_handler(AioContext *ctx, AioHandler *node) static bool aio_dispatch_ready_handlers(AioContext *ctx, AioHandlerList *ready_list, - int64_t block_ns) + int64_t dispatch_time) { bool progress = false; AioHandler *node; @@ -369,11 +371,11 @@ static bool aio_dispatch_ready_handlers(AioContext *ctx, progress = aio_dispatch_handler(ctx, node) || progress; /* - * Adjust polling time only after aio_dispatch_handler(), which can - * add the handler to ctx->poll_aio_handlers. + * Update last_dispatch_timestamp to mark this as an active + * handler for polling time adjustment and prevent idle removal. */ if (ctx->poll_max_ns && QLIST_IS_INSERTED(node, node_poll)) { - adjust_polling_time(ctx, &node->poll, block_ns); + node->last_dispatch_timestamp = dispatch_time; } } @@ -394,7 +396,7 @@ void aio_dispatch(AioContext *ctx) ctx->fdmon_ops->dispatch(ctx); } - /* block_ns is 0 because polling is disabled in the glib event loop */ + /* Set now to 0 as polling is disabled in the glib event loop */ aio_dispatch_ready_handlers(ctx, &ready_list, 0); aio_free_deleted_handlers(ctx); @@ -415,9 +417,6 @@ static bool run_poll_handlers_once(AioContext *ctx, QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) { if (node->io_poll(node->opaque)) { aio_add_poll_ready_handler(ready_list, node); - - node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS; - /* * Polling was successful, exit try_poll_mode immediately * to adjust the next polling time. @@ -458,11 +457,10 @@ static bool remove_idle_poll_handlers(AioContext *ctx, } QLIST_FOREACH_SAFE(node, &ctx->poll_aio_handlers, node_poll, tmp) { - if (node->poll_idle_timeout == 0LL) { - node->poll_idle_timeout = now + POLL_IDLE_INTERVAL_NS; - } else if (now >= node->poll_idle_timeout) { + if (node->poll_ready == false && + now >= node->last_dispatch_timestamp + POLL_IDLE_INTERVAL_NS) { trace_poll_remove(ctx, node, node->pfd.fd); - node->poll_idle_timeout = 0LL; + node->last_dispatch_timestamp = 0LL; QLIST_SAFE_REMOVE(node, node_poll); if (ctx->poll_started && node->io_poll_end) { node->io_poll_end(node->opaque); @@ -560,18 +558,13 @@ static bool run_poll_handlers(AioContext *ctx, AioHandlerList *ready_list, static bool try_poll_mode(AioContext *ctx, AioHandlerList *ready_list, int64_t *timeout) { - AioHandler *node; int64_t max_ns; if (QLIST_EMPTY_RCU(&ctx->poll_aio_handlers)) { return false; } - max_ns = 0; - QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) { - max_ns = MAX(max_ns, node->poll.ns); - } - max_ns = qemu_soonest_timeout(*timeout, max_ns); + max_ns = qemu_soonest_timeout(*timeout, ctx->poll_ns); if (max_ns && !ctx->fdmon_ops->need_wait(ctx)) { /* @@ -587,43 +580,85 @@ static bool try_poll_mode(AioContext *ctx, AioHandlerList *ready_list, return false; } -static void adjust_polling_time(AioContext *ctx, AioPolledEvent *poll, - int64_t block_ns) +static void adjust_polling_time(AioContext *ctx, int64_t block_ns) { - if (block_ns <= poll->ns) { - /* This is the sweet spot, no adjustment needed */ - } else if (block_ns > ctx->poll_max_ns) { - /* We'd have to poll for too long, poll less */ - int64_t old = poll->ns; + if (block_ns < ctx->poll_ns) { + int64_t old = ctx->poll_ns; + int64_t shrink = ctx->poll_shrink; - if (ctx->poll_shrink) { - poll->ns /= ctx->poll_shrink; - } else { - poll->ns = 0; + if (shrink == 0) { + shrink = 2; } - trace_poll_shrink(ctx, old, poll->ns); - } else if (poll->ns < ctx->poll_max_ns && - block_ns < ctx->poll_max_ns) { + if (block_ns < (ctx->poll_ns / shrink)) { + ctx->poll_ns /= shrink; + } + + trace_poll_shrink(ctx, old, ctx->poll_ns); + } else if (block_ns > ctx->poll_ns) { /* There is room to grow, poll longer */ - int64_t old = poll->ns; + int64_t old = ctx->poll_ns; int64_t grow = ctx->poll_grow; if (grow == 0) { grow = 2; } - if (poll->ns) { - poll->ns *= grow; + if (block_ns > ctx->poll_ns * grow) { + ctx->poll_ns = block_ns; } else { - poll->ns = 4000; /* start polling at 4 microseconds */ + ctx->poll_ns *= grow; } - if (poll->ns > ctx->poll_max_ns) { - poll->ns = ctx->poll_max_ns; + if (ctx->poll_ns > ctx->poll_max_ns) { + ctx->poll_ns = ctx->poll_max_ns; } - trace_poll_grow(ctx, old, poll->ns); + trace_poll_grow(ctx, old, ctx->poll_ns); + } +} + +static void update_handler_poll_times(AioContext *ctx, int64_t block_ns, + int64_t dispatch_time) +{ + AioHandler *node; + int64_t max_poll_ns = -1; + + QLIST_FOREACH(node, &ctx->poll_aio_handlers, node_poll) { + if (node->last_dispatch_timestamp == dispatch_time) { + /* + * Active handler: had an event in this aio_poll() call. + * Update poll.ns using a weighted average of the current + * block_ns and previous poll.ns to smooth adjustments. + */ + node->poll.ns = node->poll.ns + ? (node->poll.ns - (node->poll.ns >> POLL_WEIGHT_SHIFT)) + + (block_ns >> POLL_WEIGHT_SHIFT) : block_ns; + + if (node->poll.ns > ctx->poll_max_ns) { + node->poll.ns = 0; + } + /* + * Track the maximum poll.ns among active handlers to + * calculate the next polling time. + */ + max_poll_ns = MAX(max_poll_ns, node->poll.ns); + } else { + /* + * Inactive handler: no event in this aio_poll() call but + * was active before. Increase poll.ns by block_ns. If it + * exceeds poll_max_ns, reset to 0 until next event. + */ + if (node->poll.ns != 0) { + node->poll.ns += block_ns; + if (node->poll.ns > ctx->poll_max_ns) { + node->poll.ns = 0; + } + } + } + } + if (max_poll_ns >= 0) { + adjust_polling_time(ctx, max_poll_ns); } } @@ -635,6 +670,7 @@ bool aio_poll(AioContext *ctx, bool blocking) int64_t timeout; int64_t start = 0; int64_t block_ns = 0; + int64_t dispatch_ns = 0; /* * There cannot be two concurrent aio_poll calls for the same AioContext (or @@ -711,7 +747,8 @@ bool aio_poll(AioContext *ctx, bool blocking) /* Calculate blocked time for adaptive polling */ if (ctx->poll_max_ns) { - block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start; + dispatch_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); + block_ns = dispatch_ns - start; } if (ctx->fdmon_ops->dispatch) { @@ -719,10 +756,14 @@ bool aio_poll(AioContext *ctx, bool blocking) } progress |= aio_bh_poll(ctx); - progress |= aio_dispatch_ready_handlers(ctx, &ready_list, block_ns); + progress |= aio_dispatch_ready_handlers(ctx, &ready_list, dispatch_ns); aio_free_deleted_handlers(ctx); + if (ctx->poll_max_ns) { + update_handler_poll_times(ctx, block_ns, dispatch_ns); + } + qemu_lockcnt_dec(&ctx->list_lock); progress |= timerlistgroup_run_timers(&ctx->tlg); @@ -794,6 +835,7 @@ void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns, ctx->poll_max_ns = max_ns; ctx->poll_grow = grow; ctx->poll_shrink = shrink; + ctx->poll_ns = 0; aio_notify(ctx); } diff --git a/util/aio-posix.h b/util/aio-posix.h index ab894a3c0f..cd459bbbae 100644 --- a/util/aio-posix.h +++ b/util/aio-posix.h @@ -38,7 +38,7 @@ struct AioHandler { unsigned flags; /* see fdmon-io_uring.c */ CqeHandler internal_cqe_handler; /* used for POLL_ADD/POLL_REMOVE */ #endif - int64_t poll_idle_timeout; /* when to stop userspace polling */ + int64_t last_dispatch_timestamp; /* when last handler was dispatched */ bool poll_ready; /* has polling detected an event? */ AioPolledEvent poll; }; diff --git a/util/async.c b/util/async.c index 80d6b01a8a..9d3627566f 100644 --- a/util/async.c +++ b/util/async.c @@ -606,6 +606,7 @@ AioContext *aio_context_new(Error **errp) timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx); ctx->poll_max_ns = 0; + ctx->poll_ns = 0; ctx->poll_grow = 0; ctx->poll_shrink = 0;