1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/io_uring.h> 7 8 #include <uapi/linux/io_uring.h> 9 10 #include "../kernel/futex/futex.h" 11 #include "io_uring.h" 12 #include "alloc_cache.h" 13 #include "futex.h" 14 15 struct io_futex { 16 struct file *file; 17 void __user *uaddr; 18 unsigned long futex_val; 19 unsigned long futex_mask; 20 unsigned long futexv_owned; 21 u32 futex_flags; 22 unsigned int futex_nr; 23 bool futexv_unqueued; 24 }; 25 26 struct io_futex_data { 27 struct futex_q q; 28 struct io_kiocb *req; 29 }; 30 31 #define IO_FUTEX_ALLOC_CACHE_MAX 32 32 33 bool io_futex_cache_init(struct io_ring_ctx *ctx) 34 { 35 return io_alloc_cache_init(&ctx->futex_cache, IO_FUTEX_ALLOC_CACHE_MAX, 36 sizeof(struct io_futex_data), 0); 37 } 38 39 void io_futex_cache_free(struct io_ring_ctx *ctx) 40 { 41 io_alloc_cache_free(&ctx->futex_cache, kfree); 42 } 43 44 static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw) 45 { 46 req->async_data = NULL; 47 hlist_del_init(&req->hash_node); 48 io_req_task_complete(req, tw); 49 } 50 51 static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw) 52 { 53 struct io_ring_ctx *ctx = req->ctx; 54 55 io_tw_lock(ctx, tw); 56 io_cache_free(&ctx->futex_cache, req->async_data); 57 __io_futex_complete(req, tw); 58 } 59 60 static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw) 61 { 62 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 63 struct futex_vector *futexv = req->async_data; 64 65 io_tw_lock(req->ctx, tw); 66 67 if (!iof->futexv_unqueued) { 68 int res; 69 70 res = futex_unqueue_multiple(futexv, iof->futex_nr); 71 if (res != -1) 72 io_req_set_res(req, res, 0); 73 } 74 75 kfree(req->async_data); 76 req->flags &= ~REQ_F_ASYNC_DATA; 77 __io_futex_complete(req, tw); 78 } 79 80 static bool io_futexv_claim(struct io_futex *iof) 81 { 82 if (test_bit(0, &iof->futexv_owned) || 83 test_and_set_bit_lock(0, &iof->futexv_owned)) 84 return false; 85 return true; 86 } 87 88 static bool __io_futex_cancel(struct io_kiocb *req) 89 { 90 /* futex wake already done or in progress */ 91 if (req->opcode == IORING_OP_FUTEX_WAIT) { 92 struct io_futex_data *ifd = req->async_data; 93 94 if (!futex_unqueue(&ifd->q)) 95 return false; 96 req->io_task_work.func = io_futex_complete; 97 } else { 98 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 99 100 if (!io_futexv_claim(iof)) 101 return false; 102 req->io_task_work.func = io_futexv_complete; 103 } 104 105 hlist_del_init(&req->hash_node); 106 io_req_set_res(req, -ECANCELED, 0); 107 io_req_task_work_add(req); 108 return true; 109 } 110 111 int io_futex_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, 112 unsigned int issue_flags) 113 { 114 return io_cancel_remove(ctx, cd, issue_flags, &ctx->futex_list, __io_futex_cancel); 115 } 116 117 bool io_futex_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, 118 bool cancel_all) 119 { 120 return io_cancel_remove_all(ctx, tctx, &ctx->futex_list, cancel_all, __io_futex_cancel); 121 } 122 123 int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 124 { 125 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 126 u32 flags; 127 128 if (unlikely(sqe->len || sqe->futex_flags || sqe->buf_index || 129 sqe->file_index)) 130 return -EINVAL; 131 132 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 133 iof->futex_val = READ_ONCE(sqe->addr2); 134 iof->futex_mask = READ_ONCE(sqe->addr3); 135 flags = READ_ONCE(sqe->fd); 136 137 if (flags & ~FUTEX2_VALID_MASK) 138 return -EINVAL; 139 140 iof->futex_flags = futex2_to_flags(flags); 141 if (!futex_flags_valid(iof->futex_flags)) 142 return -EINVAL; 143 144 if (!futex_validate_input(iof->futex_flags, iof->futex_val) || 145 !futex_validate_input(iof->futex_flags, iof->futex_mask)) 146 return -EINVAL; 147 148 /* Mark as inflight, so file exit cancelation will find it */ 149 io_req_track_inflight(req); 150 return 0; 151 } 152 153 static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) 154 { 155 struct io_kiocb *req = q->wake_data; 156 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 157 158 if (!io_futexv_claim(iof)) 159 return; 160 if (unlikely(!__futex_wake_mark(q))) 161 return; 162 163 io_req_set_res(req, 0, 0); 164 req->io_task_work.func = io_futexv_complete; 165 io_req_task_work_add(req); 166 } 167 168 int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 169 { 170 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 171 struct futex_vector *futexv; 172 int ret; 173 174 /* No flags or mask supported for waitv */ 175 if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index || 176 sqe->addr2 || sqe->futex_flags || sqe->addr3)) 177 return -EINVAL; 178 179 iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 180 iof->futex_nr = READ_ONCE(sqe->len); 181 if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX) 182 return -EINVAL; 183 184 futexv = kcalloc(iof->futex_nr, sizeof(*futexv), GFP_KERNEL); 185 if (!futexv) 186 return -ENOMEM; 187 188 ret = futex_parse_waitv(futexv, iof->uaddr, iof->futex_nr, 189 io_futex_wakev_fn, req); 190 if (ret) { 191 kfree(futexv); 192 return ret; 193 } 194 195 /* Mark as inflight, so file exit cancelation will find it */ 196 io_req_track_inflight(req); 197 iof->futexv_owned = 0; 198 iof->futexv_unqueued = 0; 199 req->flags |= REQ_F_ASYNC_DATA; 200 req->async_data = futexv; 201 return 0; 202 } 203 204 static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) 205 { 206 struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); 207 struct io_kiocb *req = ifd->req; 208 209 if (unlikely(!__futex_wake_mark(q))) 210 return; 211 212 io_req_set_res(req, 0, 0); 213 req->io_task_work.func = io_futex_complete; 214 io_req_task_work_add(req); 215 } 216 217 int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) 218 { 219 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 220 struct futex_vector *futexv = req->async_data; 221 struct io_ring_ctx *ctx = req->ctx; 222 int ret, woken = -1; 223 224 io_ring_submit_lock(ctx, issue_flags); 225 226 ret = futex_wait_multiple_setup(futexv, iof->futex_nr, &woken); 227 228 /* 229 * Error case, ret is < 0. Mark the request as failed. 230 */ 231 if (unlikely(ret < 0)) { 232 io_ring_submit_unlock(ctx, issue_flags); 233 req_set_fail(req); 234 io_req_set_res(req, ret, 0); 235 kfree(futexv); 236 req->async_data = NULL; 237 req->flags &= ~REQ_F_ASYNC_DATA; 238 return IOU_COMPLETE; 239 } 240 241 /* 242 * 0 return means that we successfully setup the waiters, and that 243 * nobody triggered a wakeup while we were doing so. If the wakeup 244 * happened post setup, the task_work will be run post this issue and 245 * under the submission lock. 1 means We got woken while setting up, 246 * let that side do the completion. Note that 247 * futex_wait_multiple_setup() will have unqueued all the futexes in 248 * this case. Mark us as having done that already, since this is 249 * different from normal wakeup. 250 */ 251 if (!ret) { 252 /* 253 * If futex_wait_multiple_setup() returns 0 for a 254 * successful setup, then the task state will not be 255 * runnable. This is fine for the sync syscall, as 256 * it'll be blocking unless we already got one of the 257 * futexes woken, but it obviously won't work for an 258 * async invocation. Mark us runnable again. 259 */ 260 __set_current_state(TASK_RUNNING); 261 hlist_add_head(&req->hash_node, &ctx->futex_list); 262 } else { 263 iof->futexv_unqueued = 1; 264 if (woken != -1) 265 io_req_set_res(req, woken, 0); 266 } 267 268 io_ring_submit_unlock(ctx, issue_flags); 269 return IOU_ISSUE_SKIP_COMPLETE; 270 } 271 272 int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) 273 { 274 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 275 struct io_ring_ctx *ctx = req->ctx; 276 struct io_futex_data *ifd = NULL; 277 int ret; 278 279 if (!iof->futex_mask) { 280 ret = -EINVAL; 281 goto done; 282 } 283 284 io_ring_submit_lock(ctx, issue_flags); 285 ifd = io_cache_alloc(&ctx->futex_cache, GFP_NOWAIT); 286 if (!ifd) { 287 ret = -ENOMEM; 288 goto done_unlock; 289 } 290 291 req->async_data = ifd; 292 ifd->q = futex_q_init; 293 ifd->q.bitset = iof->futex_mask; 294 ifd->q.wake = io_futex_wake_fn; 295 ifd->req = req; 296 297 ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags, 298 &ifd->q, NULL, NULL); 299 if (!ret) { 300 hlist_add_head(&req->hash_node, &ctx->futex_list); 301 io_ring_submit_unlock(ctx, issue_flags); 302 303 return IOU_ISSUE_SKIP_COMPLETE; 304 } 305 306 done_unlock: 307 io_ring_submit_unlock(ctx, issue_flags); 308 done: 309 if (ret < 0) 310 req_set_fail(req); 311 io_req_set_res(req, ret, 0); 312 kfree(ifd); 313 return IOU_COMPLETE; 314 } 315 316 int io_futex_wake(struct io_kiocb *req, unsigned int issue_flags) 317 { 318 struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); 319 int ret; 320 321 /* 322 * Strict flags - ensure that waking 0 futexes yields a 0 result. 323 * See commit 43adf8449510 ("futex: FLAGS_STRICT") for details. 324 */ 325 ret = futex_wake(iof->uaddr, FLAGS_STRICT | iof->futex_flags, 326 iof->futex_val, iof->futex_mask); 327 if (ret < 0) 328 req_set_fail(req); 329 io_req_set_res(req, ret, 0); 330 return IOU_COMPLETE; 331 } 332