1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * fs/userfaultfd.c 4 * 5 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> 6 * Copyright (C) 2008-2009 Red Hat, Inc. 7 * Copyright (C) 2015 Red Hat, Inc. 8 * 9 * Some part derived from fs/eventfd.c (anon inode setup) and 10 * mm/ksm.c (mm hashing). 11 */ 12 13 #include <linux/list.h> 14 #include <linux/hashtable.h> 15 #include <linux/sched/signal.h> 16 #include <linux/sched/mm.h> 17 #include <linux/mm.h> 18 #include <linux/mm_inline.h> 19 #include <linux/mmu_notifier.h> 20 #include <linux/poll.h> 21 #include <linux/slab.h> 22 #include <linux/seq_file.h> 23 #include <linux/file.h> 24 #include <linux/bug.h> 25 #include <linux/anon_inodes.h> 26 #include <linux/syscalls.h> 27 #include <linux/userfaultfd_k.h> 28 #include <linux/mempolicy.h> 29 #include <linux/ioctl.h> 30 #include <linux/security.h> 31 #include <linux/hugetlb.h> 32 #include <linux/swapops.h> 33 #include <linux/miscdevice.h> 34 #include <linux/uio.h> 35 36 static int sysctl_unprivileged_userfaultfd __read_mostly; 37 38 #ifdef CONFIG_SYSCTL 39 static const struct ctl_table vm_userfaultfd_table[] = { 40 { 41 .procname = "unprivileged_userfaultfd", 42 .data = &sysctl_unprivileged_userfaultfd, 43 .maxlen = sizeof(sysctl_unprivileged_userfaultfd), 44 .mode = 0644, 45 .proc_handler = proc_dointvec_minmax, 46 .extra1 = SYSCTL_ZERO, 47 .extra2 = SYSCTL_ONE, 48 }, 49 }; 50 #endif 51 52 static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init; 53 54 struct userfaultfd_fork_ctx { 55 struct userfaultfd_ctx *orig; 56 struct userfaultfd_ctx *new; 57 struct list_head list; 58 }; 59 60 struct userfaultfd_unmap_ctx { 61 struct userfaultfd_ctx *ctx; 62 unsigned long start; 63 unsigned long end; 64 struct list_head list; 65 }; 66 67 struct userfaultfd_wait_queue { 68 struct uffd_msg msg; 69 wait_queue_entry_t wq; 70 struct userfaultfd_ctx *ctx; 71 bool waken; 72 }; 73 74 struct userfaultfd_wake_range { 75 unsigned long start; 76 unsigned long len; 77 }; 78 79 /* internal indication that UFFD_API ioctl was successfully executed */ 80 #define UFFD_FEATURE_INITIALIZED (1u << 31) 81 82 static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) 83 { 84 return ctx->features & UFFD_FEATURE_INITIALIZED; 85 } 86 87 static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) 88 { 89 return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); 90 } 91 92 /* 93 * Whether WP_UNPOPULATED is enabled on the uffd context. It is only 94 * meaningful when userfaultfd_wp()==true on the vma and when it's 95 * anonymous. 96 */ 97 bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) 98 { 99 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; 100 101 if (!ctx) 102 return false; 103 104 return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; 105 } 106 107 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, 108 int wake_flags, void *key) 109 { 110 struct userfaultfd_wake_range *range = key; 111 int ret; 112 struct userfaultfd_wait_queue *uwq; 113 unsigned long start, len; 114 115 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 116 ret = 0; 117 /* len == 0 means wake all */ 118 start = range->start; 119 len = range->len; 120 if (len && (start > uwq->msg.arg.pagefault.address || 121 start + len <= uwq->msg.arg.pagefault.address)) 122 goto out; 123 WRITE_ONCE(uwq->waken, true); 124 /* 125 * The Program-Order guarantees provided by the scheduler 126 * ensure uwq->waken is visible before the task is woken. 127 */ 128 ret = wake_up_state(wq->private, mode); 129 if (ret) { 130 /* 131 * Wake only once, autoremove behavior. 132 * 133 * After the effect of list_del_init is visible to the other 134 * CPUs, the waitqueue may disappear from under us, see the 135 * !list_empty_careful() in handle_userfault(). 136 * 137 * try_to_wake_up() has an implicit smp_mb(), and the 138 * wq->private is read before calling the extern function 139 * "wake_up_state" (which in turns calls try_to_wake_up). 140 */ 141 list_del_init(&wq->entry); 142 } 143 out: 144 return ret; 145 } 146 147 /** 148 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd 149 * context. 150 * @ctx: [in] Pointer to the userfaultfd context. 151 */ 152 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx) 153 { 154 refcount_inc(&ctx->refcount); 155 } 156 157 /** 158 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd 159 * context. 160 * @ctx: [in] Pointer to userfaultfd context. 161 * 162 * The userfaultfd context reference must have been previously acquired either 163 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget(). 164 */ 165 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) 166 { 167 if (refcount_dec_and_test(&ctx->refcount)) { 168 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock)); 169 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh)); 170 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock)); 171 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh)); 172 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock)); 173 VM_BUG_ON(waitqueue_active(&ctx->event_wqh)); 174 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock)); 175 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh)); 176 mmdrop(ctx->mm); 177 kmem_cache_free(userfaultfd_ctx_cachep, ctx); 178 } 179 } 180 181 static inline void msg_init(struct uffd_msg *msg) 182 { 183 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32); 184 /* 185 * Must use memset to zero out the paddings or kernel data is 186 * leaked to userland. 187 */ 188 memset(msg, 0, sizeof(struct uffd_msg)); 189 } 190 191 static inline struct uffd_msg userfault_msg(unsigned long address, 192 unsigned long real_address, 193 unsigned int flags, 194 unsigned long reason, 195 unsigned int features) 196 { 197 struct uffd_msg msg; 198 199 msg_init(&msg); 200 msg.event = UFFD_EVENT_PAGEFAULT; 201 202 msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ? 203 real_address : address; 204 205 /* 206 * These flags indicate why the userfault occurred: 207 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault. 208 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault. 209 * - Neither of these flags being set indicates a MISSING fault. 210 * 211 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write 212 * fault. Otherwise, it was a read fault. 213 */ 214 if (flags & FAULT_FLAG_WRITE) 215 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; 216 if (reason & VM_UFFD_WP) 217 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; 218 if (reason & VM_UFFD_MINOR) 219 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR; 220 if (features & UFFD_FEATURE_THREAD_ID) 221 msg.arg.pagefault.feat.ptid = task_pid_vnr(current); 222 return msg; 223 } 224 225 #ifdef CONFIG_HUGETLB_PAGE 226 /* 227 * Same functionality as userfaultfd_must_wait below with modifications for 228 * hugepmd ranges. 229 */ 230 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, 231 struct vm_fault *vmf, 232 unsigned long reason) 233 { 234 struct vm_area_struct *vma = vmf->vma; 235 pte_t *ptep, pte; 236 bool ret = true; 237 238 assert_fault_locked(vmf); 239 240 ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma)); 241 if (!ptep) 242 goto out; 243 244 ret = false; 245 pte = huge_ptep_get(vma->vm_mm, vmf->address, ptep); 246 247 /* 248 * Lockless access: we're in a wait_event so it's ok if it 249 * changes under us. PTE markers should be handled the same as none 250 * ptes here. 251 */ 252 if (huge_pte_none_mostly(pte)) 253 ret = true; 254 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) 255 ret = true; 256 out: 257 return ret; 258 } 259 #else 260 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, 261 struct vm_fault *vmf, 262 unsigned long reason) 263 { 264 return false; /* should never get here */ 265 } 266 #endif /* CONFIG_HUGETLB_PAGE */ 267 268 /* 269 * Verify the pagetables are still not ok after having reigstered into 270 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any 271 * userfault that has already been resolved, if userfaultfd_read_iter and 272 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different 273 * threads. 274 */ 275 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, 276 struct vm_fault *vmf, 277 unsigned long reason) 278 { 279 struct mm_struct *mm = ctx->mm; 280 unsigned long address = vmf->address; 281 pgd_t *pgd; 282 p4d_t *p4d; 283 pud_t *pud; 284 pmd_t *pmd, _pmd; 285 pte_t *pte; 286 pte_t ptent; 287 bool ret = true; 288 289 assert_fault_locked(vmf); 290 291 pgd = pgd_offset(mm, address); 292 if (!pgd_present(*pgd)) 293 goto out; 294 p4d = p4d_offset(pgd, address); 295 if (!p4d_present(*p4d)) 296 goto out; 297 pud = pud_offset(p4d, address); 298 if (!pud_present(*pud)) 299 goto out; 300 pmd = pmd_offset(pud, address); 301 again: 302 _pmd = pmdp_get_lockless(pmd); 303 if (pmd_none(_pmd)) 304 goto out; 305 306 ret = false; 307 if (!pmd_present(_pmd) || pmd_devmap(_pmd)) 308 goto out; 309 310 if (pmd_trans_huge(_pmd)) { 311 if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) 312 ret = true; 313 goto out; 314 } 315 316 pte = pte_offset_map(pmd, address); 317 if (!pte) { 318 ret = true; 319 goto again; 320 } 321 /* 322 * Lockless access: we're in a wait_event so it's ok if it 323 * changes under us. PTE markers should be handled the same as none 324 * ptes here. 325 */ 326 ptent = ptep_get(pte); 327 if (pte_none_mostly(ptent)) 328 ret = true; 329 if (!pte_write(ptent) && (reason & VM_UFFD_WP)) 330 ret = true; 331 pte_unmap(pte); 332 333 out: 334 return ret; 335 } 336 337 static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags) 338 { 339 if (flags & FAULT_FLAG_INTERRUPTIBLE) 340 return TASK_INTERRUPTIBLE; 341 342 if (flags & FAULT_FLAG_KILLABLE) 343 return TASK_KILLABLE; 344 345 return TASK_UNINTERRUPTIBLE; 346 } 347 348 /* 349 * The locking rules involved in returning VM_FAULT_RETRY depending on 350 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and 351 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" 352 * recommendation in __lock_page_or_retry is not an understatement. 353 * 354 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released 355 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is 356 * not set. 357 * 358 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not 359 * set, VM_FAULT_RETRY can still be returned if and only if there are 360 * fatal_signal_pending()s, and the mmap_lock must be released before 361 * returning it. 362 */ 363 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) 364 { 365 struct vm_area_struct *vma = vmf->vma; 366 struct mm_struct *mm = vma->vm_mm; 367 struct userfaultfd_ctx *ctx; 368 struct userfaultfd_wait_queue uwq; 369 vm_fault_t ret = VM_FAULT_SIGBUS; 370 bool must_wait; 371 unsigned int blocking_state; 372 373 /* 374 * We don't do userfault handling for the final child pid update 375 * and when coredumping (faults triggered by get_dump_page()). 376 */ 377 if (current->flags & (PF_EXITING|PF_DUMPCORE)) 378 goto out; 379 380 assert_fault_locked(vmf); 381 382 ctx = vma->vm_userfaultfd_ctx.ctx; 383 if (!ctx) 384 goto out; 385 386 BUG_ON(ctx->mm != mm); 387 388 /* Any unrecognized flag is a bug. */ 389 VM_BUG_ON(reason & ~__VM_UFFD_FLAGS); 390 /* 0 or > 1 flags set is a bug; we expect exactly 1. */ 391 VM_BUG_ON(!reason || (reason & (reason - 1))); 392 393 if (ctx->features & UFFD_FEATURE_SIGBUS) 394 goto out; 395 if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY)) 396 goto out; 397 398 /* 399 * Check that we can return VM_FAULT_RETRY. 400 * 401 * NOTE: it should become possible to return VM_FAULT_RETRY 402 * even if FAULT_FLAG_TRIED is set without leading to gup() 403 * -EBUSY failures, if the userfaultfd is to be extended for 404 * VM_UFFD_WP tracking and we intend to arm the userfault 405 * without first stopping userland access to the memory. For 406 * VM_UFFD_MISSING userfaults this is enough for now. 407 */ 408 if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) { 409 /* 410 * Validate the invariant that nowait must allow retry 411 * to be sure not to return SIGBUS erroneously on 412 * nowait invocations. 413 */ 414 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT); 415 #ifdef CONFIG_DEBUG_VM 416 if (printk_ratelimit()) { 417 printk(KERN_WARNING 418 "FAULT_FLAG_ALLOW_RETRY missing %x\n", 419 vmf->flags); 420 dump_stack(); 421 } 422 #endif 423 goto out; 424 } 425 426 /* 427 * Handle nowait, not much to do other than tell it to retry 428 * and wait. 429 */ 430 ret = VM_FAULT_RETRY; 431 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 432 goto out; 433 434 if (unlikely(READ_ONCE(ctx->released))) { 435 /* 436 * If a concurrent release is detected, do not return 437 * VM_FAULT_SIGBUS or VM_FAULT_NOPAGE, but instead always 438 * return VM_FAULT_RETRY with lock released proactively. 439 * 440 * If we were to return VM_FAULT_SIGBUS here, the non 441 * cooperative manager would be instead forced to 442 * always call UFFDIO_UNREGISTER before it can safely 443 * close the uffd, to avoid involuntary SIGBUS triggered. 444 * 445 * If we were to return VM_FAULT_NOPAGE, it would work for 446 * the fault path, in which the lock will be released 447 * later. However for GUP, faultin_page() does nothing 448 * special on NOPAGE, so GUP would spin retrying without 449 * releasing the mmap read lock, causing possible livelock. 450 * 451 * Here only VM_FAULT_RETRY would make sure the mmap lock 452 * be released immediately, so that the thread concurrently 453 * releasing the userfault would always make progress. 454 */ 455 release_fault_lock(vmf); 456 goto out; 457 } 458 459 /* take the reference before dropping the mmap_lock */ 460 userfaultfd_ctx_get(ctx); 461 462 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function); 463 uwq.wq.private = current; 464 uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags, 465 reason, ctx->features); 466 uwq.ctx = ctx; 467 uwq.waken = false; 468 469 blocking_state = userfaultfd_get_blocking_state(vmf->flags); 470 471 /* 472 * Take the vma lock now, in order to safely call 473 * userfaultfd_huge_must_wait() later. Since acquiring the 474 * (sleepable) vma lock can modify the current task state, that 475 * must be before explicitly calling set_current_state(). 476 */ 477 if (is_vm_hugetlb_page(vma)) 478 hugetlb_vma_lock_read(vma); 479 480 spin_lock_irq(&ctx->fault_pending_wqh.lock); 481 /* 482 * After the __add_wait_queue the uwq is visible to userland 483 * through poll/read(). 484 */ 485 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq); 486 /* 487 * The smp_mb() after __set_current_state prevents the reads 488 * following the spin_unlock to happen before the list_add in 489 * __add_wait_queue. 490 */ 491 set_current_state(blocking_state); 492 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 493 494 if (!is_vm_hugetlb_page(vma)) 495 must_wait = userfaultfd_must_wait(ctx, vmf, reason); 496 else 497 must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason); 498 if (is_vm_hugetlb_page(vma)) 499 hugetlb_vma_unlock_read(vma); 500 release_fault_lock(vmf); 501 502 if (likely(must_wait && !READ_ONCE(ctx->released))) { 503 wake_up_poll(&ctx->fd_wqh, EPOLLIN); 504 schedule(); 505 } 506 507 __set_current_state(TASK_RUNNING); 508 509 /* 510 * Here we race with the list_del; list_add in 511 * userfaultfd_ctx_read(), however because we don't ever run 512 * list_del_init() to refile across the two lists, the prev 513 * and next pointers will never point to self. list_add also 514 * would never let any of the two pointers to point to 515 * self. So list_empty_careful won't risk to see both pointers 516 * pointing to self at any time during the list refile. The 517 * only case where list_del_init() is called is the full 518 * removal in the wake function and there we don't re-list_add 519 * and it's fine not to block on the spinlock. The uwq on this 520 * kernel stack can be released after the list_del_init. 521 */ 522 if (!list_empty_careful(&uwq.wq.entry)) { 523 spin_lock_irq(&ctx->fault_pending_wqh.lock); 524 /* 525 * No need of list_del_init(), the uwq on the stack 526 * will be freed shortly anyway. 527 */ 528 list_del(&uwq.wq.entry); 529 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 530 } 531 532 /* 533 * ctx may go away after this if the userfault pseudo fd is 534 * already released. 535 */ 536 userfaultfd_ctx_put(ctx); 537 538 out: 539 return ret; 540 } 541 542 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, 543 struct userfaultfd_wait_queue *ewq) 544 { 545 struct userfaultfd_ctx *release_new_ctx; 546 547 if (WARN_ON_ONCE(current->flags & PF_EXITING)) 548 goto out; 549 550 ewq->ctx = ctx; 551 init_waitqueue_entry(&ewq->wq, current); 552 release_new_ctx = NULL; 553 554 spin_lock_irq(&ctx->event_wqh.lock); 555 /* 556 * After the __add_wait_queue the uwq is visible to userland 557 * through poll/read(). 558 */ 559 __add_wait_queue(&ctx->event_wqh, &ewq->wq); 560 for (;;) { 561 set_current_state(TASK_KILLABLE); 562 if (ewq->msg.event == 0) 563 break; 564 if (READ_ONCE(ctx->released) || 565 fatal_signal_pending(current)) { 566 /* 567 * &ewq->wq may be queued in fork_event, but 568 * __remove_wait_queue ignores the head 569 * parameter. It would be a problem if it 570 * didn't. 571 */ 572 __remove_wait_queue(&ctx->event_wqh, &ewq->wq); 573 if (ewq->msg.event == UFFD_EVENT_FORK) { 574 struct userfaultfd_ctx *new; 575 576 new = (struct userfaultfd_ctx *) 577 (unsigned long) 578 ewq->msg.arg.reserved.reserved1; 579 release_new_ctx = new; 580 } 581 break; 582 } 583 584 spin_unlock_irq(&ctx->event_wqh.lock); 585 586 wake_up_poll(&ctx->fd_wqh, EPOLLIN); 587 schedule(); 588 589 spin_lock_irq(&ctx->event_wqh.lock); 590 } 591 __set_current_state(TASK_RUNNING); 592 spin_unlock_irq(&ctx->event_wqh.lock); 593 594 if (release_new_ctx) { 595 userfaultfd_release_new(release_new_ctx); 596 userfaultfd_ctx_put(release_new_ctx); 597 } 598 599 /* 600 * ctx may go away after this if the userfault pseudo fd is 601 * already released. 602 */ 603 out: 604 atomic_dec(&ctx->mmap_changing); 605 VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0); 606 userfaultfd_ctx_put(ctx); 607 } 608 609 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx, 610 struct userfaultfd_wait_queue *ewq) 611 { 612 ewq->msg.event = 0; 613 wake_up_locked(&ctx->event_wqh); 614 __remove_wait_queue(&ctx->event_wqh, &ewq->wq); 615 } 616 617 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) 618 { 619 struct userfaultfd_ctx *ctx = NULL, *octx; 620 struct userfaultfd_fork_ctx *fctx; 621 622 octx = vma->vm_userfaultfd_ctx.ctx; 623 if (!octx) 624 return 0; 625 626 if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) { 627 userfaultfd_reset_ctx(vma); 628 return 0; 629 } 630 631 list_for_each_entry(fctx, fcs, list) 632 if (fctx->orig == octx) { 633 ctx = fctx->new; 634 break; 635 } 636 637 if (!ctx) { 638 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL); 639 if (!fctx) 640 return -ENOMEM; 641 642 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); 643 if (!ctx) { 644 kfree(fctx); 645 return -ENOMEM; 646 } 647 648 refcount_set(&ctx->refcount, 1); 649 ctx->flags = octx->flags; 650 ctx->features = octx->features; 651 ctx->released = false; 652 init_rwsem(&ctx->map_changing_lock); 653 atomic_set(&ctx->mmap_changing, 0); 654 ctx->mm = vma->vm_mm; 655 mmgrab(ctx->mm); 656 657 userfaultfd_ctx_get(octx); 658 down_write(&octx->map_changing_lock); 659 atomic_inc(&octx->mmap_changing); 660 up_write(&octx->map_changing_lock); 661 fctx->orig = octx; 662 fctx->new = ctx; 663 list_add_tail(&fctx->list, fcs); 664 } 665 666 vma->vm_userfaultfd_ctx.ctx = ctx; 667 return 0; 668 } 669 670 static void dup_fctx(struct userfaultfd_fork_ctx *fctx) 671 { 672 struct userfaultfd_ctx *ctx = fctx->orig; 673 struct userfaultfd_wait_queue ewq; 674 675 msg_init(&ewq.msg); 676 677 ewq.msg.event = UFFD_EVENT_FORK; 678 ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new; 679 680 userfaultfd_event_wait_completion(ctx, &ewq); 681 } 682 683 void dup_userfaultfd_complete(struct list_head *fcs) 684 { 685 struct userfaultfd_fork_ctx *fctx, *n; 686 687 list_for_each_entry_safe(fctx, n, fcs, list) { 688 dup_fctx(fctx); 689 list_del(&fctx->list); 690 kfree(fctx); 691 } 692 } 693 694 void dup_userfaultfd_fail(struct list_head *fcs) 695 { 696 struct userfaultfd_fork_ctx *fctx, *n; 697 698 /* 699 * An error has occurred on fork, we will tear memory down, but have 700 * allocated memory for fctx's and raised reference counts for both the 701 * original and child contexts (and on the mm for each as a result). 702 * 703 * These would ordinarily be taken care of by a user handling the event, 704 * but we are no longer doing so, so manually clean up here. 705 * 706 * mm tear down will take care of cleaning up VMA contexts. 707 */ 708 list_for_each_entry_safe(fctx, n, fcs, list) { 709 struct userfaultfd_ctx *octx = fctx->orig; 710 struct userfaultfd_ctx *ctx = fctx->new; 711 712 atomic_dec(&octx->mmap_changing); 713 VM_BUG_ON(atomic_read(&octx->mmap_changing) < 0); 714 userfaultfd_ctx_put(octx); 715 userfaultfd_ctx_put(ctx); 716 717 list_del(&fctx->list); 718 kfree(fctx); 719 } 720 } 721 722 void mremap_userfaultfd_prep(struct vm_area_struct *vma, 723 struct vm_userfaultfd_ctx *vm_ctx) 724 { 725 struct userfaultfd_ctx *ctx; 726 727 ctx = vma->vm_userfaultfd_ctx.ctx; 728 729 if (!ctx) 730 return; 731 732 if (ctx->features & UFFD_FEATURE_EVENT_REMAP) { 733 vm_ctx->ctx = ctx; 734 userfaultfd_ctx_get(ctx); 735 down_write(&ctx->map_changing_lock); 736 atomic_inc(&ctx->mmap_changing); 737 up_write(&ctx->map_changing_lock); 738 } else { 739 /* Drop uffd context if remap feature not enabled */ 740 userfaultfd_reset_ctx(vma); 741 } 742 } 743 744 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx, 745 unsigned long from, unsigned long to, 746 unsigned long len) 747 { 748 struct userfaultfd_ctx *ctx = vm_ctx->ctx; 749 struct userfaultfd_wait_queue ewq; 750 751 if (!ctx) 752 return; 753 754 if (to & ~PAGE_MASK) { 755 userfaultfd_ctx_put(ctx); 756 return; 757 } 758 759 msg_init(&ewq.msg); 760 761 ewq.msg.event = UFFD_EVENT_REMAP; 762 ewq.msg.arg.remap.from = from; 763 ewq.msg.arg.remap.to = to; 764 ewq.msg.arg.remap.len = len; 765 766 userfaultfd_event_wait_completion(ctx, &ewq); 767 } 768 769 bool userfaultfd_remove(struct vm_area_struct *vma, 770 unsigned long start, unsigned long end) 771 { 772 struct mm_struct *mm = vma->vm_mm; 773 struct userfaultfd_ctx *ctx; 774 struct userfaultfd_wait_queue ewq; 775 776 ctx = vma->vm_userfaultfd_ctx.ctx; 777 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE)) 778 return true; 779 780 userfaultfd_ctx_get(ctx); 781 down_write(&ctx->map_changing_lock); 782 atomic_inc(&ctx->mmap_changing); 783 up_write(&ctx->map_changing_lock); 784 mmap_read_unlock(mm); 785 786 msg_init(&ewq.msg); 787 788 ewq.msg.event = UFFD_EVENT_REMOVE; 789 ewq.msg.arg.remove.start = start; 790 ewq.msg.arg.remove.end = end; 791 792 userfaultfd_event_wait_completion(ctx, &ewq); 793 794 return false; 795 } 796 797 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, 798 unsigned long start, unsigned long end) 799 { 800 struct userfaultfd_unmap_ctx *unmap_ctx; 801 802 list_for_each_entry(unmap_ctx, unmaps, list) 803 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start && 804 unmap_ctx->end == end) 805 return true; 806 807 return false; 808 } 809 810 int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, 811 unsigned long end, struct list_head *unmaps) 812 { 813 struct userfaultfd_unmap_ctx *unmap_ctx; 814 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; 815 816 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) || 817 has_unmap_ctx(ctx, unmaps, start, end)) 818 return 0; 819 820 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL); 821 if (!unmap_ctx) 822 return -ENOMEM; 823 824 userfaultfd_ctx_get(ctx); 825 down_write(&ctx->map_changing_lock); 826 atomic_inc(&ctx->mmap_changing); 827 up_write(&ctx->map_changing_lock); 828 unmap_ctx->ctx = ctx; 829 unmap_ctx->start = start; 830 unmap_ctx->end = end; 831 list_add_tail(&unmap_ctx->list, unmaps); 832 833 return 0; 834 } 835 836 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf) 837 { 838 struct userfaultfd_unmap_ctx *ctx, *n; 839 struct userfaultfd_wait_queue ewq; 840 841 list_for_each_entry_safe(ctx, n, uf, list) { 842 msg_init(&ewq.msg); 843 844 ewq.msg.event = UFFD_EVENT_UNMAP; 845 ewq.msg.arg.remove.start = ctx->start; 846 ewq.msg.arg.remove.end = ctx->end; 847 848 userfaultfd_event_wait_completion(ctx->ctx, &ewq); 849 850 list_del(&ctx->list); 851 kfree(ctx); 852 } 853 } 854 855 static int userfaultfd_release(struct inode *inode, struct file *file) 856 { 857 struct userfaultfd_ctx *ctx = file->private_data; 858 struct mm_struct *mm = ctx->mm; 859 /* len == 0 means wake all */ 860 struct userfaultfd_wake_range range = { .len = 0, }; 861 862 WRITE_ONCE(ctx->released, true); 863 864 userfaultfd_release_all(mm, ctx); 865 866 /* 867 * After no new page faults can wait on this fault_*wqh, flush 868 * the last page faults that may have been already waiting on 869 * the fault_*wqh. 870 */ 871 spin_lock_irq(&ctx->fault_pending_wqh.lock); 872 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range); 873 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range); 874 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 875 876 /* Flush pending events that may still wait on event_wqh */ 877 wake_up_all(&ctx->event_wqh); 878 879 wake_up_poll(&ctx->fd_wqh, EPOLLHUP); 880 userfaultfd_ctx_put(ctx); 881 return 0; 882 } 883 884 /* fault_pending_wqh.lock must be hold by the caller */ 885 static inline struct userfaultfd_wait_queue *find_userfault_in( 886 wait_queue_head_t *wqh) 887 { 888 wait_queue_entry_t *wq; 889 struct userfaultfd_wait_queue *uwq; 890 891 lockdep_assert_held(&wqh->lock); 892 893 uwq = NULL; 894 if (!waitqueue_active(wqh)) 895 goto out; 896 /* walk in reverse to provide FIFO behavior to read userfaults */ 897 wq = list_last_entry(&wqh->head, typeof(*wq), entry); 898 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 899 out: 900 return uwq; 901 } 902 903 static inline struct userfaultfd_wait_queue *find_userfault( 904 struct userfaultfd_ctx *ctx) 905 { 906 return find_userfault_in(&ctx->fault_pending_wqh); 907 } 908 909 static inline struct userfaultfd_wait_queue *find_userfault_evt( 910 struct userfaultfd_ctx *ctx) 911 { 912 return find_userfault_in(&ctx->event_wqh); 913 } 914 915 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait) 916 { 917 struct userfaultfd_ctx *ctx = file->private_data; 918 __poll_t ret; 919 920 poll_wait(file, &ctx->fd_wqh, wait); 921 922 if (!userfaultfd_is_initialized(ctx)) 923 return EPOLLERR; 924 925 /* 926 * poll() never guarantees that read won't block. 927 * userfaults can be waken before they're read(). 928 */ 929 if (unlikely(!(file->f_flags & O_NONBLOCK))) 930 return EPOLLERR; 931 /* 932 * lockless access to see if there are pending faults 933 * __pollwait last action is the add_wait_queue but 934 * the spin_unlock would allow the waitqueue_active to 935 * pass above the actual list_add inside 936 * add_wait_queue critical section. So use a full 937 * memory barrier to serialize the list_add write of 938 * add_wait_queue() with the waitqueue_active read 939 * below. 940 */ 941 ret = 0; 942 smp_mb(); 943 if (waitqueue_active(&ctx->fault_pending_wqh)) 944 ret = EPOLLIN; 945 else if (waitqueue_active(&ctx->event_wqh)) 946 ret = EPOLLIN; 947 948 return ret; 949 } 950 951 static const struct file_operations userfaultfd_fops; 952 953 static int resolve_userfault_fork(struct userfaultfd_ctx *new, 954 struct inode *inode, 955 struct uffd_msg *msg) 956 { 957 int fd; 958 959 fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new, 960 O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); 961 if (fd < 0) 962 return fd; 963 964 msg->arg.reserved.reserved1 = 0; 965 msg->arg.fork.ufd = fd; 966 return 0; 967 } 968 969 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, 970 struct uffd_msg *msg, struct inode *inode) 971 { 972 ssize_t ret; 973 DECLARE_WAITQUEUE(wait, current); 974 struct userfaultfd_wait_queue *uwq; 975 /* 976 * Handling fork event requires sleeping operations, so 977 * we drop the event_wqh lock, then do these ops, then 978 * lock it back and wake up the waiter. While the lock is 979 * dropped the ewq may go away so we keep track of it 980 * carefully. 981 */ 982 LIST_HEAD(fork_event); 983 struct userfaultfd_ctx *fork_nctx = NULL; 984 985 /* always take the fd_wqh lock before the fault_pending_wqh lock */ 986 spin_lock_irq(&ctx->fd_wqh.lock); 987 __add_wait_queue(&ctx->fd_wqh, &wait); 988 for (;;) { 989 set_current_state(TASK_INTERRUPTIBLE); 990 spin_lock(&ctx->fault_pending_wqh.lock); 991 uwq = find_userfault(ctx); 992 if (uwq) { 993 /* 994 * Use a seqcount to repeat the lockless check 995 * in wake_userfault() to avoid missing 996 * wakeups because during the refile both 997 * waitqueue could become empty if this is the 998 * only userfault. 999 */ 1000 write_seqcount_begin(&ctx->refile_seq); 1001 1002 /* 1003 * The fault_pending_wqh.lock prevents the uwq 1004 * to disappear from under us. 1005 * 1006 * Refile this userfault from 1007 * fault_pending_wqh to fault_wqh, it's not 1008 * pending anymore after we read it. 1009 * 1010 * Use list_del() by hand (as 1011 * userfaultfd_wake_function also uses 1012 * list_del_init() by hand) to be sure nobody 1013 * changes __remove_wait_queue() to use 1014 * list_del_init() in turn breaking the 1015 * !list_empty_careful() check in 1016 * handle_userfault(). The uwq->wq.head list 1017 * must never be empty at any time during the 1018 * refile, or the waitqueue could disappear 1019 * from under us. The "wait_queue_head_t" 1020 * parameter of __remove_wait_queue() is unused 1021 * anyway. 1022 */ 1023 list_del(&uwq->wq.entry); 1024 add_wait_queue(&ctx->fault_wqh, &uwq->wq); 1025 1026 write_seqcount_end(&ctx->refile_seq); 1027 1028 /* careful to always initialize msg if ret == 0 */ 1029 *msg = uwq->msg; 1030 spin_unlock(&ctx->fault_pending_wqh.lock); 1031 ret = 0; 1032 break; 1033 } 1034 spin_unlock(&ctx->fault_pending_wqh.lock); 1035 1036 spin_lock(&ctx->event_wqh.lock); 1037 uwq = find_userfault_evt(ctx); 1038 if (uwq) { 1039 *msg = uwq->msg; 1040 1041 if (uwq->msg.event == UFFD_EVENT_FORK) { 1042 fork_nctx = (struct userfaultfd_ctx *) 1043 (unsigned long) 1044 uwq->msg.arg.reserved.reserved1; 1045 list_move(&uwq->wq.entry, &fork_event); 1046 /* 1047 * fork_nctx can be freed as soon as 1048 * we drop the lock, unless we take a 1049 * reference on it. 1050 */ 1051 userfaultfd_ctx_get(fork_nctx); 1052 spin_unlock(&ctx->event_wqh.lock); 1053 ret = 0; 1054 break; 1055 } 1056 1057 userfaultfd_event_complete(ctx, uwq); 1058 spin_unlock(&ctx->event_wqh.lock); 1059 ret = 0; 1060 break; 1061 } 1062 spin_unlock(&ctx->event_wqh.lock); 1063 1064 if (signal_pending(current)) { 1065 ret = -ERESTARTSYS; 1066 break; 1067 } 1068 if (no_wait) { 1069 ret = -EAGAIN; 1070 break; 1071 } 1072 spin_unlock_irq(&ctx->fd_wqh.lock); 1073 schedule(); 1074 spin_lock_irq(&ctx->fd_wqh.lock); 1075 } 1076 __remove_wait_queue(&ctx->fd_wqh, &wait); 1077 __set_current_state(TASK_RUNNING); 1078 spin_unlock_irq(&ctx->fd_wqh.lock); 1079 1080 if (!ret && msg->event == UFFD_EVENT_FORK) { 1081 ret = resolve_userfault_fork(fork_nctx, inode, msg); 1082 spin_lock_irq(&ctx->event_wqh.lock); 1083 if (!list_empty(&fork_event)) { 1084 /* 1085 * The fork thread didn't abort, so we can 1086 * drop the temporary refcount. 1087 */ 1088 userfaultfd_ctx_put(fork_nctx); 1089 1090 uwq = list_first_entry(&fork_event, 1091 typeof(*uwq), 1092 wq.entry); 1093 /* 1094 * If fork_event list wasn't empty and in turn 1095 * the event wasn't already released by fork 1096 * (the event is allocated on fork kernel 1097 * stack), put the event back to its place in 1098 * the event_wq. fork_event head will be freed 1099 * as soon as we return so the event cannot 1100 * stay queued there no matter the current 1101 * "ret" value. 1102 */ 1103 list_del(&uwq->wq.entry); 1104 __add_wait_queue(&ctx->event_wqh, &uwq->wq); 1105 1106 /* 1107 * Leave the event in the waitqueue and report 1108 * error to userland if we failed to resolve 1109 * the userfault fork. 1110 */ 1111 if (likely(!ret)) 1112 userfaultfd_event_complete(ctx, uwq); 1113 } else { 1114 /* 1115 * Here the fork thread aborted and the 1116 * refcount from the fork thread on fork_nctx 1117 * has already been released. We still hold 1118 * the reference we took before releasing the 1119 * lock above. If resolve_userfault_fork 1120 * failed we've to drop it because the 1121 * fork_nctx has to be freed in such case. If 1122 * it succeeded we'll hold it because the new 1123 * uffd references it. 1124 */ 1125 if (ret) 1126 userfaultfd_ctx_put(fork_nctx); 1127 } 1128 spin_unlock_irq(&ctx->event_wqh.lock); 1129 } 1130 1131 return ret; 1132 } 1133 1134 static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to) 1135 { 1136 struct file *file = iocb->ki_filp; 1137 struct userfaultfd_ctx *ctx = file->private_data; 1138 ssize_t _ret, ret = 0; 1139 struct uffd_msg msg; 1140 struct inode *inode = file_inode(file); 1141 bool no_wait; 1142 1143 if (!userfaultfd_is_initialized(ctx)) 1144 return -EINVAL; 1145 1146 no_wait = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT; 1147 for (;;) { 1148 if (iov_iter_count(to) < sizeof(msg)) 1149 return ret ? ret : -EINVAL; 1150 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode); 1151 if (_ret < 0) 1152 return ret ? ret : _ret; 1153 _ret = !copy_to_iter_full(&msg, sizeof(msg), to); 1154 if (_ret) 1155 return ret ? ret : -EFAULT; 1156 ret += sizeof(msg); 1157 /* 1158 * Allow to read more than one fault at time but only 1159 * block if waiting for the very first one. 1160 */ 1161 no_wait = true; 1162 } 1163 } 1164 1165 static void __wake_userfault(struct userfaultfd_ctx *ctx, 1166 struct userfaultfd_wake_range *range) 1167 { 1168 spin_lock_irq(&ctx->fault_pending_wqh.lock); 1169 /* wake all in the range and autoremove */ 1170 if (waitqueue_active(&ctx->fault_pending_wqh)) 1171 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 1172 range); 1173 if (waitqueue_active(&ctx->fault_wqh)) 1174 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range); 1175 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 1176 } 1177 1178 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, 1179 struct userfaultfd_wake_range *range) 1180 { 1181 unsigned seq; 1182 bool need_wakeup; 1183 1184 /* 1185 * To be sure waitqueue_active() is not reordered by the CPU 1186 * before the pagetable update, use an explicit SMP memory 1187 * barrier here. PT lock release or mmap_read_unlock(mm) still 1188 * have release semantics that can allow the 1189 * waitqueue_active() to be reordered before the pte update. 1190 */ 1191 smp_mb(); 1192 1193 /* 1194 * Use waitqueue_active because it's very frequent to 1195 * change the address space atomically even if there are no 1196 * userfaults yet. So we take the spinlock only when we're 1197 * sure we've userfaults to wake. 1198 */ 1199 do { 1200 seq = read_seqcount_begin(&ctx->refile_seq); 1201 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) || 1202 waitqueue_active(&ctx->fault_wqh); 1203 cond_resched(); 1204 } while (read_seqcount_retry(&ctx->refile_seq, seq)); 1205 if (need_wakeup) 1206 __wake_userfault(ctx, range); 1207 } 1208 1209 static __always_inline int validate_unaligned_range( 1210 struct mm_struct *mm, __u64 start, __u64 len) 1211 { 1212 __u64 task_size = mm->task_size; 1213 1214 if (len & ~PAGE_MASK) 1215 return -EINVAL; 1216 if (!len) 1217 return -EINVAL; 1218 if (start < mmap_min_addr) 1219 return -EINVAL; 1220 if (start >= task_size) 1221 return -EINVAL; 1222 if (len > task_size - start) 1223 return -EINVAL; 1224 if (start + len <= start) 1225 return -EINVAL; 1226 return 0; 1227 } 1228 1229 static __always_inline int validate_range(struct mm_struct *mm, 1230 __u64 start, __u64 len) 1231 { 1232 if (start & ~PAGE_MASK) 1233 return -EINVAL; 1234 1235 return validate_unaligned_range(mm, start, len); 1236 } 1237 1238 static int userfaultfd_register(struct userfaultfd_ctx *ctx, 1239 unsigned long arg) 1240 { 1241 struct mm_struct *mm = ctx->mm; 1242 struct vm_area_struct *vma, *cur; 1243 int ret; 1244 struct uffdio_register uffdio_register; 1245 struct uffdio_register __user *user_uffdio_register; 1246 unsigned long vm_flags; 1247 bool found; 1248 bool basic_ioctls; 1249 unsigned long start, end; 1250 struct vma_iterator vmi; 1251 bool wp_async = userfaultfd_wp_async_ctx(ctx); 1252 1253 user_uffdio_register = (struct uffdio_register __user *) arg; 1254 1255 ret = -EFAULT; 1256 if (copy_from_user(&uffdio_register, user_uffdio_register, 1257 sizeof(uffdio_register)-sizeof(__u64))) 1258 goto out; 1259 1260 ret = -EINVAL; 1261 if (!uffdio_register.mode) 1262 goto out; 1263 if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES) 1264 goto out; 1265 vm_flags = 0; 1266 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) 1267 vm_flags |= VM_UFFD_MISSING; 1268 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { 1269 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP 1270 goto out; 1271 #endif 1272 vm_flags |= VM_UFFD_WP; 1273 } 1274 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) { 1275 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 1276 goto out; 1277 #endif 1278 vm_flags |= VM_UFFD_MINOR; 1279 } 1280 1281 ret = validate_range(mm, uffdio_register.range.start, 1282 uffdio_register.range.len); 1283 if (ret) 1284 goto out; 1285 1286 start = uffdio_register.range.start; 1287 end = start + uffdio_register.range.len; 1288 1289 ret = -ENOMEM; 1290 if (!mmget_not_zero(mm)) 1291 goto out; 1292 1293 ret = -EINVAL; 1294 mmap_write_lock(mm); 1295 vma_iter_init(&vmi, mm, start); 1296 vma = vma_find(&vmi, end); 1297 if (!vma) 1298 goto out_unlock; 1299 1300 /* 1301 * If the first vma contains huge pages, make sure start address 1302 * is aligned to huge page size. 1303 */ 1304 if (is_vm_hugetlb_page(vma)) { 1305 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); 1306 1307 if (start & (vma_hpagesize - 1)) 1308 goto out_unlock; 1309 } 1310 1311 /* 1312 * Search for not compatible vmas. 1313 */ 1314 found = false; 1315 basic_ioctls = false; 1316 cur = vma; 1317 do { 1318 cond_resched(); 1319 1320 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ 1321 !!(cur->vm_flags & __VM_UFFD_FLAGS)); 1322 1323 /* check not compatible vmas */ 1324 ret = -EINVAL; 1325 if (!vma_can_userfault(cur, vm_flags, wp_async)) 1326 goto out_unlock; 1327 1328 /* 1329 * UFFDIO_COPY will fill file holes even without 1330 * PROT_WRITE. This check enforces that if this is a 1331 * MAP_SHARED, the process has write permission to the backing 1332 * file. If VM_MAYWRITE is set it also enforces that on a 1333 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further 1334 * F_WRITE_SEAL can be taken until the vma is destroyed. 1335 */ 1336 ret = -EPERM; 1337 if (unlikely(!(cur->vm_flags & VM_MAYWRITE))) 1338 goto out_unlock; 1339 1340 /* 1341 * If this vma contains ending address, and huge pages 1342 * check alignment. 1343 */ 1344 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end && 1345 end > cur->vm_start) { 1346 unsigned long vma_hpagesize = vma_kernel_pagesize(cur); 1347 1348 ret = -EINVAL; 1349 1350 if (end & (vma_hpagesize - 1)) 1351 goto out_unlock; 1352 } 1353 if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE)) 1354 goto out_unlock; 1355 1356 /* 1357 * Check that this vma isn't already owned by a 1358 * different userfaultfd. We can't allow more than one 1359 * userfaultfd to own a single vma simultaneously or we 1360 * wouldn't know which one to deliver the userfaults to. 1361 */ 1362 ret = -EBUSY; 1363 if (cur->vm_userfaultfd_ctx.ctx && 1364 cur->vm_userfaultfd_ctx.ctx != ctx) 1365 goto out_unlock; 1366 1367 /* 1368 * Note vmas containing huge pages 1369 */ 1370 if (is_vm_hugetlb_page(cur)) 1371 basic_ioctls = true; 1372 1373 found = true; 1374 } for_each_vma_range(vmi, cur, end); 1375 BUG_ON(!found); 1376 1377 ret = userfaultfd_register_range(ctx, vma, vm_flags, start, end, 1378 wp_async); 1379 1380 out_unlock: 1381 mmap_write_unlock(mm); 1382 mmput(mm); 1383 if (!ret) { 1384 __u64 ioctls_out; 1385 1386 ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : 1387 UFFD_API_RANGE_IOCTLS; 1388 1389 /* 1390 * Declare the WP ioctl only if the WP mode is 1391 * specified and all checks passed with the range 1392 */ 1393 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP)) 1394 ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT); 1395 1396 /* CONTINUE ioctl is only supported for MINOR ranges. */ 1397 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR)) 1398 ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE); 1399 1400 /* 1401 * Now that we scanned all vmas we can already tell 1402 * userland which ioctls methods are guaranteed to 1403 * succeed on this range. 1404 */ 1405 if (put_user(ioctls_out, &user_uffdio_register->ioctls)) 1406 ret = -EFAULT; 1407 } 1408 out: 1409 return ret; 1410 } 1411 1412 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, 1413 unsigned long arg) 1414 { 1415 struct mm_struct *mm = ctx->mm; 1416 struct vm_area_struct *vma, *prev, *cur; 1417 int ret; 1418 struct uffdio_range uffdio_unregister; 1419 bool found; 1420 unsigned long start, end, vma_end; 1421 const void __user *buf = (void __user *)arg; 1422 struct vma_iterator vmi; 1423 bool wp_async = userfaultfd_wp_async_ctx(ctx); 1424 1425 ret = -EFAULT; 1426 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) 1427 goto out; 1428 1429 ret = validate_range(mm, uffdio_unregister.start, 1430 uffdio_unregister.len); 1431 if (ret) 1432 goto out; 1433 1434 start = uffdio_unregister.start; 1435 end = start + uffdio_unregister.len; 1436 1437 ret = -ENOMEM; 1438 if (!mmget_not_zero(mm)) 1439 goto out; 1440 1441 mmap_write_lock(mm); 1442 ret = -EINVAL; 1443 vma_iter_init(&vmi, mm, start); 1444 vma = vma_find(&vmi, end); 1445 if (!vma) 1446 goto out_unlock; 1447 1448 /* 1449 * If the first vma contains huge pages, make sure start address 1450 * is aligned to huge page size. 1451 */ 1452 if (is_vm_hugetlb_page(vma)) { 1453 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); 1454 1455 if (start & (vma_hpagesize - 1)) 1456 goto out_unlock; 1457 } 1458 1459 /* 1460 * Search for not compatible vmas. 1461 */ 1462 found = false; 1463 cur = vma; 1464 do { 1465 cond_resched(); 1466 1467 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ 1468 !!(cur->vm_flags & __VM_UFFD_FLAGS)); 1469 1470 /* 1471 * Check not compatible vmas, not strictly required 1472 * here as not compatible vmas cannot have an 1473 * userfaultfd_ctx registered on them, but this 1474 * provides for more strict behavior to notice 1475 * unregistration errors. 1476 */ 1477 if (!vma_can_userfault(cur, cur->vm_flags, wp_async)) 1478 goto out_unlock; 1479 1480 found = true; 1481 } for_each_vma_range(vmi, cur, end); 1482 BUG_ON(!found); 1483 1484 vma_iter_set(&vmi, start); 1485 prev = vma_prev(&vmi); 1486 if (vma->vm_start < start) 1487 prev = vma; 1488 1489 ret = 0; 1490 for_each_vma_range(vmi, vma, end) { 1491 cond_resched(); 1492 1493 BUG_ON(!vma_can_userfault(vma, vma->vm_flags, wp_async)); 1494 1495 /* 1496 * Nothing to do: this vma is already registered into this 1497 * userfaultfd and with the right tracking mode too. 1498 */ 1499 if (!vma->vm_userfaultfd_ctx.ctx) 1500 goto skip; 1501 1502 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); 1503 1504 if (vma->vm_start > start) 1505 start = vma->vm_start; 1506 vma_end = min(end, vma->vm_end); 1507 1508 if (userfaultfd_missing(vma)) { 1509 /* 1510 * Wake any concurrent pending userfault while 1511 * we unregister, so they will not hang 1512 * permanently and it avoids userland to call 1513 * UFFDIO_WAKE explicitly. 1514 */ 1515 struct userfaultfd_wake_range range; 1516 range.start = start; 1517 range.len = vma_end - start; 1518 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); 1519 } 1520 1521 vma = userfaultfd_clear_vma(&vmi, prev, vma, 1522 start, vma_end); 1523 if (IS_ERR(vma)) { 1524 ret = PTR_ERR(vma); 1525 break; 1526 } 1527 1528 skip: 1529 prev = vma; 1530 start = vma->vm_end; 1531 } 1532 1533 out_unlock: 1534 mmap_write_unlock(mm); 1535 mmput(mm); 1536 out: 1537 return ret; 1538 } 1539 1540 /* 1541 * userfaultfd_wake may be used in combination with the 1542 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches. 1543 */ 1544 static int userfaultfd_wake(struct userfaultfd_ctx *ctx, 1545 unsigned long arg) 1546 { 1547 int ret; 1548 struct uffdio_range uffdio_wake; 1549 struct userfaultfd_wake_range range; 1550 const void __user *buf = (void __user *)arg; 1551 1552 ret = -EFAULT; 1553 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake))) 1554 goto out; 1555 1556 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len); 1557 if (ret) 1558 goto out; 1559 1560 range.start = uffdio_wake.start; 1561 range.len = uffdio_wake.len; 1562 1563 /* 1564 * len == 0 means wake all and we don't want to wake all here, 1565 * so check it again to be sure. 1566 */ 1567 VM_BUG_ON(!range.len); 1568 1569 wake_userfault(ctx, &range); 1570 ret = 0; 1571 1572 out: 1573 return ret; 1574 } 1575 1576 static int userfaultfd_copy(struct userfaultfd_ctx *ctx, 1577 unsigned long arg) 1578 { 1579 __s64 ret; 1580 struct uffdio_copy uffdio_copy; 1581 struct uffdio_copy __user *user_uffdio_copy; 1582 struct userfaultfd_wake_range range; 1583 uffd_flags_t flags = 0; 1584 1585 user_uffdio_copy = (struct uffdio_copy __user *) arg; 1586 1587 ret = -EAGAIN; 1588 if (atomic_read(&ctx->mmap_changing)) 1589 goto out; 1590 1591 ret = -EFAULT; 1592 if (copy_from_user(&uffdio_copy, user_uffdio_copy, 1593 /* don't copy "copy" last field */ 1594 sizeof(uffdio_copy)-sizeof(__s64))) 1595 goto out; 1596 1597 ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, 1598 uffdio_copy.len); 1599 if (ret) 1600 goto out; 1601 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); 1602 if (ret) 1603 goto out; 1604 1605 ret = -EINVAL; 1606 if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) 1607 goto out; 1608 if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP) 1609 flags |= MFILL_ATOMIC_WP; 1610 if (mmget_not_zero(ctx->mm)) { 1611 ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src, 1612 uffdio_copy.len, flags); 1613 mmput(ctx->mm); 1614 } else { 1615 return -ESRCH; 1616 } 1617 if (unlikely(put_user(ret, &user_uffdio_copy->copy))) 1618 return -EFAULT; 1619 if (ret < 0) 1620 goto out; 1621 BUG_ON(!ret); 1622 /* len == 0 would wake all */ 1623 range.len = ret; 1624 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) { 1625 range.start = uffdio_copy.dst; 1626 wake_userfault(ctx, &range); 1627 } 1628 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN; 1629 out: 1630 return ret; 1631 } 1632 1633 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, 1634 unsigned long arg) 1635 { 1636 __s64 ret; 1637 struct uffdio_zeropage uffdio_zeropage; 1638 struct uffdio_zeropage __user *user_uffdio_zeropage; 1639 struct userfaultfd_wake_range range; 1640 1641 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg; 1642 1643 ret = -EAGAIN; 1644 if (atomic_read(&ctx->mmap_changing)) 1645 goto out; 1646 1647 ret = -EFAULT; 1648 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage, 1649 /* don't copy "zeropage" last field */ 1650 sizeof(uffdio_zeropage)-sizeof(__s64))) 1651 goto out; 1652 1653 ret = validate_range(ctx->mm, uffdio_zeropage.range.start, 1654 uffdio_zeropage.range.len); 1655 if (ret) 1656 goto out; 1657 ret = -EINVAL; 1658 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE) 1659 goto out; 1660 1661 if (mmget_not_zero(ctx->mm)) { 1662 ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start, 1663 uffdio_zeropage.range.len); 1664 mmput(ctx->mm); 1665 } else { 1666 return -ESRCH; 1667 } 1668 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) 1669 return -EFAULT; 1670 if (ret < 0) 1671 goto out; 1672 /* len == 0 would wake all */ 1673 BUG_ON(!ret); 1674 range.len = ret; 1675 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) { 1676 range.start = uffdio_zeropage.range.start; 1677 wake_userfault(ctx, &range); 1678 } 1679 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN; 1680 out: 1681 return ret; 1682 } 1683 1684 static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, 1685 unsigned long arg) 1686 { 1687 int ret; 1688 struct uffdio_writeprotect uffdio_wp; 1689 struct uffdio_writeprotect __user *user_uffdio_wp; 1690 struct userfaultfd_wake_range range; 1691 bool mode_wp, mode_dontwake; 1692 1693 if (atomic_read(&ctx->mmap_changing)) 1694 return -EAGAIN; 1695 1696 user_uffdio_wp = (struct uffdio_writeprotect __user *) arg; 1697 1698 if (copy_from_user(&uffdio_wp, user_uffdio_wp, 1699 sizeof(struct uffdio_writeprotect))) 1700 return -EFAULT; 1701 1702 ret = validate_range(ctx->mm, uffdio_wp.range.start, 1703 uffdio_wp.range.len); 1704 if (ret) 1705 return ret; 1706 1707 if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE | 1708 UFFDIO_WRITEPROTECT_MODE_WP)) 1709 return -EINVAL; 1710 1711 mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP; 1712 mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE; 1713 1714 if (mode_wp && mode_dontwake) 1715 return -EINVAL; 1716 1717 if (mmget_not_zero(ctx->mm)) { 1718 ret = mwriteprotect_range(ctx, uffdio_wp.range.start, 1719 uffdio_wp.range.len, mode_wp); 1720 mmput(ctx->mm); 1721 } else { 1722 return -ESRCH; 1723 } 1724 1725 if (ret) 1726 return ret; 1727 1728 if (!mode_wp && !mode_dontwake) { 1729 range.start = uffdio_wp.range.start; 1730 range.len = uffdio_wp.range.len; 1731 wake_userfault(ctx, &range); 1732 } 1733 return ret; 1734 } 1735 1736 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) 1737 { 1738 __s64 ret; 1739 struct uffdio_continue uffdio_continue; 1740 struct uffdio_continue __user *user_uffdio_continue; 1741 struct userfaultfd_wake_range range; 1742 uffd_flags_t flags = 0; 1743 1744 user_uffdio_continue = (struct uffdio_continue __user *)arg; 1745 1746 ret = -EAGAIN; 1747 if (atomic_read(&ctx->mmap_changing)) 1748 goto out; 1749 1750 ret = -EFAULT; 1751 if (copy_from_user(&uffdio_continue, user_uffdio_continue, 1752 /* don't copy the output fields */ 1753 sizeof(uffdio_continue) - (sizeof(__s64)))) 1754 goto out; 1755 1756 ret = validate_range(ctx->mm, uffdio_continue.range.start, 1757 uffdio_continue.range.len); 1758 if (ret) 1759 goto out; 1760 1761 ret = -EINVAL; 1762 if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE | 1763 UFFDIO_CONTINUE_MODE_WP)) 1764 goto out; 1765 if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP) 1766 flags |= MFILL_ATOMIC_WP; 1767 1768 if (mmget_not_zero(ctx->mm)) { 1769 ret = mfill_atomic_continue(ctx, uffdio_continue.range.start, 1770 uffdio_continue.range.len, flags); 1771 mmput(ctx->mm); 1772 } else { 1773 return -ESRCH; 1774 } 1775 1776 if (unlikely(put_user(ret, &user_uffdio_continue->mapped))) 1777 return -EFAULT; 1778 if (ret < 0) 1779 goto out; 1780 1781 /* len == 0 would wake all */ 1782 BUG_ON(!ret); 1783 range.len = ret; 1784 if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) { 1785 range.start = uffdio_continue.range.start; 1786 wake_userfault(ctx, &range); 1787 } 1788 ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN; 1789 1790 out: 1791 return ret; 1792 } 1793 1794 static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg) 1795 { 1796 __s64 ret; 1797 struct uffdio_poison uffdio_poison; 1798 struct uffdio_poison __user *user_uffdio_poison; 1799 struct userfaultfd_wake_range range; 1800 1801 user_uffdio_poison = (struct uffdio_poison __user *)arg; 1802 1803 ret = -EAGAIN; 1804 if (atomic_read(&ctx->mmap_changing)) 1805 goto out; 1806 1807 ret = -EFAULT; 1808 if (copy_from_user(&uffdio_poison, user_uffdio_poison, 1809 /* don't copy the output fields */ 1810 sizeof(uffdio_poison) - (sizeof(__s64)))) 1811 goto out; 1812 1813 ret = validate_range(ctx->mm, uffdio_poison.range.start, 1814 uffdio_poison.range.len); 1815 if (ret) 1816 goto out; 1817 1818 ret = -EINVAL; 1819 if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE) 1820 goto out; 1821 1822 if (mmget_not_zero(ctx->mm)) { 1823 ret = mfill_atomic_poison(ctx, uffdio_poison.range.start, 1824 uffdio_poison.range.len, 0); 1825 mmput(ctx->mm); 1826 } else { 1827 return -ESRCH; 1828 } 1829 1830 if (unlikely(put_user(ret, &user_uffdio_poison->updated))) 1831 return -EFAULT; 1832 if (ret < 0) 1833 goto out; 1834 1835 /* len == 0 would wake all */ 1836 BUG_ON(!ret); 1837 range.len = ret; 1838 if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) { 1839 range.start = uffdio_poison.range.start; 1840 wake_userfault(ctx, &range); 1841 } 1842 ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN; 1843 1844 out: 1845 return ret; 1846 } 1847 1848 bool userfaultfd_wp_async(struct vm_area_struct *vma) 1849 { 1850 return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx); 1851 } 1852 1853 static inline unsigned int uffd_ctx_features(__u64 user_features) 1854 { 1855 /* 1856 * For the current set of features the bits just coincide. Set 1857 * UFFD_FEATURE_INITIALIZED to mark the features as enabled. 1858 */ 1859 return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED; 1860 } 1861 1862 static int userfaultfd_move(struct userfaultfd_ctx *ctx, 1863 unsigned long arg) 1864 { 1865 __s64 ret; 1866 struct uffdio_move uffdio_move; 1867 struct uffdio_move __user *user_uffdio_move; 1868 struct userfaultfd_wake_range range; 1869 struct mm_struct *mm = ctx->mm; 1870 1871 user_uffdio_move = (struct uffdio_move __user *) arg; 1872 1873 if (atomic_read(&ctx->mmap_changing)) 1874 return -EAGAIN; 1875 1876 if (copy_from_user(&uffdio_move, user_uffdio_move, 1877 /* don't copy "move" last field */ 1878 sizeof(uffdio_move)-sizeof(__s64))) 1879 return -EFAULT; 1880 1881 /* Do not allow cross-mm moves. */ 1882 if (mm != current->mm) 1883 return -EINVAL; 1884 1885 ret = validate_range(mm, uffdio_move.dst, uffdio_move.len); 1886 if (ret) 1887 return ret; 1888 1889 ret = validate_range(mm, uffdio_move.src, uffdio_move.len); 1890 if (ret) 1891 return ret; 1892 1893 if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES| 1894 UFFDIO_MOVE_MODE_DONTWAKE)) 1895 return -EINVAL; 1896 1897 if (mmget_not_zero(mm)) { 1898 ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src, 1899 uffdio_move.len, uffdio_move.mode); 1900 mmput(mm); 1901 } else { 1902 return -ESRCH; 1903 } 1904 1905 if (unlikely(put_user(ret, &user_uffdio_move->move))) 1906 return -EFAULT; 1907 if (ret < 0) 1908 goto out; 1909 1910 /* len == 0 would wake all */ 1911 VM_WARN_ON(!ret); 1912 range.len = ret; 1913 if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) { 1914 range.start = uffdio_move.dst; 1915 wake_userfault(ctx, &range); 1916 } 1917 ret = range.len == uffdio_move.len ? 0 : -EAGAIN; 1918 1919 out: 1920 return ret; 1921 } 1922 1923 /* 1924 * userland asks for a certain API version and we return which bits 1925 * and ioctl commands are implemented in this kernel for such API 1926 * version or -EINVAL if unknown. 1927 */ 1928 static int userfaultfd_api(struct userfaultfd_ctx *ctx, 1929 unsigned long arg) 1930 { 1931 struct uffdio_api uffdio_api; 1932 void __user *buf = (void __user *)arg; 1933 unsigned int ctx_features; 1934 int ret; 1935 __u64 features; 1936 1937 ret = -EFAULT; 1938 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api))) 1939 goto out; 1940 features = uffdio_api.features; 1941 ret = -EINVAL; 1942 if (uffdio_api.api != UFFD_API) 1943 goto err_out; 1944 ret = -EPERM; 1945 if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE)) 1946 goto err_out; 1947 1948 /* WP_ASYNC relies on WP_UNPOPULATED, choose it unconditionally */ 1949 if (features & UFFD_FEATURE_WP_ASYNC) 1950 features |= UFFD_FEATURE_WP_UNPOPULATED; 1951 1952 /* report all available features and ioctls to userland */ 1953 uffdio_api.features = UFFD_API_FEATURES; 1954 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 1955 uffdio_api.features &= 1956 ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); 1957 #endif 1958 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP 1959 uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; 1960 #endif 1961 #ifndef CONFIG_PTE_MARKER_UFFD_WP 1962 uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; 1963 uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; 1964 uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; 1965 #endif 1966 1967 ret = -EINVAL; 1968 if (features & ~uffdio_api.features) 1969 goto err_out; 1970 1971 uffdio_api.ioctls = UFFD_API_IOCTLS; 1972 ret = -EFAULT; 1973 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) 1974 goto out; 1975 1976 /* only enable the requested features for this uffd context */ 1977 ctx_features = uffd_ctx_features(features); 1978 ret = -EINVAL; 1979 if (cmpxchg(&ctx->features, 0, ctx_features) != 0) 1980 goto err_out; 1981 1982 ret = 0; 1983 out: 1984 return ret; 1985 err_out: 1986 memset(&uffdio_api, 0, sizeof(uffdio_api)); 1987 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) 1988 ret = -EFAULT; 1989 goto out; 1990 } 1991 1992 static long userfaultfd_ioctl(struct file *file, unsigned cmd, 1993 unsigned long arg) 1994 { 1995 int ret = -EINVAL; 1996 struct userfaultfd_ctx *ctx = file->private_data; 1997 1998 if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) 1999 return -EINVAL; 2000 2001 switch(cmd) { 2002 case UFFDIO_API: 2003 ret = userfaultfd_api(ctx, arg); 2004 break; 2005 case UFFDIO_REGISTER: 2006 ret = userfaultfd_register(ctx, arg); 2007 break; 2008 case UFFDIO_UNREGISTER: 2009 ret = userfaultfd_unregister(ctx, arg); 2010 break; 2011 case UFFDIO_WAKE: 2012 ret = userfaultfd_wake(ctx, arg); 2013 break; 2014 case UFFDIO_COPY: 2015 ret = userfaultfd_copy(ctx, arg); 2016 break; 2017 case UFFDIO_ZEROPAGE: 2018 ret = userfaultfd_zeropage(ctx, arg); 2019 break; 2020 case UFFDIO_MOVE: 2021 ret = userfaultfd_move(ctx, arg); 2022 break; 2023 case UFFDIO_WRITEPROTECT: 2024 ret = userfaultfd_writeprotect(ctx, arg); 2025 break; 2026 case UFFDIO_CONTINUE: 2027 ret = userfaultfd_continue(ctx, arg); 2028 break; 2029 case UFFDIO_POISON: 2030 ret = userfaultfd_poison(ctx, arg); 2031 break; 2032 } 2033 return ret; 2034 } 2035 2036 #ifdef CONFIG_PROC_FS 2037 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) 2038 { 2039 struct userfaultfd_ctx *ctx = f->private_data; 2040 wait_queue_entry_t *wq; 2041 unsigned long pending = 0, total = 0; 2042 2043 spin_lock_irq(&ctx->fault_pending_wqh.lock); 2044 list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) { 2045 pending++; 2046 total++; 2047 } 2048 list_for_each_entry(wq, &ctx->fault_wqh.head, entry) { 2049 total++; 2050 } 2051 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 2052 2053 /* 2054 * If more protocols will be added, there will be all shown 2055 * separated by a space. Like this: 2056 * protocols: aa:... bb:... 2057 */ 2058 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", 2059 pending, total, UFFD_API, ctx->features, 2060 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); 2061 } 2062 #endif 2063 2064 static const struct file_operations userfaultfd_fops = { 2065 #ifdef CONFIG_PROC_FS 2066 .show_fdinfo = userfaultfd_show_fdinfo, 2067 #endif 2068 .release = userfaultfd_release, 2069 .poll = userfaultfd_poll, 2070 .read_iter = userfaultfd_read_iter, 2071 .unlocked_ioctl = userfaultfd_ioctl, 2072 .compat_ioctl = compat_ptr_ioctl, 2073 .llseek = noop_llseek, 2074 }; 2075 2076 static void init_once_userfaultfd_ctx(void *mem) 2077 { 2078 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem; 2079 2080 init_waitqueue_head(&ctx->fault_pending_wqh); 2081 init_waitqueue_head(&ctx->fault_wqh); 2082 init_waitqueue_head(&ctx->event_wqh); 2083 init_waitqueue_head(&ctx->fd_wqh); 2084 seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock); 2085 } 2086 2087 static int new_userfaultfd(int flags) 2088 { 2089 struct userfaultfd_ctx *ctx; 2090 struct file *file; 2091 int fd; 2092 2093 BUG_ON(!current->mm); 2094 2095 /* Check the UFFD_* constants for consistency. */ 2096 BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS); 2097 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC); 2098 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK); 2099 2100 if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY)) 2101 return -EINVAL; 2102 2103 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); 2104 if (!ctx) 2105 return -ENOMEM; 2106 2107 refcount_set(&ctx->refcount, 1); 2108 ctx->flags = flags; 2109 ctx->features = 0; 2110 ctx->released = false; 2111 init_rwsem(&ctx->map_changing_lock); 2112 atomic_set(&ctx->mmap_changing, 0); 2113 ctx->mm = current->mm; 2114 2115 fd = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS); 2116 if (fd < 0) 2117 goto err_out; 2118 2119 /* Create a new inode so that the LSM can block the creation. */ 2120 file = anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx, 2121 O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL); 2122 if (IS_ERR(file)) { 2123 put_unused_fd(fd); 2124 fd = PTR_ERR(file); 2125 goto err_out; 2126 } 2127 /* prevent the mm struct to be freed */ 2128 mmgrab(ctx->mm); 2129 file->f_mode |= FMODE_NOWAIT; 2130 fd_install(fd, file); 2131 return fd; 2132 err_out: 2133 kmem_cache_free(userfaultfd_ctx_cachep, ctx); 2134 return fd; 2135 } 2136 2137 static inline bool userfaultfd_syscall_allowed(int flags) 2138 { 2139 /* Userspace-only page faults are always allowed */ 2140 if (flags & UFFD_USER_MODE_ONLY) 2141 return true; 2142 2143 /* 2144 * The user is requesting a userfaultfd which can handle kernel faults. 2145 * Privileged users are always allowed to do this. 2146 */ 2147 if (capable(CAP_SYS_PTRACE)) 2148 return true; 2149 2150 /* Otherwise, access to kernel fault handling is sysctl controlled. */ 2151 return sysctl_unprivileged_userfaultfd; 2152 } 2153 2154 SYSCALL_DEFINE1(userfaultfd, int, flags) 2155 { 2156 if (!userfaultfd_syscall_allowed(flags)) 2157 return -EPERM; 2158 2159 return new_userfaultfd(flags); 2160 } 2161 2162 static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags) 2163 { 2164 if (cmd != USERFAULTFD_IOC_NEW) 2165 return -EINVAL; 2166 2167 return new_userfaultfd(flags); 2168 } 2169 2170 static const struct file_operations userfaultfd_dev_fops = { 2171 .unlocked_ioctl = userfaultfd_dev_ioctl, 2172 .compat_ioctl = userfaultfd_dev_ioctl, 2173 .owner = THIS_MODULE, 2174 .llseek = noop_llseek, 2175 }; 2176 2177 static struct miscdevice userfaultfd_misc = { 2178 .minor = MISC_DYNAMIC_MINOR, 2179 .name = "userfaultfd", 2180 .fops = &userfaultfd_dev_fops 2181 }; 2182 2183 static int __init userfaultfd_init(void) 2184 { 2185 int ret; 2186 2187 ret = misc_register(&userfaultfd_misc); 2188 if (ret) 2189 return ret; 2190 2191 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", 2192 sizeof(struct userfaultfd_ctx), 2193 0, 2194 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 2195 init_once_userfaultfd_ctx); 2196 #ifdef CONFIG_SYSCTL 2197 register_sysctl_init("vm", vm_userfaultfd_table); 2198 #endif 2199 return 0; 2200 } 2201 __initcall(userfaultfd_init); 2202