Lines Matching +full:send +full:- +full:migration
1 // SPDX-License-Identifier: GPL-2.0-or-later
59 /* Balloon's own wq for cpu-intensive work items */
97 * at vb_dev_info->pages list.
113 /* Shrinker to return free pages - VIRTIO_BALLOON_F_FREE_PAGE_HINT */
116 /* OOM notifier to deflate on OOM - VIRTIO_BALLOON_F_DEFLATE_ON_OOM */
150 spin_lock_irqsave(&vb->wakeup_lock, flags); in start_wakeup_event()
151 vb->wakeup_signal_mask |= mask; in start_wakeup_event()
152 if (!vb->processing_wakeup_event) { in start_wakeup_event()
153 vb->processing_wakeup_event = true; in start_wakeup_event()
154 pm_stay_awake(&vb->vdev->dev); in start_wakeup_event()
156 spin_unlock_irqrestore(&vb->wakeup_lock, flags); in start_wakeup_event()
161 spin_lock_irq(&vb->wakeup_lock); in process_wakeup_event()
162 vb->wakeup_signal_mask &= ~mask; in process_wakeup_event()
163 spin_unlock_irq(&vb->wakeup_lock); in process_wakeup_event()
168 spin_lock_irq(&vb->wakeup_lock); in finish_wakeup_event()
169 if (!vb->wakeup_signal_mask && vb->processing_wakeup_event) { in finish_wakeup_event()
170 vb->processing_wakeup_event = false; in finish_wakeup_event()
171 pm_relax(&vb->vdev->dev); in finish_wakeup_event()
173 spin_unlock_irq(&vb->wakeup_lock); in finish_wakeup_event()
178 struct virtio_balloon *vb = vq->vdev->priv; in balloon_ack()
180 wake_up(&vb->acked); in balloon_ack()
188 sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns); in tell_host()
195 wait_event(vb->acked, virtqueue_get_buf(vq, &len)); in tell_host()
204 struct virtqueue *vq = vb->reporting_vq; in virtballoon_free_page_report()
221 wait_event(vb->acked, virtqueue_get_buf(vq, &unused)); in virtballoon_free_page_report()
238 pfns[i] = cpu_to_virtio32(vb->vdev, in set_page_pfns()
250 num = min(num, ARRAY_SIZE(vb->pfns)); in fill_balloon()
257 dev_info_ratelimited(&vb->vdev->dev, in fill_balloon()
268 mutex_lock(&vb->balloon_lock); in fill_balloon()
270 vb->num_pfns = 0; in fill_balloon()
273 balloon_page_enqueue(&vb->vb_dev_info, page); in fill_balloon()
275 set_page_pfns(vb, vb->pfns + vb->num_pfns, page); in fill_balloon()
276 vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; in fill_balloon()
277 if (!virtio_has_feature(vb->vdev, in fill_balloon()
279 adjust_managed_page_count(page, -1); in fill_balloon()
280 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE; in fill_balloon()
283 num_allocated_pages = vb->num_pfns; in fill_balloon()
285 if (vb->num_pfns != 0) in fill_balloon()
286 tell_host(vb, vb->inflate_vq); in fill_balloon()
287 mutex_unlock(&vb->balloon_lock); in fill_balloon()
298 if (!virtio_has_feature(vb->vdev, in release_pages_balloon()
301 list_del(&page->lru); in release_pages_balloon()
310 struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; in leak_balloon()
314 num = min(num, ARRAY_SIZE(vb->pfns)); in leak_balloon()
316 mutex_lock(&vb->balloon_lock); in leak_balloon()
318 num = min(num, (size_t)vb->num_pages); in leak_balloon()
319 for (vb->num_pfns = 0; vb->num_pfns < num; in leak_balloon()
320 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { in leak_balloon()
324 set_page_pfns(vb, vb->pfns + vb->num_pfns, page); in leak_balloon()
325 list_add(&page->lru, &pages); in leak_balloon()
326 vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE; in leak_balloon()
329 num_freed_pages = vb->num_pfns; in leak_balloon()
335 if (vb->num_pfns != 0) in leak_balloon()
336 tell_host(vb, vb->deflate_vq); in leak_balloon()
338 mutex_unlock(&vb->balloon_lock); in leak_balloon()
346 vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag); in update_stat()
347 vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val); in update_stat()
372 stall += events[ALLOCSTALL_NORMAL - ZONE_NORMAL + zid]; in update_balloon_vm_stats()
426 * While most virtqueues communicate guest-initiated requests to the hypervisor,
436 struct virtio_balloon *vb = vq->vdev->priv; in stats_request()
438 spin_lock(&vb->stop_update_lock); in stats_request()
439 if (!vb->stop_update) { in stats_request()
441 queue_work(system_freezable_wq, &vb->update_balloon_stats_work); in stats_request()
443 spin_unlock(&vb->stop_update_lock); in stats_request()
454 vq = vb->stats_vq; in stats_handle_request()
457 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); in stats_handle_request()
468 virtio_cread_le(vb->vdev, struct virtio_balloon_config, num_pages, in towards_target()
476 return target - vb->num_pages; in towards_target()
486 spin_lock_irq(&vb->free_page_list_lock); in return_free_pages_to_mm()
488 page = balloon_page_pop(&vb->free_page_list); in return_free_pages_to_mm()
494 vb->num_free_page_blocks -= num_returned; in return_free_pages_to_mm()
495 spin_unlock_irq(&vb->free_page_list_lock); in return_free_pages_to_mm()
502 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtio_balloon_queue_free_page_work()
507 &vb->config_read_bitmap)) in virtio_balloon_queue_free_page_work()
510 queue_work(vb->balloon_wq, &vb->report_free_page_work); in virtio_balloon_queue_free_page_work()
516 queue_work(system_freezable_wq, &vb->update_balloon_size_work); in start_update_balloon_size()
521 struct virtio_balloon *vb = vdev->priv; in virtballoon_changed()
524 spin_lock_irqsave(&vb->stop_update_lock, flags); in virtballoon_changed()
525 if (!vb->stop_update) { in virtballoon_changed()
529 spin_unlock_irqrestore(&vb->stop_update_lock, flags); in virtballoon_changed()
534 u32 actual = vb->num_pages; in update_balloon_size()
537 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, actual, in update_balloon_size()
567 diff -= fill_balloon(vb, diff); in update_balloon_size_func()
569 diff += leak_balloon(vb, -diff); in update_balloon_size_func()
595 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { in init_vqs()
600 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in init_vqs()
603 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { in init_vqs()
608 err = virtio_find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs, in init_vqs()
613 vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE]; in init_vqs()
614 vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE]; in init_vqs()
615 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { in init_vqs()
618 vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS]; in init_vqs()
626 sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); in init_vqs()
627 err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, in init_vqs()
630 dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n", in init_vqs()
634 virtqueue_kick(vb->stats_vq); in init_vqs()
637 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in init_vqs()
638 vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE]; in init_vqs()
640 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) in init_vqs()
641 vb->reporting_vq = vqs[VIRTIO_BALLOON_VQ_REPORTING]; in init_vqs()
649 &vb->config_read_bitmap)) { in virtio_balloon_cmd_id_received()
651 virtio_cread_le(vb->vdev, struct virtio_balloon_config, in virtio_balloon_cmd_id_received()
653 &vb->cmd_id_received_cache); in virtio_balloon_cmd_id_received()
656 return vb->cmd_id_received_cache; in virtio_balloon_cmd_id_received()
662 struct virtqueue *vq = vb->free_page_vq; in send_cmd_id_start()
669 vb->cmd_id_active = cpu_to_virtio32(vb->vdev, in send_cmd_id_start()
671 sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active)); in send_cmd_id_start()
672 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL); in send_cmd_id_start()
681 struct virtqueue *vq = vb->free_page_vq; in send_cmd_id_stop()
688 sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop)); in send_cmd_id_stop()
689 err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL); in send_cmd_id_stop()
697 struct virtqueue *vq = vb->free_page_vq; in get_free_page_and_send()
711 * the possible free pages, so return -EINTR to stop. in get_free_page_and_send()
714 return -EINTR; in get_free_page_and_send()
719 if (vq->num_free > 1) { in get_free_page_and_send()
727 spin_lock_irq(&vb->free_page_list_lock); in get_free_page_and_send()
728 balloon_page_push(&vb->free_page_list, page); in get_free_page_and_send()
729 vb->num_free_page_blocks++; in get_free_page_and_send()
730 spin_unlock_irq(&vb->free_page_list_lock); in get_free_page_and_send()
752 cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active); in send_free_pages()
762 if (err == -EINTR) in send_free_pages()
774 struct device *dev = &vb->vdev->dev; in virtio_balloon_report_free_page()
779 dev_err(dev, "Failed to send a start id, err = %d\n", err); in virtio_balloon_report_free_page()
783 dev_err(dev, "Failed to send a free page, err = %d\n", err); in virtio_balloon_report_free_page()
788 dev_err(dev, "Failed to send a stop id, err = %d\n", err); in virtio_balloon_report_free_page()
803 virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) { in report_free_page_func()
810 * virtballoon_migratepage - perform the balloon page migration on behalf of
813 * @newpage: page that will replace the isolated page after migration finishes.
815 * @mode : compaction mode -- not used for balloon page migration.
818 * function that performs the page migration on behalf of a compaction thread
819 * The page migration for virtio balloon is done in a simple swap fashion which
821 * 1) insert newpage into vb->pages list and update the host about it;
822 * 2) update the host about the old page removed from vb->pages list;
824 * This function preforms the balloon page migration task.
825 * Called through movable_operations->migrate_page
837 * this turn, as it is easier to retry the page migration later. in virtballoon_migratepage()
842 if (!mutex_trylock(&vb->balloon_lock)) in virtballoon_migratepage()
843 return -EAGAIN; in virtballoon_migratepage()
852 if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) && in virtballoon_migratepage()
855 adjust_managed_page_count(newpage, -1); in virtballoon_migratepage()
858 /* balloon's page migration 1st step -- inflate "newpage" */ in virtballoon_migratepage()
859 spin_lock_irqsave(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
861 vb_dev_info->isolated_pages--; in virtballoon_migratepage()
863 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
864 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; in virtballoon_migratepage()
865 set_page_pfns(vb, vb->pfns, newpage); in virtballoon_migratepage()
866 tell_host(vb, vb->inflate_vq); in virtballoon_migratepage()
868 /* balloon's page migration 2nd step -- deflate "page" */ in virtballoon_migratepage()
869 spin_lock_irqsave(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
871 spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); in virtballoon_migratepage()
872 vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; in virtballoon_migratepage()
873 set_page_pfns(vb, vb->pfns, page); in virtballoon_migratepage()
874 tell_host(vb, vb->deflate_vq); in virtballoon_migratepage()
876 mutex_unlock(&vb->balloon_lock); in virtballoon_migratepage()
900 struct virtio_balloon *vb = shrinker->private_data; in virtio_balloon_shrinker_scan()
902 return shrink_free_pages(vb, sc->nr_to_scan); in virtio_balloon_shrinker_scan()
908 struct virtio_balloon *vb = shrinker->private_data; in virtio_balloon_shrinker_count()
910 return vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; in virtio_balloon_shrinker_count()
929 shrinker_free(vb->shrinker); in virtio_balloon_unregister_shrinker()
934 vb->shrinker = shrinker_alloc(0, "virtio-balloon"); in virtio_balloon_register_shrinker()
935 if (!vb->shrinker) in virtio_balloon_register_shrinker()
936 return -ENOMEM; in virtio_balloon_register_shrinker()
938 vb->shrinker->scan_objects = virtio_balloon_shrinker_scan; in virtio_balloon_register_shrinker()
939 vb->shrinker->count_objects = virtio_balloon_shrinker_count; in virtio_balloon_register_shrinker()
940 vb->shrinker->private_data = vb; in virtio_balloon_register_shrinker()
942 shrinker_register(vb->shrinker); in virtio_balloon_register_shrinker()
952 if (!vdev->config->get) { in virtballoon_probe()
953 dev_err(&vdev->dev, "%s failure: config access disabled\n", in virtballoon_probe()
955 return -EINVAL; in virtballoon_probe()
958 vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL); in virtballoon_probe()
960 err = -ENOMEM; in virtballoon_probe()
964 INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func); in virtballoon_probe()
965 INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func); in virtballoon_probe()
966 spin_lock_init(&vb->stop_update_lock); in virtballoon_probe()
967 mutex_init(&vb->balloon_lock); in virtballoon_probe()
968 init_waitqueue_head(&vb->acked); in virtballoon_probe()
969 vb->vdev = vdev; in virtballoon_probe()
971 balloon_devinfo_init(&vb->vb_dev_info); in virtballoon_probe()
978 vb->vb_dev_info.migratepage = virtballoon_migratepage; in virtballoon_probe()
985 if (virtqueue_get_vring_size(vb->free_page_vq) < 2) { in virtballoon_probe()
986 err = -ENOSPC; in virtballoon_probe()
989 vb->balloon_wq = alloc_workqueue("balloon-wq", in virtballoon_probe()
991 if (!vb->balloon_wq) { in virtballoon_probe()
992 err = -ENOMEM; in virtballoon_probe()
995 INIT_WORK(&vb->report_free_page_work, report_free_page_func); in virtballoon_probe()
996 vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP; in virtballoon_probe()
997 vb->cmd_id_active = cpu_to_virtio32(vb->vdev, in virtballoon_probe()
999 vb->cmd_id_stop = cpu_to_virtio32(vb->vdev, in virtballoon_probe()
1001 spin_lock_init(&vb->free_page_list_lock); in virtballoon_probe()
1002 INIT_LIST_HEAD(&vb->free_page_list); in virtballoon_probe()
1012 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { in virtballoon_probe()
1013 vb->oom_nb.notifier_call = virtio_balloon_oom_notify; in virtballoon_probe()
1014 vb->oom_nb.priority = VIRTIO_BALLOON_OOM_NOTIFY_PRIORITY; in virtballoon_probe()
1015 err = register_oom_notifier(&vb->oom_nb); in virtballoon_probe()
1030 * little-endian. However for now it is a single byte so we in virtballoon_probe()
1031 * can pass it as-is. in virtballoon_probe()
1036 virtio_cwrite_le(vb->vdev, struct virtio_balloon_config, in virtballoon_probe()
1040 vb->pr_dev_info.report = virtballoon_free_page_report; in virtballoon_probe()
1041 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) { in virtballoon_probe()
1044 capacity = virtqueue_get_vring_size(vb->reporting_vq); in virtballoon_probe()
1046 err = -ENOSPC; in virtballoon_probe()
1061 * that value. The hard-coded order would be fine currently. in virtballoon_probe()
1064 vb->pr_dev_info.order = 5; in virtballoon_probe()
1067 err = page_reporting_register(&vb->pr_dev_info); in virtballoon_probe()
1072 spin_lock_init(&vb->wakeup_lock); in virtballoon_probe()
1081 device_set_wakeup_capable(&vb->vdev->dev, true); in virtballoon_probe()
1090 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) in virtballoon_probe()
1091 unregister_oom_notifier(&vb->oom_nb); in virtballoon_probe()
1093 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtballoon_probe()
1097 destroy_workqueue(vb->balloon_wq); in virtballoon_probe()
1099 vdev->config->del_vqs(vdev); in virtballoon_probe()
1109 while (vb->num_pages) in remove_common()
1110 leak_balloon(vb, vb->num_pages); in remove_common()
1114 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in remove_common()
1118 virtio_reset_device(vb->vdev); in remove_common()
1120 vb->vdev->config->del_vqs(vb->vdev); in remove_common()
1125 struct virtio_balloon *vb = vdev->priv; in virtballoon_remove()
1127 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) in virtballoon_remove()
1128 page_reporting_unregister(&vb->pr_dev_info); in virtballoon_remove()
1129 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) in virtballoon_remove()
1130 unregister_oom_notifier(&vb->oom_nb); in virtballoon_remove()
1131 if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) in virtballoon_remove()
1133 spin_lock_irq(&vb->stop_update_lock); in virtballoon_remove()
1134 vb->stop_update = true; in virtballoon_remove()
1135 spin_unlock_irq(&vb->stop_update_lock); in virtballoon_remove()
1136 cancel_work_sync(&vb->update_balloon_size_work); in virtballoon_remove()
1137 cancel_work_sync(&vb->update_balloon_stats_work); in virtballoon_remove()
1140 cancel_work_sync(&vb->report_free_page_work); in virtballoon_remove()
1141 destroy_workqueue(vb->balloon_wq); in virtballoon_remove()
1151 struct virtio_balloon *vb = vdev->priv; in virtballoon_freeze()
1163 struct virtio_balloon *vb = vdev->priv; in virtballoon_restore()
1166 ret = init_vqs(vdev->priv); in virtballoon_restore()