91 	struct dmirror_device		*mdevice;  member
103 	struct dmirror_device	*mdevice;  member
142 static bool dmirror_is_private_zone(struct dmirror_device *mdevice)  in dmirror_is_private_zone()  argument
144 	return (mdevice->zone_device_type ==  in dmirror_is_private_zone()
151 	return (dmirror->mdevice->zone_device_type ==  in dmirror_select_device()
173 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);  in dmirror_fops_open()
207 	return dmirror_page_to_chunk(page)->mdevice;  in dmirror_page_to_device()
271 	    range->owner == dmirror->mdevice)  in dmirror_interval_invalidate()
342 		.dev_private_owner = dmirror->mdevice,  in dmirror_fault()
496 static int dmirror_allocate_chunk(struct dmirror_device *mdevice,  in dmirror_allocate_chunk()  argument
511 	switch (mdevice->zone_device_type) {  in dmirror_allocate_chunk()
522 		devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ?  in dmirror_allocate_chunk()
536 	devmem->pagemap.owner = mdevice;  in dmirror_allocate_chunk()
538 	mutex_lock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
540 	if (mdevice->devmem_count == mdevice->devmem_capacity) {  in dmirror_allocate_chunk()
544 		new_capacity = mdevice->devmem_capacity +  in dmirror_allocate_chunk()
546 		new_chunks = krealloc(mdevice->devmem_chunks,  in dmirror_allocate_chunk()
551 		mdevice->devmem_capacity = new_capacity;  in dmirror_allocate_chunk()
552 		mdevice->devmem_chunks = new_chunks;  in dmirror_allocate_chunk()
563 	devmem->mdevice = mdevice;  in dmirror_allocate_chunk()
566 	mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;  in dmirror_allocate_chunk()
568 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
572 		mdevice->devmem_count,  in dmirror_allocate_chunk()
573 		mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),  in dmirror_allocate_chunk()
576 	spin_lock(&mdevice->lock);  in dmirror_allocate_chunk()
582 			page->zone_device_data = mdevice->free_folios;  in dmirror_allocate_chunk()
583 			mdevice->free_folios = page_folio(page);  in dmirror_allocate_chunk()
588 		page->zone_device_data = mdevice->free_pages;  in dmirror_allocate_chunk()
589 		mdevice->free_pages = page;  in dmirror_allocate_chunk()
596 			if (!mdevice->free_folios) {  in dmirror_allocate_chunk()
600 			*ppage = folio_page(mdevice->free_folios, 0);  in dmirror_allocate_chunk()
601 			mdevice->free_folios = (*ppage)->zone_device_data;  in dmirror_allocate_chunk()
602 			mdevice->calloc += HPAGE_PMD_NR;  in dmirror_allocate_chunk()
603 		} else if (mdevice->free_pages) {  in dmirror_allocate_chunk()
604 			*ppage = mdevice->free_pages;  in dmirror_allocate_chunk()
605 			mdevice->free_pages = (*ppage)->zone_device_data;  in dmirror_allocate_chunk()
606 			mdevice->calloc++;  in dmirror_allocate_chunk()
613 	spin_unlock(&mdevice->lock);  in dmirror_allocate_chunk()
618 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
634 	struct dmirror_device *mdevice = dmirror->mdevice;  in dmirror_devmem_alloc_page()  local
642 	if (dmirror_is_private_zone(mdevice)) {  in dmirror_devmem_alloc_page()
647 	spin_lock(&mdevice->lock);  in dmirror_devmem_alloc_page()
649 	if (is_large && mdevice->free_folios) {  in dmirror_devmem_alloc_page()
650 		dpage = folio_page(mdevice->free_folios, 0);  in dmirror_devmem_alloc_page()
651 		mdevice->free_folios = dpage->zone_device_data;  in dmirror_devmem_alloc_page()
652 		mdevice->calloc += 1 << order;  in dmirror_devmem_alloc_page()
653 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
654 	} else if (!is_large && mdevice->free_pages) {  in dmirror_devmem_alloc_page()
655 		dpage = mdevice->free_pages;  in dmirror_devmem_alloc_page()
656 		mdevice->free_pages = dpage->zone_device_data;  in dmirror_devmem_alloc_page()
657 		mdevice->calloc++;  in dmirror_devmem_alloc_page()
658 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
660 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
661 		if (dmirror_allocate_chunk(mdevice, &dpage, is_large))  in dmirror_devmem_alloc_page()
1081 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_system()
1155 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_device()
1216 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1222 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1337 		.dev_private_owner = dmirror->mdevice,  in dmirror_snapshot()
1433 	struct dmirror_device *mdevice = devmem->mdevice;  in dmirror_remove_free_pages()  local
1438 	for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio))  in dmirror_remove_free_pages()
1440 			mdevice->free_folios = folio_zone_device_data(folio);  in dmirror_remove_free_pages()
1441 	for (page = mdevice->free_pages; page; page = page->zone_device_data)  in dmirror_remove_free_pages()
1443 			mdevice->free_pages = page->zone_device_data;  in dmirror_remove_free_pages()
1446 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)  in dmirror_device_remove_chunks()  argument
1450 	mutex_lock(&mdevice->devmem_lock);  in dmirror_device_remove_chunks()
1451 	if (mdevice->devmem_chunks) {  in dmirror_device_remove_chunks()
1452 		for (i = 0; i < mdevice->devmem_count; i++) {  in dmirror_device_remove_chunks()
1454 				mdevice->devmem_chunks[i];  in dmirror_device_remove_chunks()
1456 			spin_lock(&mdevice->lock);  in dmirror_device_remove_chunks()
1459 			spin_unlock(&mdevice->lock);  in dmirror_device_remove_chunks()
1468 		mdevice->devmem_count = 0;  in dmirror_device_remove_chunks()
1469 		mdevice->devmem_capacity = 0;  in dmirror_device_remove_chunks()
1470 		mdevice->free_pages = NULL;  in dmirror_device_remove_chunks()
1471 		mdevice->free_folios = NULL;  in dmirror_device_remove_chunks()
1472 		kfree(mdevice->devmem_chunks);  in dmirror_device_remove_chunks()
1473 		mdevice->devmem_chunks = NULL;  in dmirror_device_remove_chunks()
1475 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_device_remove_chunks()
1533 		dmirror_device_remove_chunks(dmirror->mdevice);  in dmirror_fops_unlocked_ioctl()
1589 	struct dmirror_device *mdevice;  in dmirror_devmem_free()  local
1601 	mdevice = dmirror_page_to_device(page);  in dmirror_devmem_free()
1602 	spin_lock(&mdevice->lock);  in dmirror_devmem_free()
1606 		mdevice->cfree += 1 << order;  in dmirror_devmem_free()
1608 			page->zone_device_data = mdevice->free_folios;  in dmirror_devmem_free()
1609 			mdevice->free_folios = page_folio(page);  in dmirror_devmem_free()
1611 			page->zone_device_data = mdevice->free_pages;  in dmirror_devmem_free()
1612 			mdevice->free_pages = page;  in dmirror_devmem_free()
1615 	spin_unlock(&mdevice->lock);  in dmirror_devmem_free()
1658 	args.pgmap_owner = dmirror->mdevice;  in dmirror_devmem_fault()
1729 static int dmirror_device_init(struct dmirror_device *mdevice, int id)  in dmirror_device_init()  argument
1735 	mutex_init(&mdevice->devmem_lock);  in dmirror_device_init()
1736 	spin_lock_init(&mdevice->lock);  in dmirror_device_init()
1738 	cdev_init(&mdevice->cdevice, &dmirror_fops);  in dmirror_device_init()
1739 	mdevice->cdevice.owner = THIS_MODULE;  in dmirror_device_init()
1740 	device_initialize(&mdevice->device);  in dmirror_device_init()
1741 	mdevice->device.devt = dev;  in dmirror_device_init()
1743 	ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);  in dmirror_device_init()
1747 	ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);  in dmirror_device_init()
1752 	return dmirror_allocate_chunk(mdevice, NULL, false);  in dmirror_device_init()
1755 	put_device(&mdevice->device);  in dmirror_device_init()
1759 static void dmirror_device_remove(struct dmirror_device *mdevice)  in dmirror_device_remove()  argument
1761 	dmirror_device_remove_chunks(mdevice);  in dmirror_device_remove()
1762 	cdev_device_del(&mdevice->cdevice, &mdevice->device);  in dmirror_device_remove()
1763 	put_device(&mdevice->device);  in dmirror_device_remove()