73 static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_region *region,  in pds_vfio_dirty_alloc_bitmaps()  argument
88 	region->host_seq = host_seq_bmp;  in pds_vfio_dirty_alloc_bitmaps()
89 	region->host_ack = host_ack_bmp;  in pds_vfio_dirty_alloc_bitmaps()
90 	region->bmp_bytes = bytes;  in pds_vfio_dirty_alloc_bitmaps()
101 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_dirty_free_bitmaps()  local
103 		vfree(region->host_seq);  in pds_vfio_dirty_free_bitmaps()
104 		vfree(region->host_ack);  in pds_vfio_dirty_free_bitmaps()
105 		region->host_seq = NULL;  in pds_vfio_dirty_free_bitmaps()
106 		region->host_ack = NULL;  in pds_vfio_dirty_free_bitmaps()
107 		region->bmp_bytes = 0;  in pds_vfio_dirty_free_bitmaps()
112 				      struct pds_vfio_region *region)  in __pds_vfio_dirty_free_sgl()  argument
117 	dma_unmap_single(pdsc_dev, region->sgl_addr,  in __pds_vfio_dirty_free_sgl()
118 			 region->num_sge * sizeof(struct pds_lm_sg_elem),  in __pds_vfio_dirty_free_sgl()
120 	kfree(region->sgl);  in __pds_vfio_dirty_free_sgl()
122 	region->num_sge = 0;  in __pds_vfio_dirty_free_sgl()
123 	region->sgl = NULL;  in __pds_vfio_dirty_free_sgl()
124 	region->sgl_addr = 0;  in __pds_vfio_dirty_free_sgl()
135 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_dirty_free_sgl()  local
137 		if (region->sgl)  in pds_vfio_dirty_free_sgl()
138 			__pds_vfio_dirty_free_sgl(pds_vfio, region);  in pds_vfio_dirty_free_sgl()
143 				    struct pds_vfio_region *region,  in pds_vfio_dirty_alloc_sgl()  argument
166 	region->sgl = sgl;  in pds_vfio_dirty_alloc_sgl()
167 	region->num_sge = max_sge;  in pds_vfio_dirty_alloc_sgl()
168 	region->sgl_addr = sgl_addr;  in pds_vfio_dirty_alloc_sgl()
196 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_dirty_alloc_regions()  local
205 		err = pds_vfio_dirty_alloc_bitmaps(region,  in pds_vfio_dirty_alloc_regions()
213 		err = pds_vfio_dirty_alloc_sgl(pds_vfio, region, page_count);  in pds_vfio_dirty_alloc_regions()
220 		region->size = region_size;  in pds_vfio_dirty_alloc_regions()
221 		region->start = region_start;  in pds_vfio_dirty_alloc_regions()
222 		region->page_size = region_page_size;  in pds_vfio_dirty_alloc_regions()
223 		region->dev_bmp_offset_start_byte = dev_bmp_offset_byte;  in pds_vfio_dirty_alloc_regions()
368 				  struct pds_vfio_region *region,  in pds_vfio_dirty_seq_ack()  argument
423 		struct pds_lm_sg_elem *sg_elem = &region->sgl[i];  in pds_vfio_dirty_seq_ack()
431 	offset += region->dev_bmp_offset_start_byte;  in pds_vfio_dirty_seq_ack()
432 	dma_sync_single_for_device(pdsc_dev, region->sgl_addr, size, dma_dir);  in pds_vfio_dirty_seq_ack()
433 	err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, region->sgl_addr, num_sge,  in pds_vfio_dirty_seq_ack()
439 			num_sge, region->sgl_addr, ERR_PTR(err));  in pds_vfio_dirty_seq_ack()
440 	dma_sync_single_for_cpu(pdsc_dev, region->sgl_addr, size, dma_dir);  in pds_vfio_dirty_seq_ack()
452 				   struct pds_vfio_region *region,  in pds_vfio_dirty_write_ack()  argument
456 	return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_ack,  in pds_vfio_dirty_write_ack()
461 				   struct pds_vfio_region *region,  in pds_vfio_dirty_read_seq()  argument
464 	return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_seq,  in pds_vfio_dirty_read_seq()
469 					  struct pds_vfio_region *region,  in pds_vfio_dirty_process_bitmaps()  argument
473 	u64 page_size = region->page_size;  in pds_vfio_dirty_process_bitmaps()
474 	u64 region_start = region->start;  in pds_vfio_dirty_process_bitmaps()
480 	seq = (__le64 *)((u64)region->host_seq + bmp_offset);  in pds_vfio_dirty_process_bitmaps()
481 	ack = (__le64 *)((u64)region->host_ack + bmp_offset);  in pds_vfio_dirty_process_bitmaps()
510 		struct pds_vfio_region *region = &dirty->regions[i];  in pds_vfio_get_region()  local
512 		if (iova >= region->start &&  in pds_vfio_get_region()
513 		    iova < (region->start + region->size))  in pds_vfio_get_region()
514 			return region;  in pds_vfio_get_region()
525 	struct pds_vfio_region *region;  in pds_vfio_dirty_sync()  local
538 	region = pds_vfio_get_region(pds_vfio, iova);  in pds_vfio_dirty_sync()
539 	if (!region) {  in pds_vfio_dirty_sync()
540 		dev_err(dev, "vf%u: Failed to find region that contains iova 0x%lx length 0x%lx\n",  in pds_vfio_dirty_sync()
545 	pages = DIV_ROUND_UP(length, region->page_size);  in pds_vfio_dirty_sync()
551 		pds_vfio->vf_id, iova, length, region->page_size,  in pds_vfio_dirty_sync()
554 	if (!length || ((iova - region->start + length) > region->size)) {  in pds_vfio_dirty_sync()
561 	bmp_bytes = ALIGN(DIV_ROUND_UP(length / region->page_size,  in pds_vfio_dirty_sync()
570 	if (bmp_bytes > region->bmp_bytes) {  in pds_vfio_dirty_sync()
572 			"Calculated bitmap bytes %llu larger than region's cached bmp_bytes %llu\n",  in pds_vfio_dirty_sync()
573 			bmp_bytes, region->bmp_bytes);  in pds_vfio_dirty_sync()
577 	bmp_offset = DIV_ROUND_UP((iova - region->start) /  in pds_vfio_dirty_sync()
578 				  region->page_size, sizeof(u64));  in pds_vfio_dirty_sync()
584 	err = pds_vfio_dirty_read_seq(pds_vfio, region, bmp_offset, bmp_bytes);  in pds_vfio_dirty_sync()
588 	err = pds_vfio_dirty_process_bitmaps(pds_vfio, region, dirty_bitmap,  in pds_vfio_dirty_sync()
593 	err = pds_vfio_dirty_write_ack(pds_vfio, region, bmp_offset, bmp_bytes);  in pds_vfio_dirty_sync()