266 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_write_complete()  local
267 	struct page *page = sio->bvec[0].bv_page;  in sio_write_complete()
270 	if (ret != sio->len) {  in sio_write_complete()
283 		for (p = 0; p < sio->pages; p++) {  in sio_write_complete()
284 			page = sio->bvec[p].bv_page;  in sio_write_complete()
290 	for (p = 0; p < sio->pages; p++)  in sio_write_complete()
291 		end_page_writeback(sio->bvec[p].bv_page);  in sio_write_complete()
293 	mempool_free(sio, sio_pool);  in sio_write_complete()
298 	struct swap_iocb *sio = NULL;  in swap_writepage_fs()  local
307 		sio = *wbc->swap_plug;  in swap_writepage_fs()
308 	if (sio) {  in swap_writepage_fs()
309 		if (sio->iocb.ki_filp != swap_file ||  in swap_writepage_fs()
310 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_writepage_fs()
311 			swap_write_unplug(sio);  in swap_writepage_fs()
312 			sio = NULL;  in swap_writepage_fs()
315 	if (!sio) {  in swap_writepage_fs()
316 		sio = mempool_alloc(sio_pool, GFP_NOIO);  in swap_writepage_fs()
317 		init_sync_kiocb(&sio->iocb, swap_file);  in swap_writepage_fs()
318 		sio->iocb.ki_complete = sio_write_complete;  in swap_writepage_fs()
319 		sio->iocb.ki_pos = pos;  in swap_writepage_fs()
320 		sio->pages = 0;  in swap_writepage_fs()
321 		sio->len = 0;  in swap_writepage_fs()
323 	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);  in swap_writepage_fs()
324 	sio->len += folio_size(folio);  in swap_writepage_fs()
325 	sio->pages += 1;  in swap_writepage_fs()
326 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !wbc->swap_plug) {  in swap_writepage_fs()
327 		swap_write_unplug(sio);  in swap_writepage_fs()
328 		sio = NULL;  in swap_writepage_fs()
331 		*wbc->swap_plug = sio;  in swap_writepage_fs()
392 void swap_write_unplug(struct swap_iocb *sio)  in swap_write_unplug()  argument
395 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in swap_write_unplug()
398 	iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);  in swap_write_unplug()
399 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in swap_write_unplug()
401 		sio_write_complete(&sio->iocb, ret);  in swap_write_unplug()
406 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);  in sio_read_complete()  local
409 	if (ret == sio->len) {  in sio_read_complete()
410 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
411 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
416 		count_vm_events(PSWPIN, sio->pages);  in sio_read_complete()
418 		for (p = 0; p < sio->pages; p++) {  in sio_read_complete()
419 			struct folio *folio = page_folio(sio->bvec[p].bv_page);  in sio_read_complete()
425 	mempool_free(sio, sio_pool);  in sio_read_complete()
431 	struct swap_iocb *sio = NULL;  in swap_read_folio_fs()  local
435 		sio = *plug;  in swap_read_folio_fs()
436 	if (sio) {  in swap_read_folio_fs()
437 		if (sio->iocb.ki_filp != sis->swap_file ||  in swap_read_folio_fs()
438 		    sio->iocb.ki_pos + sio->len != pos) {  in swap_read_folio_fs()
439 			swap_read_unplug(sio);  in swap_read_folio_fs()
440 			sio = NULL;  in swap_read_folio_fs()
443 	if (!sio) {  in swap_read_folio_fs()
444 		sio = mempool_alloc(sio_pool, GFP_KERNEL);  in swap_read_folio_fs()
445 		init_sync_kiocb(&sio->iocb, sis->swap_file);  in swap_read_folio_fs()
446 		sio->iocb.ki_pos = pos;  in swap_read_folio_fs()
447 		sio->iocb.ki_complete = sio_read_complete;  in swap_read_folio_fs()
448 		sio->pages = 0;  in swap_read_folio_fs()
449 		sio->len = 0;  in swap_read_folio_fs()
451 	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);  in swap_read_folio_fs()
452 	sio->len += folio_size(folio);  in swap_read_folio_fs()
453 	sio->pages += 1;  in swap_read_folio_fs()
454 	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {  in swap_read_folio_fs()
455 		swap_read_unplug(sio);  in swap_read_folio_fs()
456 		sio = NULL;  in swap_read_folio_fs()
459 		*plug = sio;  in swap_read_folio_fs()
536 void __swap_read_unplug(struct swap_iocb *sio)  in __swap_read_unplug()  argument
539 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;  in __swap_read_unplug()
542 	iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);  in __swap_read_unplug()
543 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);  in __swap_read_unplug()
545 		sio_read_complete(&sio->iocb, ret);  in __swap_read_unplug()