Lines Matching refs:ceph_wbc
730 struct ceph_writeback_ctl ceph_wbc; in write_folio_nounlock() local
749 oldest = get_oldest_context(inode, &ceph_wbc, snapc); in write_folio_nounlock()
762 if (page_off >= ceph_wbc.i_size) { in write_folio_nounlock()
764 ceph_vinop(inode), folio->index, ceph_wbc.i_size); in write_folio_nounlock()
769 if (ceph_wbc.i_size < page_off + len) in write_folio_nounlock()
770 len = ceph_wbc.i_size - page_off; in write_folio_nounlock()
784 ceph_wbc.truncate_seq, in write_folio_nounlock()
785 ceph_wbc.truncate_size, true); in write_folio_nounlock()
1013 void ceph_folio_batch_init(struct ceph_writeback_ctl *ceph_wbc) in ceph_folio_batch_init() argument
1015 folio_batch_init(&ceph_wbc->fbatch); in ceph_folio_batch_init()
1016 ceph_wbc->processed_in_fbatch = 0; in ceph_folio_batch_init()
1020 void ceph_folio_batch_reinit(struct ceph_writeback_ctl *ceph_wbc) in ceph_folio_batch_reinit() argument
1022 folio_batch_release(&ceph_wbc->fbatch); in ceph_folio_batch_reinit()
1023 ceph_folio_batch_init(ceph_wbc); in ceph_folio_batch_reinit()
1029 struct ceph_writeback_ctl *ceph_wbc) in ceph_init_writeback_ctl() argument
1031 ceph_wbc->snapc = NULL; in ceph_init_writeback_ctl()
1032 ceph_wbc->last_snapc = NULL; in ceph_init_writeback_ctl()
1034 ceph_wbc->strip_unit_end = 0; in ceph_init_writeback_ctl()
1035 ceph_wbc->wsize = ceph_define_write_size(mapping); in ceph_init_writeback_ctl()
1037 ceph_wbc->nr_folios = 0; in ceph_init_writeback_ctl()
1038 ceph_wbc->max_pages = 0; in ceph_init_writeback_ctl()
1039 ceph_wbc->locked_pages = 0; in ceph_init_writeback_ctl()
1041 ceph_wbc->done = false; in ceph_init_writeback_ctl()
1042 ceph_wbc->should_loop = false; in ceph_init_writeback_ctl()
1043 ceph_wbc->range_whole = false; in ceph_init_writeback_ctl()
1045 ceph_wbc->start_index = wbc->range_cyclic ? mapping->writeback_index : 0; in ceph_init_writeback_ctl()
1046 ceph_wbc->index = ceph_wbc->start_index; in ceph_init_writeback_ctl()
1047 ceph_wbc->end = -1; in ceph_init_writeback_ctl()
1049 ceph_wbc->tag = wbc_to_tag(wbc); in ceph_init_writeback_ctl()
1051 ceph_wbc->op_idx = -1; in ceph_init_writeback_ctl()
1052 ceph_wbc->num_ops = 0; in ceph_init_writeback_ctl()
1053 ceph_wbc->offset = 0; in ceph_init_writeback_ctl()
1054 ceph_wbc->len = 0; in ceph_init_writeback_ctl()
1055 ceph_wbc->from_pool = false; in ceph_init_writeback_ctl()
1057 ceph_folio_batch_init(ceph_wbc); in ceph_init_writeback_ctl()
1059 ceph_wbc->pages = NULL; in ceph_init_writeback_ctl()
1060 ceph_wbc->data_pages = NULL; in ceph_init_writeback_ctl()
1066 struct ceph_writeback_ctl *ceph_wbc) in ceph_define_writeback_range() argument
1073 ceph_wbc->snapc = get_oldest_context(inode, ceph_wbc, NULL); in ceph_define_writeback_range()
1074 if (!ceph_wbc->snapc) { in ceph_define_writeback_range()
1082 ceph_wbc->snapc, ceph_wbc->snapc->seq, in ceph_define_writeback_range()
1083 ceph_wbc->snapc->num_snaps); in ceph_define_writeback_range()
1085 ceph_wbc->should_loop = false; in ceph_define_writeback_range()
1087 if (ceph_wbc->head_snapc && ceph_wbc->snapc != ceph_wbc->last_snapc) { in ceph_define_writeback_range()
1090 ceph_wbc->index = ceph_wbc->start_index; in ceph_define_writeback_range()
1091 ceph_wbc->end = -1; in ceph_define_writeback_range()
1092 if (ceph_wbc->index > 0) in ceph_define_writeback_range()
1093 ceph_wbc->should_loop = true; in ceph_define_writeback_range()
1094 doutc(cl, " cyclic, start at %lu\n", ceph_wbc->index); in ceph_define_writeback_range()
1096 ceph_wbc->index = wbc->range_start >> PAGE_SHIFT; in ceph_define_writeback_range()
1097 ceph_wbc->end = wbc->range_end >> PAGE_SHIFT; in ceph_define_writeback_range()
1099 ceph_wbc->range_whole = true; in ceph_define_writeback_range()
1101 ceph_wbc->index, ceph_wbc->end); in ceph_define_writeback_range()
1103 } else if (!ceph_wbc->head_snapc) { in ceph_define_writeback_range()
1108 if (ceph_wbc->index > 0) in ceph_define_writeback_range()
1109 ceph_wbc->should_loop = true; in ceph_define_writeback_range()
1113 ceph_put_snap_context(ceph_wbc->last_snapc); in ceph_define_writeback_range()
1114 ceph_wbc->last_snapc = ceph_wbc->snapc; in ceph_define_writeback_range()
1120 bool has_writeback_done(struct ceph_writeback_ctl *ceph_wbc) in has_writeback_done() argument
1122 return ceph_wbc->done && ceph_wbc->index > ceph_wbc->end; in has_writeback_done()
1126 bool can_next_page_be_processed(struct ceph_writeback_ctl *ceph_wbc, in can_next_page_be_processed() argument
1129 return index < ceph_wbc->nr_folios && in can_next_page_be_processed()
1130 ceph_wbc->locked_pages < ceph_wbc->max_pages; in can_next_page_be_processed()
1136 struct ceph_writeback_ctl *ceph_wbc, in ceph_check_page_before_write() argument
1152 if (pgsnapc != ceph_wbc->snapc) { in ceph_check_page_before_write()
1155 ceph_wbc->snapc, ceph_wbc->snapc->seq); in ceph_check_page_before_write()
1157 if (!ceph_wbc->should_loop && !ceph_wbc->head_snapc && in ceph_check_page_before_write()
1159 ceph_wbc->should_loop = true; in ceph_check_page_before_write()
1164 if (folio_pos(folio) >= ceph_wbc->i_size) { in ceph_check_page_before_write()
1166 folio->index, ceph_wbc->i_size); in ceph_check_page_before_write()
1168 if ((ceph_wbc->size_stable || in ceph_check_page_before_write()
1176 if (ceph_wbc->strip_unit_end && in ceph_check_page_before_write()
1177 (folio->index > ceph_wbc->strip_unit_end)) { in ceph_check_page_before_write()
1186 void __ceph_allocate_page_array(struct ceph_writeback_ctl *ceph_wbc, in __ceph_allocate_page_array() argument
1189 ceph_wbc->pages = kmalloc_objs(*ceph_wbc->pages, max_pages, GFP_NOFS); in __ceph_allocate_page_array()
1190 if (!ceph_wbc->pages) { in __ceph_allocate_page_array()
1191 ceph_wbc->from_pool = true; in __ceph_allocate_page_array()
1192 ceph_wbc->pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); in __ceph_allocate_page_array()
1193 BUG_ON(!ceph_wbc->pages); in __ceph_allocate_page_array()
1199 struct ceph_writeback_ctl *ceph_wbc, in ceph_allocate_page_array() argument
1209 ceph_wbc->offset = (u64)folio_pos(folio); in ceph_allocate_page_array()
1211 ceph_wbc->offset, ceph_wbc->wsize, in ceph_allocate_page_array()
1214 ceph_wbc->num_ops = 1; in ceph_allocate_page_array()
1215 ceph_wbc->strip_unit_end = folio->index + ((xlen - 1) >> PAGE_SHIFT); in ceph_allocate_page_array()
1217 BUG_ON(ceph_wbc->pages); in ceph_allocate_page_array()
1218 ceph_wbc->max_pages = calc_pages_for(0, (u64)xlen); in ceph_allocate_page_array()
1219 __ceph_allocate_page_array(ceph_wbc, ceph_wbc->max_pages); in ceph_allocate_page_array()
1221 ceph_wbc->len = 0; in ceph_allocate_page_array()
1225 bool is_folio_index_contiguous(const struct ceph_writeback_ctl *ceph_wbc, in is_folio_index_contiguous() argument
1228 return folio->index == (ceph_wbc->offset + ceph_wbc->len) >> PAGE_SHIFT; in is_folio_index_contiguous()
1232 bool is_num_ops_too_big(struct ceph_writeback_ctl *ceph_wbc) in is_num_ops_too_big() argument
1234 return ceph_wbc->num_ops >= in is_num_ops_too_big()
1235 (ceph_wbc->from_pool ? CEPH_OSD_SLAB_OPS : CEPH_OSD_MAX_OPS); in is_num_ops_too_big()
1247 struct ceph_writeback_ctl *ceph_wbc, struct folio *folio) in move_dirty_folio_in_page_array() argument
1252 struct page **pages = ceph_wbc->pages; in move_dirty_folio_in_page_array()
1253 unsigned int index = ceph_wbc->locked_pages; in move_dirty_folio_in_page_array()
1254 gfp_t gfp_flags = ceph_wbc->locked_pages ? GFP_NOWAIT : GFP_NOFS; in move_dirty_folio_in_page_array()
1270 BUG_ON(ceph_wbc->locked_pages == 0); in move_dirty_folio_in_page_array()
1279 ceph_wbc->locked_pages++; in move_dirty_folio_in_page_array()
1287 struct ceph_writeback_ctl *ceph_wbc) in ceph_process_folio_batch() argument
1296 for (i = 0; can_next_page_be_processed(ceph_wbc, i); i++) { in ceph_process_folio_batch()
1297 folio = ceph_wbc->fbatch.folios[i]; in ceph_process_folio_batch()
1316 if (ceph_wbc->locked_pages == 0) in ceph_process_folio_batch()
1322 ceph_wbc, folio); in ceph_process_folio_batch()
1325 ceph_wbc->fbatch.folios[i] = NULL; in ceph_process_folio_batch()
1335 ceph_wbc->fbatch.folios[i] = NULL; in ceph_process_folio_batch()
1345 if (ceph_wbc->locked_pages == 0) { in ceph_process_folio_batch()
1346 ceph_allocate_page_array(mapping, ceph_wbc, folio); in ceph_process_folio_batch()
1347 } else if (!is_folio_index_contiguous(ceph_wbc, folio)) { in ceph_process_folio_batch()
1348 if (is_num_ops_too_big(ceph_wbc)) { in ceph_process_folio_batch()
1354 ceph_wbc->num_ops++; in ceph_process_folio_batch()
1355 ceph_wbc->offset = (u64)folio_pos(folio); in ceph_process_folio_batch()
1356 ceph_wbc->len = 0; in ceph_process_folio_batch()
1365 rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, in ceph_process_folio_batch()
1373 ceph_wbc->fbatch.folios[i] = NULL; in ceph_process_folio_batch()
1374 ceph_wbc->len += folio_size(folio); in ceph_process_folio_batch()
1377 ceph_wbc->processed_in_fbatch = i; in ceph_process_folio_batch()
1403 struct ceph_writeback_ctl *ceph_wbc) in ceph_submit_write() argument
1418 offset = ceph_fscrypt_page_offset(ceph_wbc->pages[0]); in ceph_submit_write()
1419 len = ceph_wbc->wsize; in ceph_submit_write()
1423 offset, &len, 0, ceph_wbc->num_ops, in ceph_submit_write()
1425 ceph_wbc->snapc, ceph_wbc->truncate_seq, in ceph_submit_write()
1426 ceph_wbc->truncate_size, false); in ceph_submit_write()
1431 min(ceph_wbc->num_ops, in ceph_submit_write()
1435 ceph_wbc->snapc, in ceph_submit_write()
1436 ceph_wbc->truncate_seq, in ceph_submit_write()
1437 ceph_wbc->truncate_size, in ceph_submit_write()
1442 page = ceph_wbc->pages[ceph_wbc->locked_pages - 1]; in ceph_submit_write()
1446 for (i = 0; i < folio_batch_count(&ceph_wbc->fbatch); i++) { in ceph_submit_write()
1447 struct folio *folio = ceph_wbc->fbatch.folios[i]; in ceph_submit_write()
1457 for (i = 0; i < ceph_wbc->locked_pages; i++) { in ceph_submit_write()
1458 page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]); in ceph_submit_write()
1476 ceph_wbc->data_pages = ceph_wbc->pages; in ceph_submit_write()
1477 ceph_wbc->op_idx = 0; in ceph_submit_write()
1478 for (i = 0; i < ceph_wbc->locked_pages; i++) { in ceph_submit_write()
1481 page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]); in ceph_submit_write()
1490 if (ceph_wbc->op_idx + 1 == req->r_num_ops) in ceph_submit_write()
1497 osd_req_op_extent_dup_last(req, ceph_wbc->op_idx, in ceph_submit_write()
1502 osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx, in ceph_submit_write()
1503 ceph_wbc->data_pages, in ceph_submit_write()
1505 ceph_wbc->from_pool, in ceph_submit_write()
1507 osd_req_op_extent_update(req, ceph_wbc->op_idx, len); in ceph_submit_write()
1511 ceph_wbc->data_pages = ceph_wbc->pages + i; in ceph_submit_write()
1512 ceph_wbc->op_idx++; in ceph_submit_write()
1525 if (ceph_wbc->size_stable) { in ceph_submit_write()
1526 len = min(len, ceph_wbc->i_size - offset); in ceph_submit_write()
1527 } else if (i == ceph_wbc->locked_pages) { in ceph_submit_write()
1533 ceph_wbc->pages[i - 1], in ceph_submit_write()
1550 osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx, in ceph_submit_write()
1551 ceph_wbc->data_pages, len, in ceph_submit_write()
1552 0, ceph_wbc->from_pool, false); in ceph_submit_write()
1553 osd_req_op_extent_update(req, ceph_wbc->op_idx, len); in ceph_submit_write()
1555 BUG_ON(ceph_wbc->op_idx + 1 != req->r_num_ops); in ceph_submit_write()
1557 ceph_wbc->from_pool = false; in ceph_submit_write()
1558 if (i < ceph_wbc->locked_pages) { in ceph_submit_write()
1559 BUG_ON(ceph_wbc->num_ops <= req->r_num_ops); in ceph_submit_write()
1560 ceph_wbc->num_ops -= req->r_num_ops; in ceph_submit_write()
1561 ceph_wbc->locked_pages -= i; in ceph_submit_write()
1564 ceph_wbc->data_pages = ceph_wbc->pages; in ceph_submit_write()
1565 __ceph_allocate_page_array(ceph_wbc, ceph_wbc->locked_pages); in ceph_submit_write()
1566 memcpy(ceph_wbc->pages, ceph_wbc->data_pages + i, in ceph_submit_write()
1567 ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages)); in ceph_submit_write()
1568 memset(ceph_wbc->data_pages + i, 0, in ceph_submit_write()
1569 ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages)); in ceph_submit_write()
1571 BUG_ON(ceph_wbc->num_ops != req->r_num_ops); in ceph_submit_write()
1573 ceph_wbc->pages = NULL; in ceph_submit_write()
1581 if (ceph_wbc->pages) in ceph_submit_write()
1590 struct ceph_writeback_ctl *ceph_wbc) in ceph_wait_until_current_writes_complete() argument
1596 ceph_wbc->start_index == 0 && /* all dirty pages were checked */ in ceph_wait_until_current_writes_complete()
1597 !ceph_wbc->head_snapc) { in ceph_wait_until_current_writes_complete()
1598 ceph_wbc->index = 0; in ceph_wait_until_current_writes_complete()
1600 while ((ceph_wbc->index <= ceph_wbc->end) && in ceph_wait_until_current_writes_complete()
1602 &ceph_wbc->index, in ceph_wait_until_current_writes_complete()
1605 &ceph_wbc->fbatch))) { in ceph_wait_until_current_writes_complete()
1607 page = &ceph_wbc->fbatch.folios[i]->page; in ceph_wait_until_current_writes_complete()
1608 if (page_snap_context(page) != ceph_wbc->snapc) in ceph_wait_until_current_writes_complete()
1613 folio_batch_release(&ceph_wbc->fbatch); in ceph_wait_until_current_writes_complete()
1628 struct ceph_writeback_ctl ceph_wbc; in ceph_writepages_start() local
1643 ceph_init_writeback_ctl(mapping, wbc, &ceph_wbc); in ceph_writepages_start()
1651 rc = ceph_define_writeback_range(mapping, wbc, &ceph_wbc); in ceph_writepages_start()
1660 tag_pages_for_writeback(mapping, ceph_wbc.index, ceph_wbc.end); in ceph_writepages_start()
1662 while (!has_writeback_done(&ceph_wbc)) { in ceph_writepages_start()
1663 BUG_ON(ceph_wbc.locked_pages); in ceph_writepages_start()
1664 BUG_ON(ceph_wbc.pages); in ceph_writepages_start()
1666 ceph_wbc.max_pages = ceph_wbc.wsize >> PAGE_SHIFT; in ceph_writepages_start()
1669 ceph_folio_batch_reinit(&ceph_wbc); in ceph_writepages_start()
1671 ceph_wbc.nr_folios = filemap_get_folios_tag(mapping, in ceph_writepages_start()
1672 &ceph_wbc.index, in ceph_writepages_start()
1673 ceph_wbc.end, in ceph_writepages_start()
1674 ceph_wbc.tag, in ceph_writepages_start()
1675 &ceph_wbc.fbatch); in ceph_writepages_start()
1677 ceph_wbc.tag, ceph_wbc.nr_folios); in ceph_writepages_start()
1679 if (!ceph_wbc.nr_folios && !ceph_wbc.locked_pages) in ceph_writepages_start()
1683 ceph_process_folio_batch(mapping, wbc, &ceph_wbc); in ceph_writepages_start()
1684 ceph_shift_unused_folios_left(&ceph_wbc.fbatch); in ceph_writepages_start()
1687 if (!ceph_wbc.locked_pages) in ceph_writepages_start()
1690 if (ceph_wbc.processed_in_fbatch) { in ceph_writepages_start()
1691 if (folio_batch_count(&ceph_wbc.fbatch) == 0 && in ceph_writepages_start()
1692 ceph_wbc.locked_pages < ceph_wbc.max_pages) { in ceph_writepages_start()
1698 rc = ceph_submit_write(mapping, wbc, &ceph_wbc); in ceph_writepages_start()
1702 ceph_wbc.locked_pages = 0; in ceph_writepages_start()
1703 ceph_wbc.strip_unit_end = 0; in ceph_writepages_start()
1705 if (folio_batch_count(&ceph_wbc.fbatch) > 0) { in ceph_writepages_start()
1706 ceph_wbc.nr_folios = in ceph_writepages_start()
1707 folio_batch_count(&ceph_wbc.fbatch); in ceph_writepages_start()
1718 ceph_wbc.done = true; in ceph_writepages_start()
1722 (int)ceph_wbc.fbatch.nr, in ceph_writepages_start()
1723 ceph_wbc.fbatch.nr ? ceph_wbc.fbatch.folios[0] : NULL); in ceph_writepages_start()
1724 folio_batch_release(&ceph_wbc.fbatch); in ceph_writepages_start()
1727 if (ceph_wbc.should_loop && !ceph_wbc.done) { in ceph_writepages_start()
1731 ceph_wbc.end = ceph_wbc.start_index - 1; in ceph_writepages_start()
1735 ceph_wait_until_current_writes_complete(mapping, wbc, &ceph_wbc); in ceph_writepages_start()
1737 ceph_wbc.start_index = 0; in ceph_writepages_start()
1738 ceph_wbc.index = 0; in ceph_writepages_start()
1742 if (wbc->range_cyclic || (ceph_wbc.range_whole && wbc->nr_to_write > 0)) in ceph_writepages_start()
1743 mapping->writeback_index = ceph_wbc.index; in ceph_writepages_start()
1749 ceph_put_snap_context(ceph_wbc.last_snapc); in ceph_writepages_start()