Lines Matching full:line
36 struct pblk_line *line; in pblk_line_mark_bb() local
39 line = pblk_ppa_to_line(pblk, *ppa); in pblk_line_mark_bb()
42 pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n", in pblk_line_mark_bb()
43 line->id, pos); in pblk_line_mark_bb()
50 static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line, in pblk_mark_bb() argument
58 pblk_debug(pblk, "erase failed: line:%d, pos:%d\n", line->id, pos); in pblk_mark_bb()
61 atomic_dec(&line->blk_in_line); in pblk_mark_bb()
62 if (test_and_set_bit(pos, line->blk_bitmap)) in pblk_mark_bb()
63 pblk_err(pblk, "attempted to erase bb: line:%d, pos:%d\n", in pblk_mark_bb()
64 line->id, pos); in pblk_mark_bb()
84 struct pblk_line *line; in __pblk_end_io_erase() local
87 line = pblk_ppa_to_line(pblk, rqd->ppa_addr); in __pblk_end_io_erase()
89 chunk = &line->chks[pos]; in __pblk_end_io_erase()
91 atomic_dec(&line->left_seblks); in __pblk_end_io_erase()
98 pblk_mark_bb(pblk, line, rqd->ppa_addr); in __pblk_end_io_erase()
164 void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, in __pblk_map_invalidate() argument
174 spin_lock(&line->lock); in __pblk_map_invalidate()
175 WARN_ON(line->state == PBLK_LINESTATE_FREE); in __pblk_map_invalidate()
177 if (test_and_set_bit(paddr, line->invalid_bitmap)) { in __pblk_map_invalidate()
179 spin_unlock(&line->lock); in __pblk_map_invalidate()
182 le32_add_cpu(line->vsc, -1); in __pblk_map_invalidate()
184 if (line->state == PBLK_LINESTATE_CLOSED) in __pblk_map_invalidate()
185 move_list = pblk_line_gc_list(pblk, line); in __pblk_map_invalidate()
186 spin_unlock(&line->lock); in __pblk_map_invalidate()
190 spin_lock(&line->lock); in __pblk_map_invalidate()
191 /* Prevent moving a line that has just been chosen for GC */ in __pblk_map_invalidate()
192 if (line->state == PBLK_LINESTATE_GC) { in __pblk_map_invalidate()
193 spin_unlock(&line->lock); in __pblk_map_invalidate()
197 spin_unlock(&line->lock); in __pblk_map_invalidate()
199 list_move_tail(&line->list, move_list); in __pblk_map_invalidate()
206 struct pblk_line *line; in pblk_map_invalidate() local
215 line = pblk_ppa_to_line(pblk, ppa); in pblk_map_invalidate()
218 __pblk_map_invalidate(pblk, line, paddr); in pblk_map_invalidate()
407 struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line) in pblk_line_gc_list() argument
412 int packed_meta = (le32_to_cpu(*line->vsc) / pblk->min_write_pgs_data) in pblk_line_gc_list()
414 int vsc = le32_to_cpu(*line->vsc) + packed_meta; in pblk_line_gc_list()
416 lockdep_assert_held(&line->lock); in pblk_line_gc_list()
418 if (line->w_err_gc->has_write_err) { in pblk_line_gc_list()
419 if (line->gc_group != PBLK_LINEGC_WERR) { in pblk_line_gc_list()
420 line->gc_group = PBLK_LINEGC_WERR; in pblk_line_gc_list()
425 if (line->gc_group != PBLK_LINEGC_FULL) { in pblk_line_gc_list()
426 line->gc_group = PBLK_LINEGC_FULL; in pblk_line_gc_list()
430 if (line->gc_group != PBLK_LINEGC_HIGH) { in pblk_line_gc_list()
431 line->gc_group = PBLK_LINEGC_HIGH; in pblk_line_gc_list()
435 if (line->gc_group != PBLK_LINEGC_MID) { in pblk_line_gc_list()
436 line->gc_group = PBLK_LINEGC_MID; in pblk_line_gc_list()
439 } else if (vsc < line->sec_in_line) { in pblk_line_gc_list()
440 if (line->gc_group != PBLK_LINEGC_LOW) { in pblk_line_gc_list()
441 line->gc_group = PBLK_LINEGC_LOW; in pblk_line_gc_list()
444 } else if (vsc == line->sec_in_line) { in pblk_line_gc_list()
445 if (line->gc_group != PBLK_LINEGC_EMPTY) { in pblk_line_gc_list()
446 line->gc_group = PBLK_LINEGC_EMPTY; in pblk_line_gc_list()
450 line->state = PBLK_LINESTATE_CORRUPT; in pblk_line_gc_list()
451 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_gc_list()
452 line->state); in pblk_line_gc_list()
454 line->gc_group = PBLK_LINEGC_NONE; in pblk_line_gc_list()
456 pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n", in pblk_line_gc_list()
457 line->id, vsc, in pblk_line_gc_list()
458 line->sec_in_line, in pblk_line_gc_list()
598 void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_dealloc_page() argument
603 spin_lock(&line->lock); in pblk_dealloc_page()
604 addr = find_next_zero_bit(line->map_bitmap, in pblk_dealloc_page()
605 pblk->lm.sec_per_line, line->cur_sec); in pblk_dealloc_page()
606 line->cur_sec = addr - nr_secs; in pblk_dealloc_page()
608 for (i = 0; i < nr_secs; i++, line->cur_sec--) in pblk_dealloc_page()
609 WARN_ON(!test_and_clear_bit(line->cur_sec, line->map_bitmap)); in pblk_dealloc_page()
610 spin_unlock(&line->lock); in pblk_dealloc_page()
613 u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in __pblk_alloc_page() argument
618 lockdep_assert_held(&line->lock); in __pblk_alloc_page()
621 if (line->cur_sec + nr_secs > pblk->lm.sec_per_line) { in __pblk_alloc_page()
623 nr_secs = pblk->lm.sec_per_line - line->cur_sec; in __pblk_alloc_page()
626 line->cur_sec = addr = find_next_zero_bit(line->map_bitmap, in __pblk_alloc_page()
627 pblk->lm.sec_per_line, line->cur_sec); in __pblk_alloc_page()
628 for (i = 0; i < nr_secs; i++, line->cur_sec++) in __pblk_alloc_page()
629 WARN_ON(test_and_set_bit(line->cur_sec, line->map_bitmap)); in __pblk_alloc_page()
634 u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_alloc_page() argument
641 spin_lock(&line->lock); in pblk_alloc_page()
642 addr = __pblk_alloc_page(pblk, line, nr_secs); in pblk_alloc_page()
643 line->left_msecs -= nr_secs; in pblk_alloc_page()
644 WARN(line->left_msecs < 0, "pblk: page allocation out of bounds\n"); in pblk_alloc_page()
645 spin_unlock(&line->lock); in pblk_alloc_page()
650 u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line) in pblk_lookup_page() argument
654 spin_lock(&line->lock); in pblk_lookup_page()
655 paddr = find_next_zero_bit(line->map_bitmap, in pblk_lookup_page()
656 pblk->lm.sec_per_line, line->cur_sec); in pblk_lookup_page()
657 spin_unlock(&line->lock); in pblk_lookup_page()
662 u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) in pblk_line_smeta_start() argument
670 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); in pblk_line_smeta_start()
677 int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line) in pblk_line_smeta_read() argument
682 u64 paddr = pblk_line_smeta_start(pblk, line); in pblk_line_smeta_read()
697 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_smeta_read()
699 ret = pblk_submit_io_sync(pblk, &rqd, line->smeta); in pblk_line_smeta_read()
717 static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line, in pblk_line_smeta_write() argument
723 __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); in pblk_line_smeta_write()
742 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_smeta_write()
746 ret = pblk_submit_io_sync_sem(pblk, &rqd, line->smeta); in pblk_line_smeta_write()
764 int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, in pblk_line_emeta_read() argument
773 u64 paddr = line->emeta_ssec; in pblk_line_emeta_read()
777 int line_id = line->id; in pblk_line_emeta_read()
811 while (test_bit(pos, line->blk_bitmap)) { in pblk_line_emeta_read()
885 int pblk_line_erase(struct pblk *pblk, struct pblk_line *line) in pblk_line_erase() argument
893 spin_lock(&line->lock); in pblk_line_erase()
894 bit = find_next_zero_bit(line->erase_bitmap, lm->blk_per_line, in pblk_line_erase()
897 spin_unlock(&line->lock); in pblk_line_erase()
902 ppa.a.blk = line->id; in pblk_line_erase()
904 atomic_dec(&line->left_eblks); in pblk_line_erase()
905 WARN_ON(test_and_set_bit(bit, line->erase_bitmap)); in pblk_line_erase()
906 spin_unlock(&line->lock); in pblk_line_erase()
910 pblk_err(pblk, "failed to erase line %d\n", line->id); in pblk_line_erase()
918 static void pblk_line_setup_metadata(struct pblk_line *line, in pblk_line_setup_metadata() argument
936 line->meta_line = meta_line; in pblk_line_setup_metadata()
938 line->smeta = l_mg->sline_meta[meta_line]; in pblk_line_setup_metadata()
939 line->emeta = l_mg->eline_meta[meta_line]; in pblk_line_setup_metadata()
941 memset(line->smeta, 0, lm->smeta_len); in pblk_line_setup_metadata()
942 memset(line->emeta->buf, 0, lm->emeta_len[0]); in pblk_line_setup_metadata()
944 line->emeta->mem = 0; in pblk_line_setup_metadata()
945 atomic_set(&line->emeta->sync, 0); in pblk_line_setup_metadata()
951 static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_metadata() argument
958 struct pblk_emeta *emeta = line->emeta; in pblk_line_init_metadata()
960 struct line_smeta *smeta_buf = (struct line_smeta *)line->smeta; in pblk_line_init_metadata()
963 /* After erasing the line, new bad blocks might appear and we risk in pblk_line_init_metadata()
964 * having an invalid line in pblk_line_init_metadata()
967 bitmap_weight(line->blk_bitmap, lm->blk_per_line); in pblk_line_init_metadata()
970 spin_lock(&line->lock); in pblk_line_init_metadata()
971 line->state = PBLK_LINESTATE_BAD; in pblk_line_init_metadata()
972 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_init_metadata()
973 line->state); in pblk_line_init_metadata()
974 spin_unlock(&line->lock); in pblk_line_init_metadata()
976 list_add_tail(&line->list, &l_mg->bad_list); in pblk_line_init_metadata()
979 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_init_metadata()
985 line->lun_bitmap = ((void *)(smeta_buf)) + sizeof(struct line_smeta); in pblk_line_init_metadata()
987 /* Mark LUNs allocated in this line (all for now) */ in pblk_line_init_metadata()
988 bitmap_set(line->lun_bitmap, 0, lm->lun_bitmap_len); in pblk_line_init_metadata()
992 smeta_buf->header.id = cpu_to_le32(line->id); in pblk_line_init_metadata()
993 smeta_buf->header.type = cpu_to_le16(line->type); in pblk_line_init_metadata()
998 smeta_buf->seq_nr = cpu_to_le64(line->seq_nr); in pblk_line_init_metadata()
1003 memcpy(line->lun_bitmap, cur->lun_bitmap, lm->lun_bitmap_len); in pblk_line_init_metadata()
1005 cur->emeta->buf->next_id = cpu_to_le32(line->id); in pblk_line_init_metadata()
1024 emeta_buf->seq_nr = cpu_to_le64(line->seq_nr); in pblk_line_init_metadata()
1025 emeta_buf->nr_lbas = cpu_to_le64(line->sec_in_line); in pblk_line_init_metadata()
1034 static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line) in pblk_line_alloc_bitmaps() argument
1039 line->map_bitmap = mempool_alloc(l_mg->bitmap_pool, GFP_KERNEL); in pblk_line_alloc_bitmaps()
1040 if (!line->map_bitmap) in pblk_line_alloc_bitmaps()
1043 memset(line->map_bitmap, 0, lm->sec_bitmap_len); in pblk_line_alloc_bitmaps()
1046 line->invalid_bitmap = mempool_alloc(l_mg->bitmap_pool, GFP_KERNEL); in pblk_line_alloc_bitmaps()
1047 if (!line->invalid_bitmap) { in pblk_line_alloc_bitmaps()
1048 mempool_free(line->map_bitmap, l_mg->bitmap_pool); in pblk_line_alloc_bitmaps()
1049 line->map_bitmap = NULL; in pblk_line_alloc_bitmaps()
1059 static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_bb() argument
1070 line->sec_in_line = lm->sec_per_line; in pblk_line_init_bb()
1072 /* Capture bad block information on line mapping bitmaps */ in pblk_line_init_bb()
1073 while ((bit = find_next_bit(line->blk_bitmap, lm->blk_per_line, in pblk_line_init_bb()
1078 bitmap_or(line->map_bitmap, line->map_bitmap, l_mg->bb_aux, in pblk_line_init_bb()
1080 line->sec_in_line -= geo->clba; in pblk_line_init_bb()
1084 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); in pblk_line_init_bb()
1086 bitmap_set(line->map_bitmap, off, lm->smeta_sec); in pblk_line_init_bb()
1087 line->sec_in_line -= lm->smeta_sec; in pblk_line_init_bb()
1088 line->cur_sec = off + lm->smeta_sec; in pblk_line_init_bb()
1090 if (init && pblk_line_smeta_write(pblk, line, off)) { in pblk_line_init_bb()
1091 pblk_debug(pblk, "line smeta I/O failed. Retry\n"); in pblk_line_init_bb()
1095 bitmap_copy(line->invalid_bitmap, line->map_bitmap, lm->sec_per_line); in pblk_line_init_bb()
1104 if (!test_bit(off, line->invalid_bitmap)) { in pblk_line_init_bb()
1105 bitmap_set(line->invalid_bitmap, off, geo->ws_opt); in pblk_line_init_bb()
1110 line->emeta_ssec = off; in pblk_line_init_bb()
1111 line->sec_in_line -= lm->emeta_sec[0]; in pblk_line_init_bb()
1112 line->nr_valid_lbas = 0; in pblk_line_init_bb()
1113 line->left_msecs = line->sec_in_line; in pblk_line_init_bb()
1114 *line->vsc = cpu_to_le32(line->sec_in_line); in pblk_line_init_bb()
1116 if (lm->sec_per_line - line->sec_in_line != in pblk_line_init_bb()
1117 bitmap_weight(line->invalid_bitmap, lm->sec_per_line)) { in pblk_line_init_bb()
1118 spin_lock(&line->lock); in pblk_line_init_bb()
1119 line->state = PBLK_LINESTATE_BAD; in pblk_line_init_bb()
1120 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_init_bb()
1121 line->state); in pblk_line_init_bb()
1122 spin_unlock(&line->lock); in pblk_line_init_bb()
1124 list_add_tail(&line->list, &l_mg->bad_list); in pblk_line_init_bb()
1125 pblk_err(pblk, "unexpected line %d is bad\n", line->id); in pblk_line_init_bb()
1133 static int pblk_prepare_new_line(struct pblk *pblk, struct pblk_line *line) in pblk_prepare_new_line() argument
1138 int blk_to_erase = atomic_read(&line->blk_in_line); in pblk_prepare_new_line()
1144 int state = line->chks[pos].state; in pblk_prepare_new_line()
1149 line->erase_bitmap); in pblk_prepare_new_line()
1157 static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) in pblk_line_prepare() argument
1160 int blk_in_line = atomic_read(&line->blk_in_line); in pblk_line_prepare()
1164 bitmap_copy(line->erase_bitmap, line->blk_bitmap, lm->blk_per_line); in pblk_line_prepare()
1166 spin_lock(&line->lock); in pblk_line_prepare()
1168 /* If we have not written to this line, we need to mark up free chunks in pblk_line_prepare()
1171 if (line->state == PBLK_LINESTATE_NEW) { in pblk_line_prepare()
1172 blk_to_erase = pblk_prepare_new_line(pblk, line); in pblk_line_prepare()
1173 line->state = PBLK_LINESTATE_FREE; in pblk_line_prepare()
1174 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_prepare()
1175 line->state); in pblk_line_prepare()
1181 spin_unlock(&line->lock); in pblk_line_prepare()
1185 if (line->state != PBLK_LINESTATE_FREE) { in pblk_line_prepare()
1186 WARN(1, "pblk: corrupted line %d, state %d\n", in pblk_line_prepare()
1187 line->id, line->state); in pblk_line_prepare()
1188 spin_unlock(&line->lock); in pblk_line_prepare()
1192 line->state = PBLK_LINESTATE_OPEN; in pblk_line_prepare()
1193 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_prepare()
1194 line->state); in pblk_line_prepare()
1196 atomic_set(&line->left_eblks, blk_to_erase); in pblk_line_prepare()
1197 atomic_set(&line->left_seblks, blk_to_erase); in pblk_line_prepare()
1199 line->meta_distance = lm->meta_distance; in pblk_line_prepare()
1200 spin_unlock(&line->lock); in pblk_line_prepare()
1202 kref_init(&line->ref); in pblk_line_prepare()
1203 atomic_set(&line->sec_to_update, 0); in pblk_line_prepare()
1208 /* Line allocations in the recovery path are always single threaded */
1209 int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_alloc() argument
1215 l_mg->data_line = line; in pblk_line_recov_alloc()
1216 list_del(&line->list); in pblk_line_recov_alloc()
1218 ret = pblk_line_prepare(pblk, line); in pblk_line_recov_alloc()
1220 list_add(&line->list, &l_mg->free_list); in pblk_line_recov_alloc()
1226 ret = pblk_line_alloc_bitmaps(pblk, line); in pblk_line_recov_alloc()
1230 if (!pblk_line_init_bb(pblk, line, 0)) { in pblk_line_recov_alloc()
1235 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_recov_alloc()
1240 list_add(&line->list, &l_mg->free_list); in pblk_line_recov_alloc()
1246 void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_close() argument
1250 mempool_free(line->map_bitmap, l_mg->bitmap_pool); in pblk_line_recov_close()
1251 line->map_bitmap = NULL; in pblk_line_recov_close()
1252 line->smeta = NULL; in pblk_line_recov_close()
1253 line->emeta = NULL; in pblk_line_recov_close()
1256 static void pblk_line_reinit(struct pblk_line *line) in pblk_line_reinit() argument
1258 *line->vsc = cpu_to_le32(EMPTY_ENTRY); in pblk_line_reinit()
1260 line->map_bitmap = NULL; in pblk_line_reinit()
1261 line->invalid_bitmap = NULL; in pblk_line_reinit()
1262 line->smeta = NULL; in pblk_line_reinit()
1263 line->emeta = NULL; in pblk_line_reinit()
1266 void pblk_line_free(struct pblk_line *line) in pblk_line_free() argument
1268 struct pblk *pblk = line->pblk; in pblk_line_free()
1271 mempool_free(line->map_bitmap, l_mg->bitmap_pool); in pblk_line_free()
1272 mempool_free(line->invalid_bitmap, l_mg->bitmap_pool); in pblk_line_free()
1274 pblk_line_reinit(line); in pblk_line_free()
1281 struct pblk_line *line; in pblk_line_get() local
1292 line = list_first_entry(&l_mg->free_list, struct pblk_line, list); in pblk_line_get()
1293 list_del(&line->list); in pblk_line_get()
1296 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); in pblk_line_get()
1298 spin_lock(&line->lock); in pblk_line_get()
1299 line->state = PBLK_LINESTATE_BAD; in pblk_line_get()
1300 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_get()
1301 line->state); in pblk_line_get()
1302 spin_unlock(&line->lock); in pblk_line_get()
1304 list_add_tail(&line->list, &l_mg->bad_list); in pblk_line_get()
1306 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_get()
1310 ret = pblk_line_prepare(pblk, line); in pblk_line_get()
1314 list_add(&line->list, &l_mg->bad_list); in pblk_line_get()
1317 list_add(&line->list, &l_mg->corrupt_list); in pblk_line_get()
1320 pblk_err(pblk, "failed to prepare line %d\n", line->id); in pblk_line_get()
1321 list_add(&line->list, &l_mg->free_list); in pblk_line_get()
1327 return line; in pblk_line_get()
1331 struct pblk_line *line) in pblk_line_retry() argument
1345 retry_line->map_bitmap = line->map_bitmap; in pblk_line_retry()
1346 retry_line->invalid_bitmap = line->invalid_bitmap; in pblk_line_retry()
1347 retry_line->smeta = line->smeta; in pblk_line_retry()
1348 retry_line->emeta = line->emeta; in pblk_line_retry()
1349 retry_line->meta_line = line->meta_line; in pblk_line_retry()
1351 pblk_line_reinit(line); in pblk_line_retry()
1356 pblk_rl_free_lines_dec(&pblk->rl, line, false); in pblk_line_retry()
1374 struct pblk_line *line; in pblk_line_get_first_data() local
1377 line = pblk_line_get(pblk); in pblk_line_get_first_data()
1378 if (!line) { in pblk_line_get_first_data()
1383 line->seq_nr = l_mg->d_seq_nr++; in pblk_line_get_first_data()
1384 line->type = PBLK_LINETYPE_DATA; in pblk_line_get_first_data()
1385 l_mg->data_line = line; in pblk_line_get_first_data()
1387 pblk_line_setup_metadata(line, l_mg, &pblk->lm); in pblk_line_get_first_data()
1389 /* Allocate next line for preparation */ in pblk_line_get_first_data()
1392 /* If we cannot get a new line, we need to stop the pipeline. in pblk_line_get_first_data()
1405 if (pblk_line_alloc_bitmaps(pblk, line)) in pblk_line_get_first_data()
1408 if (pblk_line_erase(pblk, line)) { in pblk_line_get_first_data()
1409 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1410 if (!line) in pblk_line_get_first_data()
1415 if (!pblk_line_init_metadata(pblk, line, NULL)) { in pblk_line_get_first_data()
1416 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1417 if (!line) in pblk_line_get_first_data()
1423 if (!pblk_line_init_bb(pblk, line, 1)) { in pblk_line_get_first_data()
1424 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1425 if (!line) in pblk_line_get_first_data()
1431 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_get_first_data()
1433 return line; in pblk_line_get_first_data()
1438 struct pblk_line *line; in pblk_ppa_to_line_put() local
1440 line = pblk_ppa_to_line(pblk, ppa); in pblk_ppa_to_line_put()
1441 kref_put(&line->ref, pblk_line_put_wq); in pblk_ppa_to_line_put()
1453 static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line) in pblk_stop_writes() argument
1466 struct pblk_line *line, *tline; in pblk_line_close_meta_sync() local
1478 list_for_each_entry_safe(line, tline, &list, list) { in pblk_line_close_meta_sync()
1479 struct pblk_emeta *emeta = line->emeta; in pblk_line_close_meta_sync()
1484 ret = pblk_submit_meta_io(pblk, line); in pblk_line_close_meta_sync()
1486 pblk_err(pblk, "sync meta line %d failed (%d)\n", in pblk_line_close_meta_sync()
1487 line->id, ret); in pblk_line_close_meta_sync()
1563 /* If line is not fully erased, erase it */ in pblk_line_replace_data()
1595 /* Allocate next line for preparation */ in pblk_line_replace_data()
1599 /* If we cannot get a new line, we need to stop the pipeline. in pblk_line_replace_data()
1615 static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line) in __pblk_line_put() argument
1620 spin_lock(&line->lock); in __pblk_line_put()
1621 WARN_ON(line->state != PBLK_LINESTATE_GC); in __pblk_line_put()
1622 if (line->w_err_gc->has_gc_err) { in __pblk_line_put()
1623 spin_unlock(&line->lock); in __pblk_line_put()
1624 pblk_err(pblk, "line %d had errors during GC\n", line->id); in __pblk_line_put()
1625 pblk_put_line_back(pblk, line); in __pblk_line_put()
1626 line->w_err_gc->has_gc_err = 0; in __pblk_line_put()
1630 line->state = PBLK_LINESTATE_FREE; in __pblk_line_put()
1631 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in __pblk_line_put()
1632 line->state); in __pblk_line_put()
1633 line->gc_group = PBLK_LINEGC_NONE; in __pblk_line_put()
1634 pblk_line_free(line); in __pblk_line_put()
1636 if (line->w_err_gc->has_write_err) { in __pblk_line_put()
1638 line->w_err_gc->has_write_err = 0; in __pblk_line_put()
1641 spin_unlock(&line->lock); in __pblk_line_put()
1645 list_add_tail(&line->list, &l_mg->free_list); in __pblk_line_put()
1649 pblk_rl_free_lines_inc(&pblk->rl, line); in __pblk_line_put()
1657 struct pblk_line *line = line_put_ws->line; in pblk_line_put_ws() local
1659 __pblk_line_put(pblk, line); in pblk_line_put_ws()
1665 struct pblk_line *line = container_of(ref, struct pblk_line, ref); in pblk_line_put() local
1666 struct pblk *pblk = line->pblk; in pblk_line_put()
1668 __pblk_line_put(pblk, line); in pblk_line_put()
1673 struct pblk_line *line = container_of(ref, struct pblk_line, ref); in pblk_line_put_wq() local
1674 struct pblk *pblk = line->pblk; in pblk_line_put_wq()
1682 line_put_ws->line = line; in pblk_line_put_wq()
1712 pblk_err(pblk, "could not async erase line:%d,blk:%d\n", in pblk_blk_erase_async()
1725 /* For now, always erase next line */
1731 int pblk_line_is_full(struct pblk_line *line) in pblk_line_is_full() argument
1733 return (line->left_msecs == 0); in pblk_line_is_full()
1742 void pblk_line_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_close() argument
1752 WARN(!bitmap_full(line->map_bitmap, lm->sec_per_line), in pblk_line_close()
1753 "pblk: corrupt closed line %d\n", line->id); in pblk_line_close()
1757 WARN_ON(!test_and_clear_bit(line->meta_line, &l_mg->meta_bitmap)); in pblk_line_close()
1761 spin_lock(&line->lock); in pblk_line_close()
1762 WARN_ON(line->state != PBLK_LINESTATE_OPEN); in pblk_line_close()
1763 line->state = PBLK_LINESTATE_CLOSED; in pblk_line_close()
1764 move_list = pblk_line_gc_list(pblk, line); in pblk_line_close()
1765 list_add_tail(&line->list, move_list); in pblk_line_close()
1767 mempool_free(line->map_bitmap, l_mg->bitmap_pool); in pblk_line_close()
1768 line->map_bitmap = NULL; in pblk_line_close()
1769 line->smeta = NULL; in pblk_line_close()
1770 line->emeta = NULL; in pblk_line_close()
1775 int state = line->chks[pos].state; in pblk_line_close()
1781 spin_unlock(&line->lock); in pblk_line_close()
1784 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_line_close()
1785 line->state); in pblk_line_close()
1788 void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) in pblk_line_close_meta() argument
1792 struct pblk_emeta *emeta = line->emeta; in pblk_line_close_meta()
1796 /* No need for exact vsc value; avoid a big line lock and take aprox. */ in pblk_line_close_meta()
1798 memcpy(emeta_to_bb(emeta_buf), line->blk_bitmap, lm->blk_bitmap_len); in pblk_line_close_meta()
1808 emeta_buf->header.id = cpu_to_le32(line->id); in pblk_line_close_meta()
1809 emeta_buf->header.type = cpu_to_le16(line->type); in pblk_line_close_meta()
1816 emeta_buf->nr_valid_lbas = cpu_to_le64(line->nr_valid_lbas); in pblk_line_close_meta()
1820 spin_lock(&line->lock); in pblk_line_close_meta()
1825 if (line->emeta_ssec != line->cur_sec) in pblk_line_close_meta()
1826 line->emeta_ssec = line->cur_sec; in pblk_line_close_meta()
1828 list_add_tail(&line->list, &l_mg->emeta_list); in pblk_line_close_meta()
1829 spin_unlock(&line->lock); in pblk_line_close_meta()
1835 static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line) in pblk_save_lba_list() argument
1839 struct pblk_w_err_gc *w_err_gc = line->w_err_gc; in pblk_save_lba_list()
1840 struct pblk_emeta *emeta = line->emeta; in pblk_save_lba_list()
1852 struct pblk_line *line = line_ws->line; in pblk_line_close_ws() local
1853 struct pblk_w_err_gc *w_err_gc = line->w_err_gc; in pblk_line_close_ws()
1856 * so keep a copy of the lba list until we've gc'd the line in pblk_line_close_ws()
1859 pblk_save_lba_list(pblk, line); in pblk_line_close_ws()
1861 pblk_line_close(pblk, line); in pblk_line_close_ws()
1865 void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, in pblk_gen_run_ws() argument
1874 line_ws->line = line; in pblk_gen_run_ws()
2075 /* If the L2P entry maps to a line, the reference is valid */ in pblk_lookup_l2p_seq()
2077 struct pblk_line *line = pblk_ppa_to_line(pblk, ppa); in pblk_lookup_l2p_seq() local
2083 kref_get(&line->ref); in pblk_lookup_l2p_seq()