Lines Matching refs:t
60 static inline sector_t *get_node(struct dm_table *t, in get_node() argument
63 return t->index[l] + (n * KEYS_PER_NODE); in get_node()
70 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n) in high() argument
72 for (; l < t->depth - 1; l++) in high()
75 if (n >= t->counts[l]) in high()
78 return get_node(t, l, n)[KEYS_PER_NODE - 1]; in high()
85 static int setup_btree_index(unsigned int l, struct dm_table *t) in setup_btree_index() argument
90 for (n = 0U; n < t->counts[l]; n++) { in setup_btree_index()
91 node = get_node(t, l, n); in setup_btree_index()
94 node[k] = high(t, l + 1, get_child(n, k)); in setup_btree_index()
104 static int alloc_targets(struct dm_table *t, unsigned int num) in alloc_targets() argument
121 t->num_allocated = num; in alloc_targets()
122 t->highs = n_highs; in alloc_targets()
123 t->targets = n_targets; in alloc_targets()
131 struct dm_table *t; in dm_table_create() local
136 t = kzalloc_obj(*t); in dm_table_create()
138 if (!t) in dm_table_create()
141 INIT_LIST_HEAD(&t->devices); in dm_table_create()
149 kfree(t); in dm_table_create()
153 if (alloc_targets(t, num_targets)) { in dm_table_create()
154 kfree(t); in dm_table_create()
158 t->type = DM_TYPE_NONE; in dm_table_create()
159 t->mode = mode; in dm_table_create()
160 t->md = md; in dm_table_create()
161 t->flush_bypasses_map = true; in dm_table_create()
162 *result = t; in dm_table_create()
180 static void dm_table_destroy_crypto_profile(struct dm_table *t);
182 void dm_table_destroy(struct dm_table *t) in dm_table_destroy() argument
184 if (!t) in dm_table_destroy()
188 if (t->depth >= 2) in dm_table_destroy()
189 kvfree(t->index[t->depth - 2]); in dm_table_destroy()
192 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_destroy()
193 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_destroy()
201 kvfree(t->highs); in dm_table_destroy()
204 free_devices(&t->devices, t->md); in dm_table_destroy()
206 dm_free_md_mempools(t->mempools); in dm_table_destroy()
208 dm_table_destroy_crypto_profile(t); in dm_table_destroy()
210 kfree(t); in dm_table_destroy()
371 struct dm_table *t = ti->table; in dm_get_device() local
373 BUG_ON(!t); in dm_get_device()
379 if (dev == disk_devt(t->md->disk)) in dm_get_device()
382 dd = find_device(&t->devices, dev); in dm_get_device()
388 r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); in dm_get_device()
395 list_add(&dd->list, &t->devices); in dm_get_device()
399 r = upgrade_mode(dd, mode, t->md); in dm_get_device()
482 static int adjoin(struct dm_table *t, struct dm_target *ti) in adjoin() argument
486 if (!t->num_targets) in adjoin()
489 prev = &t->targets[t->num_targets - 1]; in adjoin()
603 static int validate_hardware_logical_block_alignment(struct dm_table *t, in validate_hardware_logical_block_alignment() argument
631 for (i = 0; i < t->num_targets; i++) { in validate_hardware_logical_block_alignment()
632 ti = dm_table_get_target(t, i); in validate_hardware_logical_block_alignment()
660 dm_device_name(t->md), i, in validate_hardware_logical_block_alignment()
670 int dm_table_add_target(struct dm_table *t, const char *type, in dm_table_add_target() argument
677 if (t->singleton) { in dm_table_add_target()
679 dm_device_name(t->md), t->targets->type->name); in dm_table_add_target()
683 BUG_ON(t->num_targets >= t->num_allocated); in dm_table_add_target()
685 ti = t->targets + t->num_targets; in dm_table_add_target()
689 DMERR("%s: zero-length target", dm_device_name(t->md)); in dm_table_add_target()
693 DMERR("%s: too large device", dm_device_name(t->md)); in dm_table_add_target()
699 DMERR("%s: %s: unknown target type", dm_device_name(t->md), type); in dm_table_add_target()
704 if (t->num_targets) { in dm_table_add_target()
708 t->singleton = true; in dm_table_add_target()
712 !(t->mode & BLK_OPEN_WRITE)) { in dm_table_add_target()
717 if (t->immutable_target_type) { in dm_table_add_target()
718 if (t->immutable_target_type != ti->type) { in dm_table_add_target()
723 if (t->num_targets) { in dm_table_add_target()
727 t->immutable_target_type = ti->type; in dm_table_add_target()
730 ti->table = t; in dm_table_add_target()
738 if (!adjoin(t, ti)) { in dm_table_add_target()
754 t->highs[t->num_targets++] = ti->begin + ti->len - 1; in dm_table_add_target()
758 dm_device_name(t->md), type); in dm_table_add_target()
764 t->flush_bypasses_map = false; in dm_table_add_target()
769 DMERR("%s: %s: %s (%pe)", dm_device_name(t->md), type, ti->error, ERR_PTR(r)); in dm_table_add_target()
843 void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type) in dm_table_set_type() argument
845 t->type = type; in dm_table_set_type()
867 static bool dm_table_supports_dax(struct dm_table *t, in dm_table_supports_dax() argument
871 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_dax()
872 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_dax()
899 static int dm_table_determine_type(struct dm_table *t) in dm_table_determine_type() argument
903 struct list_head *devices = dm_table_get_devices(t); in dm_table_determine_type()
904 enum dm_queue_mode live_md_type = dm_get_md_type(t->md); in dm_table_determine_type()
906 if (t->type != DM_TYPE_NONE) { in dm_table_determine_type()
908 if (t->type == DM_TYPE_BIO_BASED) { in dm_table_determine_type()
912 BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED); in dm_table_determine_type()
916 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_determine_type()
917 ti = dm_table_get_target(t, i); in dm_table_determine_type()
946 t->type = DM_TYPE_BIO_BASED; in dm_table_determine_type()
947 if (dm_table_supports_dax(t, device_not_dax_capable) || in dm_table_determine_type()
949 t->type = DM_TYPE_DAX_BIO_BASED; in dm_table_determine_type()
956 t->type = DM_TYPE_REQUEST_BASED; in dm_table_determine_type()
965 if (t->num_targets > 1) { in dm_table_determine_type()
972 struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx); in dm_table_determine_type()
976 t->type = live_table->type; in dm_table_determine_type()
977 dm_put_live_table(t->md, srcu_idx); in dm_table_determine_type()
981 ti = dm_table_get_immutable_target(t); in dm_table_determine_type()
1000 enum dm_queue_mode dm_table_get_type(struct dm_table *t) in dm_table_get_type() argument
1002 return t->type; in dm_table_get_type()
1005 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t) in dm_table_get_immutable_target_type() argument
1007 return t->immutable_target_type; in dm_table_get_immutable_target_type()
1010 struct dm_target *dm_table_get_immutable_target(struct dm_table *t) in dm_table_get_immutable_target() argument
1013 if (t->num_targets > 1 || in dm_table_get_immutable_target()
1014 !dm_target_is_immutable(t->targets[0].type)) in dm_table_get_immutable_target()
1017 return t->targets; in dm_table_get_immutable_target()
1020 struct dm_target *dm_table_get_wildcard_target(struct dm_table *t) in dm_table_get_wildcard_target() argument
1022 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_get_wildcard_target()
1023 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_get_wildcard_target()
1032 bool dm_table_request_based(struct dm_table *t) in dm_table_request_based() argument
1034 return __table_type_request_based(dm_table_get_type(t)); in dm_table_request_based()
1037 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md) in dm_table_alloc_md_mempools() argument
1039 enum dm_queue_mode type = dm_table_get_type(t); in dm_table_alloc_md_mempools()
1063 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_alloc_md_mempools()
1064 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_alloc_md_mempools()
1081 t->mempools = pools; in dm_table_alloc_md_mempools()
1089 static int setup_indexes(struct dm_table *t) in setup_indexes() argument
1096 for (i = t->depth - 2; i >= 0; i--) { in setup_indexes()
1097 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE); in setup_indexes()
1098 total += t->counts[i]; in setup_indexes()
1106 for (i = t->depth - 2; i >= 0; i--) { in setup_indexes()
1107 t->index[i] = indexes; in setup_indexes()
1108 indexes += (KEYS_PER_NODE * t->counts[i]); in setup_indexes()
1109 setup_btree_index(i, t); in setup_indexes()
1118 static int dm_table_build_index(struct dm_table *t) in dm_table_build_index() argument
1124 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); in dm_table_build_index()
1125 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); in dm_table_build_index()
1128 t->counts[t->depth - 1] = leaf_nodes; in dm_table_build_index()
1129 t->index[t->depth - 1] = t->highs; in dm_table_build_index()
1131 if (t->depth >= 2) in dm_table_build_index()
1132 r = setup_indexes(t); in dm_table_build_index()
1162 struct dm_table *t; in dm_keyslot_evict() local
1165 t = dm_get_live_table(md, &srcu_idx); in dm_keyslot_evict()
1166 if (!t) in dm_keyslot_evict()
1169 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_keyslot_evict()
1170 struct dm_target *ti = dm_table_get_target(t, i); in dm_keyslot_evict()
1259 struct dm_table *t; in dm_exec_wrappedkey_op() local
1265 t = dm_get_live_table(md, &srcu_idx); in dm_exec_wrappedkey_op()
1266 if (!t) in dm_exec_wrappedkey_op()
1278 for (i = 0; i < t->num_targets; i++) { in dm_exec_wrappedkey_op()
1279 ti = dm_table_get_target(t, i); in dm_exec_wrappedkey_op()
1372 static void dm_table_destroy_crypto_profile(struct dm_table *t) in dm_table_destroy_crypto_profile() argument
1374 dm_destroy_crypto_profile(t->crypto_profile); in dm_table_destroy_crypto_profile()
1375 t->crypto_profile = NULL; in dm_table_destroy_crypto_profile()
1387 static int dm_table_construct_crypto_profile(struct dm_table *t) in dm_table_construct_crypto_profile() argument
1397 dmcp->md = t->md; in dm_table_construct_crypto_profile()
1407 for (i = 0; i < t->num_targets; i++) { in dm_table_construct_crypto_profile()
1408 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_construct_crypto_profile()
1428 if (t->md->queue && in dm_table_construct_crypto_profile()
1430 t->md->queue->crypto_profile)) { in dm_table_construct_crypto_profile()
1457 t->crypto_profile = profile; in dm_table_construct_crypto_profile()
1463 struct dm_table *t) in dm_update_crypto_profile() argument
1465 if (!t->crypto_profile) in dm_update_crypto_profile()
1470 blk_crypto_register(t->crypto_profile, q); in dm_update_crypto_profile()
1473 t->crypto_profile); in dm_update_crypto_profile()
1474 dm_destroy_crypto_profile(t->crypto_profile); in dm_update_crypto_profile()
1476 t->crypto_profile = NULL; in dm_update_crypto_profile()
1481 static int dm_table_construct_crypto_profile(struct dm_table *t) in dm_table_construct_crypto_profile() argument
1490 static void dm_table_destroy_crypto_profile(struct dm_table *t) in dm_table_destroy_crypto_profile() argument
1495 struct dm_table *t) in dm_update_crypto_profile() argument
1505 int dm_table_complete(struct dm_table *t) in dm_table_complete() argument
1509 r = dm_table_determine_type(t); in dm_table_complete()
1515 r = dm_table_build_index(t); in dm_table_complete()
1521 r = dm_table_construct_crypto_profile(t); in dm_table_complete()
1527 r = dm_table_alloc_md_mempools(t, t->md); in dm_table_complete()
1535 void dm_table_event_callback(struct dm_table *t, in dm_table_event_callback() argument
1539 t->event_fn = fn; in dm_table_event_callback()
1540 t->event_context = context; in dm_table_event_callback()
1544 void dm_table_event(struct dm_table *t) in dm_table_event() argument
1547 if (t->event_fn) in dm_table_event()
1548 t->event_fn(t->event_context); in dm_table_event()
1553 inline sector_t dm_table_get_size(struct dm_table *t) in dm_table_get_size() argument
1555 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0; in dm_table_get_size()
1565 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) in dm_table_find_target() argument
1570 if (unlikely(sector >= dm_table_get_size(t))) in dm_table_find_target()
1573 for (l = 0; l < t->depth; l++) { in dm_table_find_target()
1575 node = get_node(t, l, n); in dm_table_find_target()
1582 return &t->targets[(KEYS_PER_NODE * n) + k]; in dm_table_find_target()
1608 static bool dm_table_any_dev_attr(struct dm_table *t, in dm_table_any_dev_attr() argument
1611 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_any_dev_attr()
1612 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_any_dev_attr()
1638 bool dm_table_has_no_data_devices(struct dm_table *t) in dm_table_has_no_data_devices() argument
1640 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_has_no_data_devices()
1641 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_has_no_data_devices()
1655 bool dm_table_is_wildcard(struct dm_table *t) in dm_table_is_wildcard() argument
1657 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_is_wildcard()
1658 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_is_wildcard()
1688 static bool dm_table_supports_zoned(struct dm_table *t, bool zoned) in dm_table_supports_zoned() argument
1690 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_zoned()
1691 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_zoned()
1734 static int validate_hardware_zoned(struct dm_table *t, bool zoned, in validate_hardware_zoned() argument
1740 if (!dm_table_supports_zoned(t, zoned)) { in validate_hardware_zoned()
1742 dm_device_name(t->md)); in validate_hardware_zoned()
1750 if (dm_table_any_dev_attr(t, device_not_matches_zone_sectors, &zone_sectors)) { in validate_hardware_zoned()
1752 dm_device_name(t->md)); in validate_hardware_zoned()
1762 int dm_calculate_queue_limits(struct dm_table *t, in dm_calculate_queue_limits() argument
1771 t->integrity_supported = true; in dm_calculate_queue_limits()
1772 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_calculate_queue_limits()
1773 struct dm_target *ti = dm_table_get_target(t, i); in dm_calculate_queue_limits()
1776 t->integrity_supported = false; in dm_calculate_queue_limits()
1779 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_calculate_queue_limits()
1780 struct dm_target *ti = dm_table_get_target(t, i); in dm_calculate_queue_limits()
1826 dm_device_name(t->md), in dm_calculate_queue_limits()
1830 if (t->integrity_supported || in dm_calculate_queue_limits()
1835 dm_device_name(t->md), in dm_calculate_queue_limits()
1838 t->integrity_supported = false; in dm_calculate_queue_limits()
1858 if (validate_hardware_zoned(t, zoned, zone_sectors)) in dm_calculate_queue_limits()
1861 return validate_hardware_logical_block_alignment(t, limits); in dm_calculate_queue_limits()
1868 static bool dm_table_supports_flush(struct dm_table *t) in dm_table_supports_flush() argument
1870 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_flush()
1871 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_flush()
1906 static bool dm_table_supports_write_zeroes(struct dm_table *t) in dm_table_supports_write_zeroes() argument
1908 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_write_zeroes()
1909 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_write_zeroes()
1922 static bool dm_table_supports_nowait(struct dm_table *t) in dm_table_supports_nowait() argument
1924 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_nowait()
1925 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_nowait()
1940 static bool dm_table_supports_discards(struct dm_table *t) in dm_table_supports_discards() argument
1942 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_discards()
1943 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_discards()
1969 static bool dm_table_supports_secure_erase(struct dm_table *t) in dm_table_supports_secure_erase() argument
1971 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_secure_erase()
1972 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_secure_erase()
1992 static bool dm_table_supports_atomic_writes(struct dm_table *t) in dm_table_supports_atomic_writes() argument
1994 for (unsigned int i = 0; i < t->num_targets; i++) { in dm_table_supports_atomic_writes()
1995 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_supports_atomic_writes()
2011 bool dm_table_supports_size_change(struct dm_table *t, sector_t old_size, in dm_table_supports_size_change() argument
2014 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && dm_has_zone_plugs(t->md) && in dm_table_supports_size_change()
2018 dm_device_name(t->md)); in dm_table_supports_size_change()
2028 int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, in dm_table_set_restrictions() argument
2034 if (!dm_table_supports_nowait(t)) in dm_table_set_restrictions()
2041 if (!__table_type_bio_based(t->type)) in dm_table_set_restrictions()
2044 if (!dm_table_supports_discards(t)) { in dm_table_set_restrictions()
2050 if (!dm_table_supports_write_zeroes(t)) { in dm_table_set_restrictions()
2055 if (!dm_table_supports_secure_erase(t)) in dm_table_set_restrictions()
2058 if (dm_table_supports_flush(t)) in dm_table_set_restrictions()
2061 if (dm_table_supports_dax(t, device_not_dax_capable)) in dm_table_set_restrictions()
2069 r = dm_set_zones_restrictions(t, q, limits); in dm_table_set_restrictions()
2072 } else if (dm_has_zone_plugs(t->md)) { in dm_table_set_restrictions()
2075 dm_device_name(t->md)); in dm_table_set_restrictions()
2080 if (dm_table_supports_atomic_writes(t)) in dm_table_set_restrictions()
2094 r = dm_revalidate_zones(t, q); in dm_table_set_restrictions()
2102 dm_finalize_zone_settings(t, limits); in dm_table_set_restrictions()
2104 if (dm_table_supports_dax(t, device_not_dax_synchronous_capable)) in dm_table_set_restrictions()
2105 set_dax_synchronous(t->md->dax_dev); in dm_table_set_restrictions()
2107 if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL)) in dm_table_set_restrictions()
2108 dax_write_cache(t->md->dax_dev, true); in dm_table_set_restrictions()
2110 dm_update_crypto_profile(q, t); in dm_table_set_restrictions()
2114 struct list_head *dm_table_get_devices(struct dm_table *t) in dm_table_get_devices() argument
2116 return &t->devices; in dm_table_get_devices()
2119 blk_mode_t dm_table_get_mode(struct dm_table *t) in dm_table_get_mode() argument
2121 return t->mode; in dm_table_get_mode()
2131 static void suspend_targets(struct dm_table *t, enum suspend_mode mode) in suspend_targets() argument
2133 lockdep_assert_held(&t->md->suspend_lock); in suspend_targets()
2135 for (unsigned int i = 0; i < t->num_targets; i++) { in suspend_targets()
2136 struct dm_target *ti = dm_table_get_target(t, i); in suspend_targets()
2155 void dm_table_presuspend_targets(struct dm_table *t) in dm_table_presuspend_targets() argument
2157 if (!t) in dm_table_presuspend_targets()
2160 suspend_targets(t, PRESUSPEND); in dm_table_presuspend_targets()
2163 void dm_table_presuspend_undo_targets(struct dm_table *t) in dm_table_presuspend_undo_targets() argument
2165 if (!t) in dm_table_presuspend_undo_targets()
2168 suspend_targets(t, PRESUSPEND_UNDO); in dm_table_presuspend_undo_targets()
2171 void dm_table_postsuspend_targets(struct dm_table *t) in dm_table_postsuspend_targets() argument
2173 if (!t) in dm_table_postsuspend_targets()
2176 suspend_targets(t, POSTSUSPEND); in dm_table_postsuspend_targets()
2179 int dm_table_resume_targets(struct dm_table *t) in dm_table_resume_targets() argument
2184 lockdep_assert_held(&t->md->suspend_lock); in dm_table_resume_targets()
2186 for (i = 0; i < t->num_targets; i++) { in dm_table_resume_targets()
2187 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_resume_targets()
2195 dm_device_name(t->md), ti->type->name, r); in dm_table_resume_targets()
2200 for (i = 0; i < t->num_targets; i++) { in dm_table_resume_targets()
2201 struct dm_target *ti = dm_table_get_target(t, i); in dm_table_resume_targets()
2210 struct mapped_device *dm_table_get_md(struct dm_table *t) in dm_table_get_md() argument
2212 return t->md; in dm_table_get_md()
2216 const char *dm_table_device_name(struct dm_table *t) in dm_table_device_name() argument
2218 return dm_device_name(t->md); in dm_table_device_name()
2222 void dm_table_run_md_queue_async(struct dm_table *t) in dm_table_run_md_queue_async() argument
2224 if (!dm_table_request_based(t)) in dm_table_run_md_queue_async()
2227 if (t->md->queue) in dm_table_run_md_queue_async()
2228 blk_mq_run_hw_queues(t->md->queue, true); in dm_table_run_md_queue_async()