Lines Matching +full:add +full:- +full:pmem
1 // SPDX-License-Identifier: GPL-2.0-only
37 struct cxl_region_params *p = &cxlr->params; in uuid_show()
43 if (cxlr->mode != CXL_DECODER_PMEM) in uuid_show()
46 rc = sysfs_emit(buf, "%pUb\n", &p->uuid); in uuid_show()
63 p = &cxlr->params; in is_dup()
65 if (uuid_equal(&p->uuid, uuid)) { in is_dup()
67 return -EBUSY; in is_dup()
77 struct cxl_region_params *p = &cxlr->params; in uuid_store()
82 return -EINVAL; in uuid_store()
89 return -EINVAL; in uuid_store()
95 if (uuid_equal(&p->uuid, &temp)) in uuid_store()
98 rc = -EBUSY; in uuid_store()
99 if (p->state >= CXL_CONFIG_ACTIVE) in uuid_store()
106 uuid_copy(&p->uuid, &temp); in uuid_store()
119 return xa_load(&port->regions, (unsigned long)cxlr); in cxl_rr_load()
127 &cxlr->dev, in cxl_region_invalidate_memregion()
131 dev_err(&cxlr->dev, in cxl_region_invalidate_memregion()
133 return -ENXIO; in cxl_region_invalidate_memregion()
143 struct cxl_region_params *p = &cxlr->params; in cxl_region_decode_reset()
155 for (i = count - 1; i >= 0; i--) { in cxl_region_decode_reset()
156 struct cxl_endpoint_decoder *cxled = p->targets[i]; in cxl_region_decode_reset()
159 struct cxl_dev_state *cxlds = cxlmd->cxlds; in cxl_region_decode_reset()
162 if (cxlds->rcd) in cxl_region_decode_reset()
165 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) in cxl_region_decode_reset()
166 iter = to_cxl_port(iter->dev.parent); in cxl_region_decode_reset()
169 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { in cxl_region_decode_reset()
174 cxld = cxl_rr->decoder; in cxl_region_decode_reset()
175 if (cxld->reset) in cxl_region_decode_reset()
176 rc = cxld->reset(cxld); in cxl_region_decode_reset()
179 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); in cxl_region_decode_reset()
183 rc = cxled->cxld.reset(&cxled->cxld); in cxl_region_decode_reset()
186 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); in cxl_region_decode_reset()
190 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); in cxl_region_decode_reset()
199 if (cxld->commit) in commit_decoder()
200 return cxld->commit(cxld); in commit_decoder()
202 if (is_switch_decoder(&cxld->dev)) in commit_decoder()
203 cxlsd = to_cxl_switch_decoder(&cxld->dev); in commit_decoder()
205 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1, in commit_decoder()
206 "->commit() is required\n")) in commit_decoder()
207 return -ENXIO; in commit_decoder()
213 struct cxl_region_params *p = &cxlr->params; in cxl_region_decode_commit()
216 for (i = 0; i < p->nr_targets; i++) { in cxl_region_decode_commit()
217 struct cxl_endpoint_decoder *cxled = p->targets[i]; in cxl_region_decode_commit()
226 iter = to_cxl_port(iter->dev.parent)) { in cxl_region_decode_commit()
228 cxld = cxl_rr->decoder; in cxl_region_decode_commit()
237 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { in cxl_region_decode_commit()
239 cxld = cxl_rr->decoder; in cxl_region_decode_commit()
240 if (cxld->reset) in cxl_region_decode_commit()
241 cxld->reset(cxld); in cxl_region_decode_commit()
244 cxled->cxld.reset(&cxled->cxld); in cxl_region_decode_commit()
261 struct cxl_region_params *p = &cxlr->params; in commit_store()
274 if (commit && p->state >= CXL_CONFIG_COMMIT) in commit_store()
276 if (!commit && p->state < CXL_CONFIG_COMMIT) in commit_store()
280 if (commit && p->state < CXL_CONFIG_ACTIVE) { in commit_store()
281 rc = -ENXIO; in commit_store()
296 p->state = CXL_CONFIG_COMMIT; in commit_store()
298 p->state = CXL_CONFIG_RESET_PENDING; in commit_store()
300 device_release_driver(&cxlr->dev); in commit_store()
307 if (p->state == CXL_CONFIG_RESET_PENDING) { in commit_store()
308 rc = cxl_region_decode_reset(cxlr, p->interleave_ways); in commit_store()
315 p->state = CXL_CONFIG_COMMIT; in commit_store()
317 p->state = CXL_CONFIG_ACTIVE; in commit_store()
333 struct cxl_region_params *p = &cxlr->params; in commit_show()
339 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); in commit_show()
356 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) in cxl_region_visible()
358 return a->mode; in cxl_region_visible()
365 struct cxl_region_params *p = &cxlr->params; in interleave_ways_show()
371 rc = sysfs_emit(buf, "%d\n", p->interleave_ways); in interleave_ways_show()
383 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); in interleave_ways_store()
384 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; in interleave_ways_store()
386 struct cxl_region_params *p = &cxlr->params; in interleave_ways_store()
403 if (!is_power_of_2(val / cxld->interleave_ways) || in interleave_ways_store()
404 (val % cxld->interleave_ways)) { in interleave_ways_store()
405 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); in interleave_ways_store()
406 return -EINVAL; in interleave_ways_store()
412 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { in interleave_ways_store()
413 rc = -EBUSY; in interleave_ways_store()
417 save = p->interleave_ways; in interleave_ways_store()
418 p->interleave_ways = val; in interleave_ways_store()
419 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); in interleave_ways_store()
421 p->interleave_ways = save; in interleave_ways_store()
435 struct cxl_region_params *p = &cxlr->params; in interleave_granularity_show()
441 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); in interleave_granularity_show()
451 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); in interleave_granularity_store()
452 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; in interleave_granularity_store()
454 struct cxl_region_params *p = &cxlr->params; in interleave_granularity_store()
467 * When the host-bridge is interleaved, disallow region granularity != in interleave_granularity_store()
474 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) in interleave_granularity_store()
475 return -EINVAL; in interleave_granularity_store()
480 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { in interleave_granularity_store()
481 rc = -EBUSY; in interleave_granularity_store()
485 p->interleave_granularity = val; in interleave_granularity_store()
498 struct cxl_region_params *p = &cxlr->params; in resource_show()
499 u64 resource = -1ULL; in resource_show()
505 if (p->res) in resource_show()
506 resource = p->res->start; in resource_show()
519 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode)); in mode_show()
525 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); in alloc_hpa()
526 struct cxl_region_params *p = &cxlr->params; in alloc_hpa()
533 if (p->res && resource_size(p->res) == size) in alloc_hpa()
537 if (p->res) in alloc_hpa()
538 return -EBUSY; in alloc_hpa()
540 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) in alloc_hpa()
541 return -EBUSY; in alloc_hpa()
543 /* ways, granularity and uuid (if PMEM) need to be set before HPA */ in alloc_hpa()
544 if (!p->interleave_ways || !p->interleave_granularity || in alloc_hpa()
545 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) in alloc_hpa()
546 return -ENXIO; in alloc_hpa()
548 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder); in alloc_hpa()
550 return -EINVAL; in alloc_hpa()
552 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, in alloc_hpa()
553 dev_name(&cxlr->dev)); in alloc_hpa()
555 dev_dbg(&cxlr->dev, in alloc_hpa()
557 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res); in alloc_hpa()
561 p->res = res; in alloc_hpa()
562 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; in alloc_hpa()
569 struct cxl_region_params *p = &cxlr->params; in cxl_region_iomem_release()
571 if (device_is_registered(&cxlr->dev)) in cxl_region_iomem_release()
573 if (p->res) { in cxl_region_iomem_release()
578 if (p->res->parent) in cxl_region_iomem_release()
579 remove_resource(p->res); in cxl_region_iomem_release()
580 kfree(p->res); in cxl_region_iomem_release()
581 p->res = NULL; in cxl_region_iomem_release()
587 struct cxl_region_params *p = &cxlr->params; in free_hpa()
591 if (!p->res) in free_hpa()
594 if (p->state >= CXL_CONFIG_ACTIVE) in free_hpa()
595 return -EBUSY; in free_hpa()
598 p->state = CXL_CONFIG_IDLE; in free_hpa()
633 struct cxl_region_params *p = &cxlr->params; in size_show()
640 if (p->res) in size_show()
641 size = resource_size(p->res); in size_show()
667 struct cxl_region_params *p = &cxlr->params; in show_targetN()
675 if (pos >= p->interleave_ways) { in show_targetN()
676 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, in show_targetN()
677 p->interleave_ways); in show_targetN()
678 rc = -ENXIO; in show_targetN()
682 cxled = p->targets[pos]; in show_targetN()
686 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); in show_targetN()
704 if (cxld->id != *id) in match_free_decoder()
707 if (!cxld->region) in match_free_decoder()
725 r = &cxld->hpa_range; in match_auto_decoder()
727 if (p->res && p->res->start == r->start && p->res->end == r->end) in match_auto_decoder()
742 return &cxled->cxld; in cxl_region_find_decoder()
744 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) in cxl_region_find_decoder()
745 dev = device_find_child(&port->dev, &cxlr->params, in cxl_region_find_decoder()
748 dev = device_find_child(&port->dev, &id, match_free_decoder); in cxl_region_find_decoder()
765 struct cxl_decoder *cxld_iter = rr->decoder; in auto_order_ok()
768 * Allow the out of order assembly of auto-discovered regions. in auto_order_ok()
773 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n", in auto_order_ok()
774 dev_name(&cxld->dev), cxld->id, in auto_order_ok()
775 dev_name(&cxld_iter->dev), cxld_iter->id); in auto_order_ok()
777 if (cxld_iter->id > cxld->id) in auto_order_ok()
787 struct cxl_region_params *p = &cxlr->params; in alloc_region_ref()
792 xa_for_each(&port->regions, index, iter) { in alloc_region_ref()
793 struct cxl_region_params *ip = &iter->region->params; in alloc_region_ref()
795 if (!ip->res || ip->res->start < p->res->start) in alloc_region_ref()
798 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { in alloc_region_ref()
802 if (auto_order_ok(port, iter->region, cxld)) in alloc_region_ref()
805 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n", in alloc_region_ref()
806 dev_name(&port->dev), in alloc_region_ref()
807 dev_name(&iter->region->dev), ip->res, p->res); in alloc_region_ref()
809 return ERR_PTR(-EBUSY); in alloc_region_ref()
814 return ERR_PTR(-ENOMEM); in alloc_region_ref()
815 cxl_rr->port = port; in alloc_region_ref()
816 cxl_rr->region = cxlr; in alloc_region_ref()
817 cxl_rr->nr_targets = 1; in alloc_region_ref()
818 xa_init(&cxl_rr->endpoints); in alloc_region_ref()
820 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); in alloc_region_ref()
822 dev_dbg(&cxlr->dev, in alloc_region_ref()
824 dev_name(&port->dev), rc); in alloc_region_ref()
834 struct cxl_region *cxlr = cxl_rr->region; in cxl_rr_free_decoder()
835 struct cxl_decoder *cxld = cxl_rr->decoder; in cxl_rr_free_decoder()
840 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); in cxl_rr_free_decoder()
841 if (cxld->region == cxlr) { in cxl_rr_free_decoder()
842 cxld->region = NULL; in cxl_rr_free_decoder()
843 put_device(&cxlr->dev); in cxl_rr_free_decoder()
849 struct cxl_port *port = cxl_rr->port; in free_region_ref()
850 struct cxl_region *cxlr = cxl_rr->region; in free_region_ref()
853 xa_erase(&port->regions, (unsigned long)cxlr); in free_region_ref()
854 xa_destroy(&cxl_rr->endpoints); in free_region_ref()
862 struct cxl_port *port = cxl_rr->port; in cxl_rr_ep_add()
863 struct cxl_region *cxlr = cxl_rr->region; in cxl_rr_ep_add()
864 struct cxl_decoder *cxld = cxl_rr->decoder; in cxl_rr_ep_add()
868 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, in cxl_rr_ep_add()
873 cxl_rr->nr_eps++; in cxl_rr_ep_add()
875 if (!cxld->region) { in cxl_rr_ep_add()
876 cxld->region = cxlr; in cxl_rr_ep_add()
877 get_device(&cxlr->dev); in cxl_rr_ep_add()
891 dev_dbg(&cxlr->dev, "%s: no decoder available\n", in cxl_rr_alloc_decoder()
892 dev_name(&port->dev)); in cxl_rr_alloc_decoder()
893 return -EBUSY; in cxl_rr_alloc_decoder()
896 if (cxld->region) { in cxl_rr_alloc_decoder()
897 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", in cxl_rr_alloc_decoder()
898 dev_name(&port->dev), dev_name(&cxld->dev), in cxl_rr_alloc_decoder()
899 dev_name(&cxld->region->dev)); in cxl_rr_alloc_decoder()
900 return -EBUSY; in cxl_rr_alloc_decoder()
905 * assumption with an assertion. Switch-decoders change mapping-type in cxl_rr_alloc_decoder()
908 dev_WARN_ONCE(&cxlr->dev, in cxl_rr_alloc_decoder()
910 cxld->target_type != cxlr->type, in cxl_rr_alloc_decoder()
911 "%s:%s mismatch decoder type %d -> %d\n", in cxl_rr_alloc_decoder()
912 dev_name(&cxled_to_memdev(cxled)->dev), in cxl_rr_alloc_decoder()
913 dev_name(&cxld->dev), cxld->target_type, cxlr->type); in cxl_rr_alloc_decoder()
914 cxld->target_type = cxlr->type; in cxl_rr_alloc_decoder()
915 cxl_rr->decoder = cxld; in cxl_rr_alloc_decoder()
920 * cxl_port_attach_region() - track a region's interest in a port by endpoint
921 * @port: port to add a new region reference 'struct cxl_region_ref'
932 * - validate that there are no other regions with a higher HPA already
934 * - establish a region reference if one is not already present
936 * - additionally allocate a decoder instance that will host @cxlr on
939 * - pin the region reference by the endpoint
940 * - account for how many entries in @port's target list are needed to
953 int rc = -EBUSY; in cxl_port_attach_region()
968 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { in cxl_port_attach_region()
971 if (ep_iter->next == ep->next) { in cxl_port_attach_region()
981 if (!found || !ep->next) { in cxl_port_attach_region()
982 cxl_rr->nr_targets++; in cxl_port_attach_region()
988 dev_dbg(&cxlr->dev, in cxl_port_attach_region()
990 dev_name(&port->dev)); in cxl_port_attach_region()
999 cxld = cxl_rr->decoder; in cxl_port_attach_region()
1003 dev_dbg(&cxlr->dev, in cxl_port_attach_region()
1005 dev_name(&port->dev), dev_name(&cxlmd->dev), in cxl_port_attach_region()
1006 dev_name(&cxld->dev)); in cxl_port_attach_region()
1010 dev_dbg(&cxlr->dev, in cxl_port_attach_region()
1011 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", in cxl_port_attach_region()
1012 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_attach_region()
1013 dev_name(&cxld->dev), dev_name(&cxlmd->dev), in cxl_port_attach_region()
1014 dev_name(&cxled->cxld.dev), pos, in cxl_port_attach_region()
1015 ep ? ep->next ? dev_name(ep->next->uport_dev) : in cxl_port_attach_region()
1016 dev_name(&cxlmd->dev) : in cxl_port_attach_region()
1018 cxl_rr->nr_eps, cxl_rr->nr_targets); in cxl_port_attach_region()
1023 cxl_rr->nr_targets--; in cxl_port_attach_region()
1024 if (cxl_rr->nr_eps == 0) in cxl_port_attach_region()
1046 if (cxl_rr->decoder == &cxled->cxld) in cxl_port_detach_region()
1047 cxl_rr->nr_eps--; in cxl_port_detach_region()
1049 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); in cxl_port_detach_region()
1055 cxl_rr->nr_eps--; in cxl_port_detach_region()
1056 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { in cxl_port_detach_region()
1057 if (ep_iter->next == ep->next) { in cxl_port_detach_region()
1063 cxl_rr->nr_targets--; in cxl_port_detach_region()
1066 if (cxl_rr->nr_eps == 0) in cxl_port_detach_region()
1075 struct cxl_region *cxlr = cxl_rr->region; in check_last_peer()
1076 struct cxl_region_params *p = &cxlr->params; in check_last_peer()
1078 struct cxl_port *port = cxl_rr->port; in check_last_peer()
1081 int pos = cxled->pos; in check_last_peer()
1085 * then that endpoint, at index 'position - distance', must also be in check_last_peer()
1089 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", in check_last_peer()
1090 dev_name(port->uport_dev), dev_name(&port->dev), in check_last_peer()
1091 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); in check_last_peer()
1092 return -ENXIO; in check_last_peer()
1094 cxled_peer = p->targets[pos - distance]; in check_last_peer()
1097 if (ep->dport != ep_peer->dport) { in check_last_peer()
1098 dev_dbg(&cxlr->dev, in check_last_peer()
1100 dev_name(port->uport_dev), dev_name(&port->dev), in check_last_peer()
1101 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, in check_last_peer()
1102 dev_name(&cxlmd_peer->dev), in check_last_peer()
1103 dev_name(&cxled_peer->cxld.dev)); in check_last_peer()
1104 return -ENXIO; in check_last_peer()
1114 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); in cxl_port_setup_targets()
1115 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; in cxl_port_setup_targets()
1116 struct cxl_port *parent_port = to_cxl_port(port->dev.parent); in cxl_port_setup_targets()
1120 struct cxl_region_params *p = &cxlr->params; in cxl_port_setup_targets()
1121 struct cxl_decoder *cxld = cxl_rr->decoder; in cxl_port_setup_targets()
1130 if (!is_power_of_2(cxl_rr->nr_targets)) { in cxl_port_setup_targets()
1131 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", in cxl_port_setup_targets()
1132 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1133 cxl_rr->nr_targets); in cxl_port_setup_targets()
1134 return -EINVAL; in cxl_port_setup_targets()
1137 cxlsd = to_cxl_switch_decoder(&cxld->dev); in cxl_port_setup_targets()
1138 if (cxl_rr->nr_targets_set) { in cxl_port_setup_targets()
1145 if (cxl_rr->nr_targets == 1) in cxl_port_setup_targets()
1148 distance = p->nr_targets / cxl_rr->nr_targets; in cxl_port_setup_targets()
1149 for (i = 0; i < cxl_rr->nr_targets_set; i++) in cxl_port_setup_targets()
1150 if (ep->dport == cxlsd->target[i]) { in cxl_port_setup_targets()
1165 * does not allow interleaved host-bridges with in cxl_port_setup_targets()
1168 parent_ig = p->interleave_granularity; in cxl_port_setup_targets()
1169 parent_iw = cxlrd->cxlsd.cxld.interleave_ways; in cxl_port_setup_targets()
1171 * For purposes of address bit routing, use power-of-2 math for in cxl_port_setup_targets()
1181 parent_cxld = parent_rr->decoder; in cxl_port_setup_targets()
1182 parent_ig = parent_cxld->interleave_granularity; in cxl_port_setup_targets()
1183 parent_iw = parent_cxld->interleave_ways; in cxl_port_setup_targets()
1188 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", in cxl_port_setup_targets()
1189 dev_name(parent_port->uport_dev), in cxl_port_setup_targets()
1190 dev_name(&parent_port->dev), parent_ig); in cxl_port_setup_targets()
1196 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", in cxl_port_setup_targets()
1197 dev_name(parent_port->uport_dev), in cxl_port_setup_targets()
1198 dev_name(&parent_port->dev), parent_iw); in cxl_port_setup_targets()
1202 iw = cxl_rr->nr_targets; in cxl_port_setup_targets()
1205 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", in cxl_port_setup_targets()
1206 dev_name(port->uport_dev), dev_name(&port->dev), iw); in cxl_port_setup_targets()
1216 dev_dbg(&cxlr->dev, in cxl_port_setup_targets()
1218 dev_name(&parent_port->dev), parent_ig, parent_iw); in cxl_port_setup_targets()
1224 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", in cxl_port_setup_targets()
1225 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1230 if (iw > 8 || iw > cxlsd->nr_targets) { in cxl_port_setup_targets()
1231 dev_dbg(&cxlr->dev, in cxl_port_setup_targets()
1233 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1234 dev_name(&cxld->dev), iw, cxlsd->nr_targets); in cxl_port_setup_targets()
1235 return -ENXIO; in cxl_port_setup_targets()
1238 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { in cxl_port_setup_targets()
1239 if (cxld->interleave_ways != iw || in cxl_port_setup_targets()
1240 cxld->interleave_granularity != ig || in cxl_port_setup_targets()
1241 cxld->hpa_range.start != p->res->start || in cxl_port_setup_targets()
1242 cxld->hpa_range.end != p->res->end || in cxl_port_setup_targets()
1243 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { in cxl_port_setup_targets()
1244 dev_err(&cxlr->dev, in cxl_port_setup_targets()
1246 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1247 __func__, iw, ig, p->res); in cxl_port_setup_targets()
1248 dev_err(&cxlr->dev, in cxl_port_setup_targets()
1250 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1251 __func__, cxld->interleave_ways, in cxl_port_setup_targets()
1252 cxld->interleave_granularity, in cxl_port_setup_targets()
1253 (cxld->flags & CXL_DECODER_F_ENABLE) ? in cxl_port_setup_targets()
1256 cxld->hpa_range.start, cxld->hpa_range.end); in cxl_port_setup_targets()
1257 return -ENXIO; in cxl_port_setup_targets()
1260 cxld->interleave_ways = iw; in cxl_port_setup_targets()
1261 cxld->interleave_granularity = ig; in cxl_port_setup_targets()
1262 cxld->hpa_range = (struct range) { in cxl_port_setup_targets()
1263 .start = p->res->start, in cxl_port_setup_targets()
1264 .end = p->res->end, in cxl_port_setup_targets()
1267 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev), in cxl_port_setup_targets()
1268 dev_name(&port->dev), iw, ig); in cxl_port_setup_targets()
1270 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { in cxl_port_setup_targets()
1271 dev_dbg(&cxlr->dev, in cxl_port_setup_targets()
1272 "%s:%s: targets full trying to add %s:%s at %d\n", in cxl_port_setup_targets()
1273 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1274 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); in cxl_port_setup_targets()
1275 return -ENXIO; in cxl_port_setup_targets()
1277 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { in cxl_port_setup_targets()
1278 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { in cxl_port_setup_targets()
1279 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", in cxl_port_setup_targets()
1280 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1281 dev_name(&cxlsd->cxld.dev), in cxl_port_setup_targets()
1282 dev_name(ep->dport->dport_dev), in cxl_port_setup_targets()
1283 cxl_rr->nr_targets_set); in cxl_port_setup_targets()
1284 return -ENXIO; in cxl_port_setup_targets()
1287 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; in cxl_port_setup_targets()
1290 cxl_rr->nr_targets_set += inc; in cxl_port_setup_targets()
1291 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", in cxl_port_setup_targets()
1292 dev_name(port->uport_dev), dev_name(&port->dev), in cxl_port_setup_targets()
1293 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), in cxl_port_setup_targets()
1294 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); in cxl_port_setup_targets()
1311 cxl_rr->nr_targets_set = 0; in cxl_port_reset_targets()
1313 cxld = cxl_rr->decoder; in cxl_port_reset_targets()
1314 cxld->hpa_range = (struct range) { in cxl_port_reset_targets()
1316 .end = -1, in cxl_port_reset_targets()
1322 struct cxl_region_params *p = &cxlr->params; in cxl_region_teardown_targets()
1331 * In the auto-discovery case skip automatic teardown since the in cxl_region_teardown_targets()
1334 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) in cxl_region_teardown_targets()
1337 for (i = 0; i < p->nr_targets; i++) { in cxl_region_teardown_targets()
1338 cxled = p->targets[i]; in cxl_region_teardown_targets()
1340 cxlds = cxlmd->cxlds; in cxl_region_teardown_targets()
1342 if (cxlds->rcd) in cxl_region_teardown_targets()
1346 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) in cxl_region_teardown_targets()
1347 iter = to_cxl_port(iter->dev.parent); in cxl_region_teardown_targets()
1350 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) in cxl_region_teardown_targets()
1357 struct cxl_region_params *p = &cxlr->params; in cxl_region_setup_targets()
1365 for (i = 0; i < p->nr_targets; i++) { in cxl_region_setup_targets()
1366 cxled = p->targets[i]; in cxl_region_setup_targets()
1368 cxlds = cxlmd->cxlds; in cxl_region_setup_targets()
1371 if (!cxlds->rcd) { in cxl_region_setup_targets()
1379 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) in cxl_region_setup_targets()
1380 iter = to_cxl_port(iter->dev.parent); in cxl_region_setup_targets()
1387 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { in cxl_region_setup_targets()
1397 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n"); in cxl_region_setup_targets()
1399 return -ENXIO; in cxl_region_setup_targets()
1410 struct cxl_region_params *p = &cxlr->params; in cxl_region_validate_position()
1413 if (pos < 0 || pos >= p->interleave_ways) { in cxl_region_validate_position()
1414 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, in cxl_region_validate_position()
1415 p->interleave_ways); in cxl_region_validate_position()
1416 return -ENXIO; in cxl_region_validate_position()
1419 if (p->targets[pos] == cxled) in cxl_region_validate_position()
1422 if (p->targets[pos]) { in cxl_region_validate_position()
1423 struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; in cxl_region_validate_position()
1426 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", in cxl_region_validate_position()
1427 pos, dev_name(&cxlmd_target->dev), in cxl_region_validate_position()
1428 dev_name(&cxled_target->cxld.dev)); in cxl_region_validate_position()
1429 return -EBUSY; in cxl_region_validate_position()
1432 for (i = 0; i < p->interleave_ways; i++) { in cxl_region_validate_position()
1436 cxled_target = p->targets[i]; in cxl_region_validate_position()
1442 dev_dbg(&cxlr->dev, in cxl_region_validate_position()
1444 dev_name(&cxlmd->dev), pos, in cxl_region_validate_position()
1445 dev_name(&cxled_target->cxld.dev)); in cxl_region_validate_position()
1446 return -EBUSY; in cxl_region_validate_position()
1462 if (cxlrd->calc_hb(cxlrd, pos) != dport) { in cxl_region_attach_position()
1463 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", in cxl_region_attach_position()
1464 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in cxl_region_attach_position()
1465 dev_name(&cxlrd->cxlsd.cxld.dev)); in cxl_region_attach_position()
1466 return -ENXIO; in cxl_region_attach_position()
1470 iter = to_cxl_port(iter->dev.parent)) { in cxl_region_attach_position()
1480 iter = to_cxl_port(iter->dev.parent)) in cxl_region_attach_position()
1488 struct cxl_region_params *p = &cxlr->params; in cxl_region_attach_auto()
1490 if (cxled->state != CXL_DECODER_STATE_AUTO) { in cxl_region_attach_auto()
1491 dev_err(&cxlr->dev, in cxl_region_attach_auto()
1492 "%s: unable to add decoder to autodetected region\n", in cxl_region_attach_auto()
1493 dev_name(&cxled->cxld.dev)); in cxl_region_attach_auto()
1494 return -EINVAL; in cxl_region_attach_auto()
1498 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n", in cxl_region_attach_auto()
1499 dev_name(&cxled->cxld.dev), pos); in cxl_region_attach_auto()
1500 return -EINVAL; in cxl_region_attach_auto()
1503 if (p->nr_targets >= p->interleave_ways) { in cxl_region_attach_auto()
1504 dev_err(&cxlr->dev, "%s: no more target slots available\n", in cxl_region_attach_auto()
1505 dev_name(&cxled->cxld.dev)); in cxl_region_attach_auto()
1506 return -ENXIO; in cxl_region_attach_auto()
1515 pos = p->nr_targets; in cxl_region_attach_auto()
1516 p->targets[pos] = cxled; in cxl_region_attach_auto()
1517 cxled->pos = pos; in cxl_region_attach_auto()
1518 p->nr_targets++; in cxl_region_attach_auto()
1528 return cxled_a->pos - cxled_b->pos; in cmp_interleave_pos()
1533 if (!port->parent_dport) in next_port()
1535 return port->parent_dport->port; in next_port()
1547 r1 = &cxlsd->cxld.hpa_range; in match_switch_decoder_by_range()
1551 return (r1->start == r2->start && r1->end == r2->end); in match_switch_decoder_by_range()
1560 int rc = -ENXIO; in find_pos_and_ways()
1566 dev = device_find_child(&parent->dev, range, in find_pos_and_ways()
1569 dev_err(port->uport_dev, in find_pos_and_ways()
1570 "failed to find decoder mapping %#llx-%#llx\n", in find_pos_and_ways()
1571 range->start, range->end); in find_pos_and_ways()
1575 *ways = cxlsd->cxld.interleave_ways; in find_pos_and_ways()
1578 if (cxlsd->target[i] == port->parent_dport) { in find_pos_and_ways()
1590 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1602 * -ENXIO on failure
1608 struct range *range = &cxled->cxld.hpa_range; in cxl_calc_interleave_pos()
1613 * Example: the expected interleave order of the 4-way region shown in cxl_calc_interleave_pos()
1623 * uses the mem position in the host-bridge and the ways of the host- in cxl_calc_interleave_pos()
1625 * iteration uses the host-bridge position in the root_port and the ways in cxl_calc_interleave_pos()
1653 dev_dbg(&cxlmd->dev, in cxl_calc_interleave_pos()
1654 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n", in cxl_calc_interleave_pos()
1655 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent), in cxl_calc_interleave_pos()
1656 dev_name(&port->dev), range->start, range->end, pos); in cxl_calc_interleave_pos()
1663 struct cxl_region_params *p = &cxlr->params; in cxl_region_sort_targets()
1666 for (i = 0; i < p->nr_targets; i++) { in cxl_region_sort_targets()
1667 struct cxl_endpoint_decoder *cxled = p->targets[i]; in cxl_region_sort_targets()
1669 cxled->pos = cxl_calc_interleave_pos(cxled); in cxl_region_sort_targets()
1672 * cxled->pos so that follow-on code paths can reliably in cxl_region_sort_targets()
1673 * do p->targets[cxled->pos] to self-reference their entry. in cxl_region_sort_targets()
1675 if (cxled->pos < 0) in cxl_region_sort_targets()
1676 rc = -ENXIO; in cxl_region_sort_targets()
1679 sort(p->targets, p->nr_targets, sizeof(p->targets[0]), in cxl_region_sort_targets()
1682 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful"); in cxl_region_sort_targets()
1689 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); in cxl_region_attach()
1691 struct cxl_region_params *p = &cxlr->params; in cxl_region_attach()
1694 int rc = -ENXIO; in cxl_region_attach()
1696 if (cxled->mode != cxlr->mode) { in cxl_region_attach()
1697 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n", in cxl_region_attach()
1698 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode); in cxl_region_attach()
1699 return -EINVAL; in cxl_region_attach()
1702 if (cxled->mode == CXL_DECODER_DEAD) { in cxl_region_attach()
1703 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); in cxl_region_attach()
1704 return -ENODEV; in cxl_region_attach()
1708 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { in cxl_region_attach()
1709 dev_dbg(&cxlr->dev, "region already active\n"); in cxl_region_attach()
1710 return -EBUSY; in cxl_region_attach()
1711 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { in cxl_region_attach()
1712 dev_dbg(&cxlr->dev, "interleave config missing\n"); in cxl_region_attach()
1713 return -ENXIO; in cxl_region_attach()
1716 if (p->nr_targets >= p->interleave_ways) { in cxl_region_attach()
1717 dev_dbg(&cxlr->dev, "region already has %d endpoints\n", in cxl_region_attach()
1718 p->nr_targets); in cxl_region_attach()
1719 return -EINVAL; in cxl_region_attach()
1724 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); in cxl_region_attach()
1726 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", in cxl_region_attach()
1727 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in cxl_region_attach()
1728 dev_name(cxlr->dev.parent)); in cxl_region_attach()
1729 return -ENXIO; in cxl_region_attach()
1732 if (cxled->cxld.target_type != cxlr->type) { in cxl_region_attach()
1733 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", in cxl_region_attach()
1734 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in cxl_region_attach()
1735 cxled->cxld.target_type, cxlr->type); in cxl_region_attach()
1736 return -ENXIO; in cxl_region_attach()
1739 if (!cxled->dpa_res) { in cxl_region_attach()
1740 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", in cxl_region_attach()
1741 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); in cxl_region_attach()
1742 return -ENXIO; in cxl_region_attach()
1745 if (resource_size(cxled->dpa_res) * p->interleave_ways != in cxl_region_attach()
1746 resource_size(p->res)) { in cxl_region_attach()
1747 dev_dbg(&cxlr->dev, in cxl_region_attach()
1748 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", in cxl_region_attach()
1749 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in cxl_region_attach()
1750 (u64)resource_size(cxled->dpa_res), p->interleave_ways, in cxl_region_attach()
1751 (u64)resource_size(p->res)); in cxl_region_attach()
1752 return -EINVAL; in cxl_region_attach()
1755 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { in cxl_region_attach()
1763 if (p->nr_targets < p->interleave_ways) in cxl_region_attach()
1775 for (i = 0; i < p->nr_targets; i++) { in cxl_region_attach()
1776 cxled = p->targets[i]; in cxl_region_attach()
1779 ep_port->host_bridge); in cxl_region_attach()
1794 p->state = CXL_CONFIG_COMMIT; in cxl_region_attach()
1807 p->targets[pos] = cxled; in cxl_region_attach()
1808 cxled->pos = pos; in cxl_region_attach()
1809 p->nr_targets++; in cxl_region_attach()
1811 if (p->nr_targets == p->interleave_ways) { in cxl_region_attach()
1815 p->state = CXL_CONFIG_ACTIVE; in cxl_region_attach()
1818 cxled->cxld.interleave_ways = p->interleave_ways; in cxl_region_attach()
1819 cxled->cxld.interleave_granularity = p->interleave_granularity; in cxl_region_attach()
1820 cxled->cxld.hpa_range = (struct range) { in cxl_region_attach()
1821 .start = p->res->start, in cxl_region_attach()
1822 .end = p->res->end, in cxl_region_attach()
1825 if (p->nr_targets != p->interleave_ways) in cxl_region_attach()
1829 * Test the auto-discovery position calculator function in cxl_region_attach()
1830 * against this successfully created user-defined region. in cxl_region_attach()
1834 for (int i = 0; i < p->nr_targets; i++) { in cxl_region_attach()
1835 struct cxl_endpoint_decoder *cxled = p->targets[i]; in cxl_region_attach()
1839 dev_dbg(&cxled->cxld.dev, in cxl_region_attach()
1840 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n", in cxl_region_attach()
1841 (test_pos == cxled->pos) ? "success" : "fail", in cxl_region_attach()
1842 test_pos, cxled->pos); in cxl_region_attach()
1851 struct cxl_region *cxlr = cxled->cxld.region; in cxl_region_detach()
1860 p = &cxlr->params; in cxl_region_detach()
1861 get_device(&cxlr->dev); in cxl_region_detach()
1863 if (p->state > CXL_CONFIG_ACTIVE) { in cxl_region_detach()
1868 rc = cxl_region_decode_reset(cxlr, p->interleave_ways); in cxl_region_detach()
1871 p->state = CXL_CONFIG_ACTIVE; in cxl_region_detach()
1875 iter = to_cxl_port(iter->dev.parent)) in cxl_region_detach()
1878 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || in cxl_region_detach()
1879 p->targets[cxled->pos] != cxled) { in cxl_region_detach()
1882 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", in cxl_region_detach()
1883 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in cxl_region_detach()
1884 cxled->pos); in cxl_region_detach()
1888 if (p->state == CXL_CONFIG_ACTIVE) { in cxl_region_detach()
1889 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; in cxl_region_detach()
1892 p->targets[cxled->pos] = NULL; in cxl_region_detach()
1893 p->nr_targets--; in cxl_region_detach()
1894 cxled->cxld.hpa_range = (struct range) { in cxl_region_detach()
1896 .end = -1, in cxl_region_detach()
1901 device_release_driver(&cxlr->dev); in cxl_region_detach()
1904 put_device(&cxlr->dev); in cxl_region_detach()
1911 cxled->mode = CXL_DECODER_DEAD; in cxl_decoder_kill_region()
1938 struct cxl_region_params *p = &cxlr->params; in detach_target()
1945 if (pos >= p->interleave_ways) { in detach_target()
1946 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, in detach_target()
1947 p->interleave_ways); in detach_target()
1948 rc = -ENXIO; in detach_target()
1952 if (!p->targets[pos]) { in detach_target()
1957 rc = cxl_region_detach(p->targets[pos]); in detach_target()
1975 return -ENODEV; in store_targetN()
1978 rc = -EINVAL; in store_targetN()
2049 struct cxl_region_params *p = &cxlr->params; in cxl_region_target_visible()
2051 if (n < p->interleave_ways) in cxl_region_target_visible()
2052 return a->mode; in cxl_region_target_visible()
2075 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); in cxl_region_release()
2077 int id = atomic_read(&cxlrd->region_id); in cxl_region_release()
2084 if (cxlr->id < id) in cxl_region_release()
2085 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { in cxl_region_release()
2090 memregion_free(cxlr->id); in cxl_region_release()
2092 put_device(dev->parent); in cxl_region_release()
2104 return dev->type == &cxl_region_type; in is_cxl_region()
2110 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, in to_cxl_region()
2120 struct cxl_region_params *p = &cxlr->params; in unregister_region()
2123 device_del(&cxlr->dev); in unregister_region()
2127 * read-only, so no need to hold the region rwsem to access the in unregister_region()
2130 for (i = 0; i < p->interleave_ways; i++) in unregister_region()
2134 put_device(&cxlr->dev); in unregister_region()
2147 return ERR_PTR(-ENOMEM); in cxl_region_alloc()
2150 dev = &cxlr->dev; in cxl_region_alloc()
2152 lockdep_set_class(&dev->mutex, &cxl_region_key); in cxl_region_alloc()
2153 dev->parent = &cxlrd->cxlsd.cxld.dev; in cxl_region_alloc()
2158 get_device(dev->parent); in cxl_region_alloc()
2160 dev->bus = &cxl_bus_type; in cxl_region_alloc()
2161 dev->type = &cxl_region_type; in cxl_region_alloc()
2162 cxlr->id = id; in cxl_region_alloc()
2168 * devm_cxl_add_region - Adds a region to a decoder
2172 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2185 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); in devm_cxl_add_region()
2195 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode); in devm_cxl_add_region()
2196 return ERR_PTR(-EINVAL); in devm_cxl_add_region()
2202 cxlr->mode = mode; in devm_cxl_add_region()
2203 cxlr->type = type; in devm_cxl_add_region()
2205 dev = &cxlr->dev; in devm_cxl_add_region()
2214 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); in devm_cxl_add_region()
2218 dev_dbg(port->uport_dev, "%s: created %s\n", in devm_cxl_add_region()
2219 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); in devm_cxl_add_region()
2229 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); in __create_region_show()
2253 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { in __create_region()
2255 return ERR_PTR(-EBUSY); in __create_region()
2271 return -EINVAL; in create_pmem_region_store()
2291 return -EINVAL; in create_ram_region_store()
2311 if (cxld->region) in region_show()
2312 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); in region_show()
2324 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; in cxl_find_region_by_name()
2327 region_dev = device_find_child_by_name(&cxld->dev, name); in cxl_find_region_by_name()
2329 return ERR_PTR(-ENODEV); in cxl_find_region_by_name()
2339 struct cxl_port *port = to_cxl_port(dev->parent); in delete_region_store()
2346 devm_release_action(port->uport_dev, unregister_region, cxlr); in delete_region_store()
2347 put_device(&cxlr->dev); in delete_region_store()
2358 for (i = 0; i < cxlr_pmem->nr_mappings; i++) { in cxl_pmem_region_release()
2359 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; in cxl_pmem_region_release()
2361 put_device(&cxlmd->dev); in cxl_pmem_region_release()
2380 return dev->type == &cxl_pmem_region_type; in is_cxl_pmem_region()
2402 struct cxl_dev_state *cxlds = cxlmd->cxlds; in cxl_get_poison_unmapped()
2410 * Knowing that PMEM must always follow RAM, get poison in cxl_get_poison_unmapped()
2412 * ram: scan remains of ram range, then any pmem range in cxl_get_poison_unmapped()
2413 * pmem: scan remains of pmem range in cxl_get_poison_unmapped()
2416 if (ctx->mode == CXL_DECODER_RAM) { in cxl_get_poison_unmapped()
2417 offset = ctx->offset; in cxl_get_poison_unmapped()
2418 length = resource_size(&cxlds->ram_res) - offset; in cxl_get_poison_unmapped()
2420 if (rc == -EFAULT) in cxl_get_poison_unmapped()
2425 if (ctx->mode == CXL_DECODER_PMEM) { in cxl_get_poison_unmapped()
2426 offset = ctx->offset; in cxl_get_poison_unmapped()
2427 length = resource_size(&cxlds->dpa_res) - offset; in cxl_get_poison_unmapped()
2430 } else if (resource_size(&cxlds->pmem_res)) { in cxl_get_poison_unmapped()
2431 offset = cxlds->pmem_res.start; in cxl_get_poison_unmapped()
2432 length = resource_size(&cxlds->pmem_res); in cxl_get_poison_unmapped()
2452 if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) in poison_by_decoder()
2456 * Regions are only created with single mode decoders: pmem or ram. in poison_by_decoder()
2459 * that poison reads of pmem and ram must be separated. in poison_by_decoder()
2462 if (cxled->mode == CXL_DECODER_MIXED) { in poison_by_decoder()
2468 if (cxled->skip) { in poison_by_decoder()
2469 offset = cxled->dpa_res->start - cxled->skip; in poison_by_decoder()
2470 length = cxled->skip; in poison_by_decoder()
2472 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) in poison_by_decoder()
2478 offset = cxled->dpa_res->start; in poison_by_decoder()
2479 length = cxled->dpa_res->end - offset + 1; in poison_by_decoder()
2480 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); in poison_by_decoder()
2481 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) in poison_by_decoder()
2487 if (cxled->cxld.id == ctx->port->commit_end) { in poison_by_decoder()
2488 ctx->offset = cxled->dpa_res->end + 1; in poison_by_decoder()
2489 ctx->mode = cxled->mode; in poison_by_decoder()
2505 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); in cxl_get_poison_by_endpoint()
2507 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), in cxl_get_poison_by_endpoint()
2517 struct cxl_region_params *p = &cxlr->params; in cxl_pmem_region_alloc()
2524 if (p->state != CXL_CONFIG_COMMIT) { in cxl_pmem_region_alloc()
2525 cxlr_pmem = ERR_PTR(-ENXIO); in cxl_pmem_region_alloc()
2529 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), in cxl_pmem_region_alloc()
2532 cxlr_pmem = ERR_PTR(-ENOMEM); in cxl_pmem_region_alloc()
2536 cxlr_pmem->hpa_range.start = p->res->start; in cxl_pmem_region_alloc()
2537 cxlr_pmem->hpa_range.end = p->res->end; in cxl_pmem_region_alloc()
2540 cxlr_pmem->nr_mappings = p->nr_targets; in cxl_pmem_region_alloc()
2541 for (i = 0; i < p->nr_targets; i++) { in cxl_pmem_region_alloc()
2542 struct cxl_endpoint_decoder *cxled = p->targets[i]; in cxl_pmem_region_alloc()
2544 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; in cxl_pmem_region_alloc()
2553 cxlr_pmem = ERR_PTR(-ENODEV); in cxl_pmem_region_alloc()
2556 cxlr->cxl_nvb = cxl_nvb; in cxl_pmem_region_alloc()
2558 m->cxlmd = cxlmd; in cxl_pmem_region_alloc()
2559 get_device(&cxlmd->dev); in cxl_pmem_region_alloc()
2560 m->start = cxled->dpa_res->start; in cxl_pmem_region_alloc()
2561 m->size = resource_size(cxled->dpa_res); in cxl_pmem_region_alloc()
2562 m->position = i; in cxl_pmem_region_alloc()
2565 dev = &cxlr_pmem->dev; in cxl_pmem_region_alloc()
2566 cxlr_pmem->cxlr = cxlr; in cxl_pmem_region_alloc()
2567 cxlr->cxlr_pmem = cxlr_pmem; in cxl_pmem_region_alloc()
2569 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); in cxl_pmem_region_alloc()
2571 dev->parent = &cxlr->dev; in cxl_pmem_region_alloc()
2572 dev->bus = &cxl_bus_type; in cxl_pmem_region_alloc()
2573 dev->type = &cxl_pmem_region_type; in cxl_pmem_region_alloc()
2600 return dev->type == &cxl_dax_region_type; in is_cxl_dax_region()
2616 struct cxl_region_params *p = &cxlr->params; in cxl_dax_region_alloc()
2621 if (p->state != CXL_CONFIG_COMMIT) { in cxl_dax_region_alloc()
2622 cxlr_dax = ERR_PTR(-ENXIO); in cxl_dax_region_alloc()
2628 cxlr_dax = ERR_PTR(-ENOMEM); in cxl_dax_region_alloc()
2632 cxlr_dax->hpa_range.start = p->res->start; in cxl_dax_region_alloc()
2633 cxlr_dax->hpa_range.end = p->res->end; in cxl_dax_region_alloc()
2635 dev = &cxlr_dax->dev; in cxl_dax_region_alloc()
2636 cxlr_dax->cxlr = cxlr; in cxl_dax_region_alloc()
2638 lockdep_set_class(&dev->mutex, &cxl_dax_region_key); in cxl_dax_region_alloc()
2640 dev->parent = &cxlr->dev; in cxl_dax_region_alloc()
2641 dev->bus = &cxl_bus_type; in cxl_dax_region_alloc()
2642 dev->type = &cxl_dax_region_type; in cxl_dax_region_alloc()
2652 struct cxl_region *cxlr = cxlr_pmem->cxlr; in cxlr_pmem_unregister()
2653 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; in cxlr_pmem_unregister()
2656 * Either the bridge is in ->remove() context under the device_lock(), in cxlr_pmem_unregister()
2661 device_lock_assert(&cxl_nvb->dev); in cxlr_pmem_unregister()
2662 cxlr->cxlr_pmem = NULL; in cxlr_pmem_unregister()
2663 cxlr_pmem->cxlr = NULL; in cxlr_pmem_unregister()
2664 device_unregister(&cxlr_pmem->dev); in cxlr_pmem_unregister()
2670 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; in cxlr_release_nvdimm()
2672 device_lock(&cxl_nvb->dev); in cxlr_release_nvdimm()
2673 if (cxlr->cxlr_pmem) in cxlr_release_nvdimm()
2674 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, in cxlr_release_nvdimm()
2675 cxlr->cxlr_pmem); in cxlr_release_nvdimm()
2676 device_unlock(&cxl_nvb->dev); in cxlr_release_nvdimm()
2677 cxlr->cxl_nvb = NULL; in cxlr_release_nvdimm()
2678 put_device(&cxl_nvb->dev); in cxlr_release_nvdimm()
2682 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2683 * @cxlr: parent CXL region for this pmem region bridge device
2697 cxl_nvb = cxlr->cxl_nvb; in devm_cxl_add_pmem_region()
2699 dev = &cxlr_pmem->dev; in devm_cxl_add_pmem_region()
2700 rc = dev_set_name(dev, "pmem_region%d", cxlr->id); in devm_cxl_add_pmem_region()
2708 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), in devm_cxl_add_pmem_region()
2711 device_lock(&cxl_nvb->dev); in devm_cxl_add_pmem_region()
2712 if (cxl_nvb->dev.driver) in devm_cxl_add_pmem_region()
2713 rc = devm_add_action_or_reset(&cxl_nvb->dev, in devm_cxl_add_pmem_region()
2716 rc = -ENXIO; in devm_cxl_add_pmem_region()
2717 device_unlock(&cxl_nvb->dev); in devm_cxl_add_pmem_region()
2723 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); in devm_cxl_add_pmem_region()
2728 put_device(&cxl_nvb->dev); in devm_cxl_add_pmem_region()
2729 cxlr->cxl_nvb = NULL; in devm_cxl_add_pmem_region()
2737 device_unregister(&cxlr_dax->dev); in cxlr_dax_unregister()
2750 dev = &cxlr_dax->dev; in devm_cxl_add_dax_region()
2751 rc = dev_set_name(dev, "dax_region%d", cxlr->id); in devm_cxl_add_dax_region()
2759 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), in devm_cxl_add_dax_region()
2762 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister, in devm_cxl_add_dax_region()
2778 r1 = &cxlrd->cxlsd.cxld.hpa_range; in match_root_decoder_by_range()
2793 p = &cxlr->params; in match_region_by_range()
2796 if (p->res && p->res->start == r->start && p->res->end == r->end) in match_region_by_range()
2809 struct range *hpa = &cxled->cxld.hpa_range; in construct_region()
2816 cxlr = __create_region(cxlrd, cxled->mode, in construct_region()
2817 atomic_read(&cxlrd->region_id)); in construct_region()
2818 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY); in construct_region()
2821 dev_err(cxlmd->dev.parent, in construct_region()
2823 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in construct_region()
2829 p = &cxlr->params; in construct_region()
2830 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { in construct_region()
2831 dev_err(cxlmd->dev.parent, in construct_region()
2833 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in construct_region()
2835 rc = -EBUSY; in construct_region()
2839 set_bit(CXL_REGION_F_AUTO, &cxlr->flags); in construct_region()
2843 rc = -ENOMEM; in construct_region()
2847 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa), in construct_region()
2848 dev_name(&cxlr->dev)); in construct_region()
2849 rc = insert_resource(cxlrd->res, res); in construct_region()
2852 * Platform-firmware may not have split resources like "System in construct_region()
2855 dev_warn(cxlmd->dev.parent, in construct_region()
2857 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), in construct_region()
2858 __func__, dev_name(&cxlr->dev)); in construct_region()
2861 p->res = res; in construct_region()
2862 p->interleave_ways = cxled->cxld.interleave_ways; in construct_region()
2863 p->interleave_granularity = cxled->cxld.interleave_granularity; in construct_region()
2864 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; in construct_region()
2866 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); in construct_region()
2870 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n", in construct_region()
2871 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__, in construct_region()
2872 dev_name(&cxlr->dev), p->res, p->interleave_ways, in construct_region()
2873 p->interleave_granularity); in construct_region()
2876 get_device(&cxlr->dev); in construct_region()
2883 devm_release_action(port->uport_dev, unregister_region, cxlr); in construct_region()
2890 struct range *hpa = &cxled->cxld.hpa_range; in cxl_add_to_region()
2891 struct cxl_decoder *cxld = &cxled->cxld; in cxl_add_to_region()
2899 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range, in cxl_add_to_region()
2902 dev_err(cxlmd->dev.parent, in cxl_add_to_region()
2904 dev_name(&cxlmd->dev), dev_name(&cxld->dev), in cxl_add_to_region()
2905 cxld->hpa_range.start, cxld->hpa_range.end); in cxl_add_to_region()
2906 return -ENXIO; in cxl_add_to_region()
2913 * one does the construction and the others add to that. in cxl_add_to_region()
2915 mutex_lock(&cxlrd->range_lock); in cxl_add_to_region()
2916 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa, in cxl_add_to_region()
2920 region_dev = &cxlr->dev; in cxl_add_to_region()
2923 mutex_unlock(&cxlrd->range_lock); in cxl_add_to_region()
2929 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE); in cxl_add_to_region()
2932 p = &cxlr->params; in cxl_add_to_region()
2933 attach = p->state == CXL_CONFIG_COMMIT; in cxl_add_to_region()
2939 * the platform-firmware memory map, otherwise the driver for in cxl_add_to_region()
2942 if (device_attach(&cxlr->dev) < 0) in cxl_add_to_region()
2943 dev_err(&cxlr->dev, "failed to enable, range: %pr\n", in cxl_add_to_region()
2944 p->res); in cxl_add_to_region()
2957 struct cxl_region_params *p = &cxlr->params; in is_system_ram()
2959 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res); in is_system_ram()
2966 struct cxl_region_params *p = &cxlr->params; in cxl_region_probe()
2971 dev_dbg(&cxlr->dev, "probe interrupted\n"); in cxl_region_probe()
2975 if (p->state < CXL_CONFIG_COMMIT) { in cxl_region_probe()
2976 dev_dbg(&cxlr->dev, "config state: %d\n", p->state); in cxl_region_probe()
2977 rc = -ENXIO; in cxl_region_probe()
2981 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) { in cxl_region_probe()
2982 dev_err(&cxlr->dev, in cxl_region_probe()
2983 "failed to activate, re-commit region and retry\n"); in cxl_region_probe()
2984 rc = -ENXIO; in cxl_region_probe()
2998 switch (cxlr->mode) { in cxl_region_probe()
3008 p->res->start, p->res->end, cxlr, in cxl_region_probe()
3013 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", in cxl_region_probe()
3014 cxlr->mode); in cxl_region_probe()
3015 return -ENXIO; in cxl_region_probe()