1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. */
3
4 #include <linux/io-64-nonatomic-lo-hi.h>
5 #include <linux/firmware.h>
6 #include <linux/device.h>
7 #include <linux/slab.h>
8 #include <linux/idr.h>
9 #include <linux/pci.h>
10 #include <cxlmem.h>
11 #include "trace.h"
12 #include "core.h"
13
14 static DECLARE_RWSEM(cxl_memdev_rwsem);
15
16 /*
17 * An entire PCI topology full of devices should be enough for any
18 * config
19 */
20 #define CXL_MEM_MAX_DEVS 65536
21
22 static int cxl_mem_major;
23 static DEFINE_IDA(cxl_memdev_ida);
24
cxl_memdev_release(struct device * dev)25 static void cxl_memdev_release(struct device *dev)
26 {
27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
28
29 ida_free(&cxl_memdev_ida, cxlmd->id);
30 kfree(cxlmd);
31 }
32
cxl_memdev_devnode(const struct device * dev,umode_t * mode,kuid_t * uid,kgid_t * gid)33 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
34 kgid_t *gid)
35 {
36 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
37 }
38
firmware_version_show(struct device * dev,struct device_attribute * attr,char * buf)39 static ssize_t firmware_version_show(struct device *dev,
40 struct device_attribute *attr, char *buf)
41 {
42 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
43 struct cxl_dev_state *cxlds = cxlmd->cxlds;
44 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
45
46 if (!mds)
47 return sysfs_emit(buf, "\n");
48 return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
49 }
50 static DEVICE_ATTR_RO(firmware_version);
51
payload_max_show(struct device * dev,struct device_attribute * attr,char * buf)52 static ssize_t payload_max_show(struct device *dev,
53 struct device_attribute *attr, char *buf)
54 {
55 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
56 struct cxl_dev_state *cxlds = cxlmd->cxlds;
57 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
58
59 if (!mds)
60 return sysfs_emit(buf, "\n");
61 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
62 }
63 static DEVICE_ATTR_RO(payload_max);
64
label_storage_size_show(struct device * dev,struct device_attribute * attr,char * buf)65 static ssize_t label_storage_size_show(struct device *dev,
66 struct device_attribute *attr, char *buf)
67 {
68 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
69 struct cxl_dev_state *cxlds = cxlmd->cxlds;
70 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
71
72 if (!mds)
73 return sysfs_emit(buf, "\n");
74 return sysfs_emit(buf, "%zu\n", mds->lsa_size);
75 }
76 static DEVICE_ATTR_RO(label_storage_size);
77
cxl_ram_size(struct cxl_dev_state * cxlds)78 static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds)
79 {
80 /* Static RAM is only expected at partition 0. */
81 if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
82 return 0;
83 return resource_size(&cxlds->part[0].res);
84 }
85
ram_size_show(struct device * dev,struct device_attribute * attr,char * buf)86 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
87 char *buf)
88 {
89 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
90 struct cxl_dev_state *cxlds = cxlmd->cxlds;
91 unsigned long long len = cxl_ram_size(cxlds);
92
93 return sysfs_emit(buf, "%#llx\n", len);
94 }
95
96 static struct device_attribute dev_attr_ram_size =
97 __ATTR(size, 0444, ram_size_show, NULL);
98
pmem_size_show(struct device * dev,struct device_attribute * attr,char * buf)99 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
100 char *buf)
101 {
102 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
103 struct cxl_dev_state *cxlds = cxlmd->cxlds;
104 unsigned long long len = cxl_pmem_size(cxlds);
105
106 return sysfs_emit(buf, "%#llx\n", len);
107 }
108
109 static struct device_attribute dev_attr_pmem_size =
110 __ATTR(size, 0444, pmem_size_show, NULL);
111
serial_show(struct device * dev,struct device_attribute * attr,char * buf)112 static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
113 char *buf)
114 {
115 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
116 struct cxl_dev_state *cxlds = cxlmd->cxlds;
117
118 return sysfs_emit(buf, "%#llx\n", cxlds->serial);
119 }
120 static DEVICE_ATTR_RO(serial);
121
numa_node_show(struct device * dev,struct device_attribute * attr,char * buf)122 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
123 char *buf)
124 {
125 return sysfs_emit(buf, "%d\n", dev_to_node(dev));
126 }
127 static DEVICE_ATTR_RO(numa_node);
128
security_state_show(struct device * dev,struct device_attribute * attr,char * buf)129 static ssize_t security_state_show(struct device *dev,
130 struct device_attribute *attr,
131 char *buf)
132 {
133 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
134 struct cxl_dev_state *cxlds = cxlmd->cxlds;
135 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
136 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
137 unsigned long state = mds->security.state;
138 int rc = 0;
139
140 /* sync with latest submission state */
141 mutex_lock(&cxl_mbox->mbox_mutex);
142 if (mds->security.sanitize_active)
143 rc = sysfs_emit(buf, "sanitize\n");
144 mutex_unlock(&cxl_mbox->mbox_mutex);
145 if (rc)
146 return rc;
147
148 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
149 return sysfs_emit(buf, "disabled\n");
150 if (state & CXL_PMEM_SEC_STATE_FROZEN ||
151 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
152 state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
153 return sysfs_emit(buf, "frozen\n");
154 if (state & CXL_PMEM_SEC_STATE_LOCKED)
155 return sysfs_emit(buf, "locked\n");
156 else
157 return sysfs_emit(buf, "unlocked\n");
158 }
159 static struct device_attribute dev_attr_security_state =
160 __ATTR(state, 0444, security_state_show, NULL);
161
security_sanitize_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)162 static ssize_t security_sanitize_store(struct device *dev,
163 struct device_attribute *attr,
164 const char *buf, size_t len)
165 {
166 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
167 bool sanitize;
168 ssize_t rc;
169
170 if (kstrtobool(buf, &sanitize) || !sanitize)
171 return -EINVAL;
172
173 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
174 if (rc)
175 return rc;
176
177 return len;
178 }
179 static struct device_attribute dev_attr_security_sanitize =
180 __ATTR(sanitize, 0200, NULL, security_sanitize_store);
181
security_erase_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)182 static ssize_t security_erase_store(struct device *dev,
183 struct device_attribute *attr,
184 const char *buf, size_t len)
185 {
186 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
187 ssize_t rc;
188 bool erase;
189
190 if (kstrtobool(buf, &erase) || !erase)
191 return -EINVAL;
192
193 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
194 if (rc)
195 return rc;
196
197 return len;
198 }
199 static struct device_attribute dev_attr_security_erase =
200 __ATTR(erase, 0200, NULL, security_erase_store);
201
cxl_get_poison_by_memdev(struct cxl_memdev * cxlmd)202 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
203 {
204 struct cxl_dev_state *cxlds = cxlmd->cxlds;
205 u64 offset, length;
206 int rc = 0;
207
208 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
209 for (int i = 0; i < cxlds->nr_partitions; i++) {
210 const struct resource *res = &cxlds->part[i].res;
211
212 offset = res->start;
213 length = resource_size(res);
214 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
215 /*
216 * Invalid Physical Address is not an error for
217 * volatile addresses. Device support is optional.
218 */
219 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
220 rc = 0;
221 }
222 return rc;
223 }
224
cxl_trigger_poison_list(struct cxl_memdev * cxlmd)225 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
226 {
227 struct cxl_port *port;
228 int rc;
229
230 port = cxlmd->endpoint;
231 if (!port || !is_cxl_endpoint(port))
232 return -EINVAL;
233
234 rc = down_read_interruptible(&cxl_region_rwsem);
235 if (rc)
236 return rc;
237
238 rc = down_read_interruptible(&cxl_dpa_rwsem);
239 if (rc) {
240 up_read(&cxl_region_rwsem);
241 return rc;
242 }
243
244 if (cxl_num_decoders_committed(port) == 0) {
245 /* No regions mapped to this memdev */
246 rc = cxl_get_poison_by_memdev(cxlmd);
247 } else {
248 /* Regions mapped, collect poison by endpoint */
249 rc = cxl_get_poison_by_endpoint(port);
250 }
251 up_read(&cxl_dpa_rwsem);
252 up_read(&cxl_region_rwsem);
253
254 return rc;
255 }
256 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL");
257
cxl_validate_poison_dpa(struct cxl_memdev * cxlmd,u64 dpa)258 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
259 {
260 struct cxl_dev_state *cxlds = cxlmd->cxlds;
261
262 if (!IS_ENABLED(CONFIG_DEBUG_FS))
263 return 0;
264
265 if (!resource_size(&cxlds->dpa_res)) {
266 dev_dbg(cxlds->dev, "device has no dpa resource\n");
267 return -EINVAL;
268 }
269 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) {
270 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
271 dpa, &cxlds->dpa_res);
272 return -EINVAL;
273 }
274 if (!IS_ALIGNED(dpa, 64)) {
275 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
276 return -EINVAL;
277 }
278
279 return 0;
280 }
281
cxl_inject_poison(struct cxl_memdev * cxlmd,u64 dpa)282 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
283 {
284 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
285 struct cxl_mbox_inject_poison inject;
286 struct cxl_poison_record record;
287 struct cxl_mbox_cmd mbox_cmd;
288 struct cxl_region *cxlr;
289 int rc;
290
291 if (!IS_ENABLED(CONFIG_DEBUG_FS))
292 return 0;
293
294 rc = down_read_interruptible(&cxl_region_rwsem);
295 if (rc)
296 return rc;
297
298 rc = down_read_interruptible(&cxl_dpa_rwsem);
299 if (rc) {
300 up_read(&cxl_region_rwsem);
301 return rc;
302 }
303
304 rc = cxl_validate_poison_dpa(cxlmd, dpa);
305 if (rc)
306 goto out;
307
308 inject.address = cpu_to_le64(dpa);
309 mbox_cmd = (struct cxl_mbox_cmd) {
310 .opcode = CXL_MBOX_OP_INJECT_POISON,
311 .size_in = sizeof(inject),
312 .payload_in = &inject,
313 };
314 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
315 if (rc)
316 goto out;
317
318 cxlr = cxl_dpa_to_region(cxlmd, dpa);
319 if (cxlr)
320 dev_warn_once(cxl_mbox->host,
321 "poison inject dpa:%#llx region: %s\n", dpa,
322 dev_name(&cxlr->dev));
323
324 record = (struct cxl_poison_record) {
325 .address = cpu_to_le64(dpa),
326 .length = cpu_to_le32(1),
327 };
328 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
329 out:
330 up_read(&cxl_dpa_rwsem);
331 up_read(&cxl_region_rwsem);
332
333 return rc;
334 }
335 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");
336
cxl_clear_poison(struct cxl_memdev * cxlmd,u64 dpa)337 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
338 {
339 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
340 struct cxl_mbox_clear_poison clear;
341 struct cxl_poison_record record;
342 struct cxl_mbox_cmd mbox_cmd;
343 struct cxl_region *cxlr;
344 int rc;
345
346 if (!IS_ENABLED(CONFIG_DEBUG_FS))
347 return 0;
348
349 rc = down_read_interruptible(&cxl_region_rwsem);
350 if (rc)
351 return rc;
352
353 rc = down_read_interruptible(&cxl_dpa_rwsem);
354 if (rc) {
355 up_read(&cxl_region_rwsem);
356 return rc;
357 }
358
359 rc = cxl_validate_poison_dpa(cxlmd, dpa);
360 if (rc)
361 goto out;
362
363 /*
364 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
365 * is defined to accept 64 bytes of write-data, along with the
366 * address to clear. This driver uses zeroes as write-data.
367 */
368 clear = (struct cxl_mbox_clear_poison) {
369 .address = cpu_to_le64(dpa)
370 };
371
372 mbox_cmd = (struct cxl_mbox_cmd) {
373 .opcode = CXL_MBOX_OP_CLEAR_POISON,
374 .size_in = sizeof(clear),
375 .payload_in = &clear,
376 };
377
378 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
379 if (rc)
380 goto out;
381
382 cxlr = cxl_dpa_to_region(cxlmd, dpa);
383 if (cxlr)
384 dev_warn_once(cxl_mbox->host,
385 "poison clear dpa:%#llx region: %s\n", dpa,
386 dev_name(&cxlr->dev));
387
388 record = (struct cxl_poison_record) {
389 .address = cpu_to_le64(dpa),
390 .length = cpu_to_le32(1),
391 };
392 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
393 out:
394 up_read(&cxl_dpa_rwsem);
395 up_read(&cxl_region_rwsem);
396
397 return rc;
398 }
399 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");
400
401 static struct attribute *cxl_memdev_attributes[] = {
402 &dev_attr_serial.attr,
403 &dev_attr_firmware_version.attr,
404 &dev_attr_payload_max.attr,
405 &dev_attr_label_storage_size.attr,
406 &dev_attr_numa_node.attr,
407 NULL,
408 };
409
to_pmem_perf(struct cxl_dev_state * cxlds)410 static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds)
411 {
412 for (int i = 0; i < cxlds->nr_partitions; i++)
413 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
414 return &cxlds->part[i].perf;
415 return NULL;
416 }
417
pmem_qos_class_show(struct device * dev,struct device_attribute * attr,char * buf)418 static ssize_t pmem_qos_class_show(struct device *dev,
419 struct device_attribute *attr, char *buf)
420 {
421 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
422 struct cxl_dev_state *cxlds = cxlmd->cxlds;
423
424 return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class);
425 }
426
427 static struct device_attribute dev_attr_pmem_qos_class =
428 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
429
430 static struct attribute *cxl_memdev_pmem_attributes[] = {
431 &dev_attr_pmem_size.attr,
432 &dev_attr_pmem_qos_class.attr,
433 NULL,
434 };
435
to_ram_perf(struct cxl_dev_state * cxlds)436 static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds)
437 {
438 if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
439 return NULL;
440 return &cxlds->part[0].perf;
441 }
442
ram_qos_class_show(struct device * dev,struct device_attribute * attr,char * buf)443 static ssize_t ram_qos_class_show(struct device *dev,
444 struct device_attribute *attr, char *buf)
445 {
446 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
447 struct cxl_dev_state *cxlds = cxlmd->cxlds;
448
449 return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class);
450 }
451
452 static struct device_attribute dev_attr_ram_qos_class =
453 __ATTR(qos_class, 0444, ram_qos_class_show, NULL);
454
455 static struct attribute *cxl_memdev_ram_attributes[] = {
456 &dev_attr_ram_size.attr,
457 &dev_attr_ram_qos_class.attr,
458 NULL,
459 };
460
461 static struct attribute *cxl_memdev_security_attributes[] = {
462 &dev_attr_security_state.attr,
463 &dev_attr_security_sanitize.attr,
464 &dev_attr_security_erase.attr,
465 NULL,
466 };
467
cxl_memdev_visible(struct kobject * kobj,struct attribute * a,int n)468 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
469 int n)
470 {
471 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
472 return 0;
473 return a->mode;
474 }
475
476 static struct attribute_group cxl_memdev_attribute_group = {
477 .attrs = cxl_memdev_attributes,
478 .is_visible = cxl_memdev_visible,
479 };
480
cxl_ram_visible(struct kobject * kobj,struct attribute * a,int n)481 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
482 {
483 struct device *dev = kobj_to_dev(kobj);
484 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
485 struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds);
486
487 if (a == &dev_attr_ram_qos_class.attr &&
488 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
489 return 0;
490
491 return a->mode;
492 }
493
494 static struct attribute_group cxl_memdev_ram_attribute_group = {
495 .name = "ram",
496 .attrs = cxl_memdev_ram_attributes,
497 .is_visible = cxl_ram_visible,
498 };
499
cxl_pmem_visible(struct kobject * kobj,struct attribute * a,int n)500 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
501 {
502 struct device *dev = kobj_to_dev(kobj);
503 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
504 struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds);
505
506 if (a == &dev_attr_pmem_qos_class.attr &&
507 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
508 return 0;
509
510 return a->mode;
511 }
512
513 static struct attribute_group cxl_memdev_pmem_attribute_group = {
514 .name = "pmem",
515 .attrs = cxl_memdev_pmem_attributes,
516 .is_visible = cxl_pmem_visible,
517 };
518
cxl_memdev_security_visible(struct kobject * kobj,struct attribute * a,int n)519 static umode_t cxl_memdev_security_visible(struct kobject *kobj,
520 struct attribute *a, int n)
521 {
522 struct device *dev = kobj_to_dev(kobj);
523 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
524 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
525
526 if (a == &dev_attr_security_sanitize.attr &&
527 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
528 return 0;
529
530 if (a == &dev_attr_security_erase.attr &&
531 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
532 return 0;
533
534 return a->mode;
535 }
536
537 static struct attribute_group cxl_memdev_security_attribute_group = {
538 .name = "security",
539 .attrs = cxl_memdev_security_attributes,
540 .is_visible = cxl_memdev_security_visible,
541 };
542
543 static const struct attribute_group *cxl_memdev_attribute_groups[] = {
544 &cxl_memdev_attribute_group,
545 &cxl_memdev_ram_attribute_group,
546 &cxl_memdev_pmem_attribute_group,
547 &cxl_memdev_security_attribute_group,
548 NULL,
549 };
550
cxl_memdev_update_perf(struct cxl_memdev * cxlmd)551 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
552 {
553 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);
554 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);
555 }
556 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL");
557
558 static const struct device_type cxl_memdev_type = {
559 .name = "cxl_memdev",
560 .release = cxl_memdev_release,
561 .devnode = cxl_memdev_devnode,
562 .groups = cxl_memdev_attribute_groups,
563 };
564
is_cxl_memdev(const struct device * dev)565 bool is_cxl_memdev(const struct device *dev)
566 {
567 return dev->type == &cxl_memdev_type;
568 }
569 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
570
571 /**
572 * set_exclusive_cxl_commands() - atomically disable user cxl commands
573 * @mds: The device state to operate on
574 * @cmds: bitmap of commands to mark exclusive
575 *
576 * Grab the cxl_memdev_rwsem in write mode to flush in-flight
577 * invocations of the ioctl path and then disable future execution of
578 * commands with the command ids set in @cmds.
579 */
set_exclusive_cxl_commands(struct cxl_memdev_state * mds,unsigned long * cmds)580 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
581 unsigned long *cmds)
582 {
583 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
584
585 guard(rwsem_write)(&cxl_memdev_rwsem);
586 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
587 cmds, CXL_MEM_COMMAND_ID_MAX);
588 }
589 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL");
590
591 /**
592 * clear_exclusive_cxl_commands() - atomically enable user cxl commands
593 * @mds: The device state to modify
594 * @cmds: bitmap of commands to mark available for userspace
595 */
clear_exclusive_cxl_commands(struct cxl_memdev_state * mds,unsigned long * cmds)596 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
597 unsigned long *cmds)
598 {
599 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
600
601 guard(rwsem_write)(&cxl_memdev_rwsem);
602 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
603 cmds, CXL_MEM_COMMAND_ID_MAX);
604 }
605 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL");
606
cxl_memdev_shutdown(struct device * dev)607 static void cxl_memdev_shutdown(struct device *dev)
608 {
609 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
610
611 guard(rwsem_write)(&cxl_memdev_rwsem);
612 cxlmd->cxlds = NULL;
613 }
614
cxl_memdev_unregister(void * _cxlmd)615 static void cxl_memdev_unregister(void *_cxlmd)
616 {
617 struct cxl_memdev *cxlmd = _cxlmd;
618 struct device *dev = &cxlmd->dev;
619
620 cdev_device_del(&cxlmd->cdev, dev);
621 cxl_memdev_shutdown(dev);
622 put_device(dev);
623 }
624
detach_memdev(struct work_struct * work)625 static void detach_memdev(struct work_struct *work)
626 {
627 struct cxl_memdev *cxlmd;
628
629 cxlmd = container_of(work, typeof(*cxlmd), detach_work);
630 device_release_driver(&cxlmd->dev);
631 put_device(&cxlmd->dev);
632 }
633
634 static struct lock_class_key cxl_memdev_key;
635
cxl_memdev_alloc(struct cxl_dev_state * cxlds,const struct file_operations * fops)636 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
637 const struct file_operations *fops)
638 {
639 struct cxl_memdev *cxlmd;
640 struct device *dev;
641 struct cdev *cdev;
642 int rc;
643
644 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL);
645 if (!cxlmd)
646 return ERR_PTR(-ENOMEM);
647
648 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
649 if (rc < 0)
650 goto err;
651 cxlmd->id = rc;
652 cxlmd->depth = -1;
653
654 dev = &cxlmd->dev;
655 device_initialize(dev);
656 lockdep_set_class(&dev->mutex, &cxl_memdev_key);
657 dev->parent = cxlds->dev;
658 dev->bus = &cxl_bus_type;
659 dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
660 dev->type = &cxl_memdev_type;
661 device_set_pm_not_required(dev);
662 INIT_WORK(&cxlmd->detach_work, detach_memdev);
663
664 cdev = &cxlmd->cdev;
665 cdev_init(cdev, fops);
666 return cxlmd;
667
668 err:
669 kfree(cxlmd);
670 return ERR_PTR(rc);
671 }
672
__cxl_memdev_ioctl(struct cxl_memdev * cxlmd,unsigned int cmd,unsigned long arg)673 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
674 unsigned long arg)
675 {
676 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
677 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
678
679 switch (cmd) {
680 case CXL_MEM_QUERY_COMMANDS:
681 return cxl_query_cmd(cxl_mbox, (void __user *)arg);
682 case CXL_MEM_SEND_COMMAND:
683 return cxl_send_cmd(cxl_mbox, (void __user *)arg);
684 default:
685 return -ENOTTY;
686 }
687 }
688
cxl_memdev_ioctl(struct file * file,unsigned int cmd,unsigned long arg)689 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
690 unsigned long arg)
691 {
692 struct cxl_memdev *cxlmd = file->private_data;
693 struct cxl_dev_state *cxlds;
694
695 guard(rwsem_read)(&cxl_memdev_rwsem);
696 cxlds = cxlmd->cxlds;
697 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
698 return __cxl_memdev_ioctl(cxlmd, cmd, arg);
699
700 return -ENXIO;
701 }
702
cxl_memdev_open(struct inode * inode,struct file * file)703 static int cxl_memdev_open(struct inode *inode, struct file *file)
704 {
705 struct cxl_memdev *cxlmd =
706 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
707
708 get_device(&cxlmd->dev);
709 file->private_data = cxlmd;
710
711 return 0;
712 }
713
cxl_memdev_release_file(struct inode * inode,struct file * file)714 static int cxl_memdev_release_file(struct inode *inode, struct file *file)
715 {
716 struct cxl_memdev *cxlmd =
717 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
718
719 put_device(&cxlmd->dev);
720
721 return 0;
722 }
723
724 /**
725 * cxl_mem_get_fw_info - Get Firmware info
726 * @mds: The device data for the operation
727 *
728 * Retrieve firmware info for the device specified.
729 *
730 * Return: 0 if no error: or the result of the mailbox command.
731 *
732 * See CXL-3.0 8.2.9.3.1 Get FW Info
733 */
cxl_mem_get_fw_info(struct cxl_memdev_state * mds)734 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
735 {
736 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
737 struct cxl_mbox_get_fw_info info;
738 struct cxl_mbox_cmd mbox_cmd;
739 int rc;
740
741 mbox_cmd = (struct cxl_mbox_cmd) {
742 .opcode = CXL_MBOX_OP_GET_FW_INFO,
743 .size_out = sizeof(info),
744 .payload_out = &info,
745 };
746
747 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
748 if (rc < 0)
749 return rc;
750
751 mds->fw.num_slots = info.num_slots;
752 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
753 info.slot_info);
754
755 return 0;
756 }
757
758 /**
759 * cxl_mem_activate_fw - Activate Firmware
760 * @mds: The device data for the operation
761 * @slot: slot number to activate
762 *
763 * Activate firmware in a given slot for the device specified.
764 *
765 * Return: 0 if no error: or the result of the mailbox command.
766 *
767 * See CXL-3.0 8.2.9.3.3 Activate FW
768 */
cxl_mem_activate_fw(struct cxl_memdev_state * mds,int slot)769 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
770 {
771 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
772 struct cxl_mbox_activate_fw activate;
773 struct cxl_mbox_cmd mbox_cmd;
774
775 if (slot == 0 || slot > mds->fw.num_slots)
776 return -EINVAL;
777
778 mbox_cmd = (struct cxl_mbox_cmd) {
779 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
780 .size_in = sizeof(activate),
781 .payload_in = &activate,
782 };
783
784 /* Only offline activation supported for now */
785 activate.action = CXL_FW_ACTIVATE_OFFLINE;
786 activate.slot = slot;
787
788 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
789 }
790
791 /**
792 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
793 * @mds: The device data for the operation
794 *
795 * Abort an in-progress firmware transfer for the device specified.
796 *
797 * Return: 0 if no error: or the result of the mailbox command.
798 *
799 * See CXL-3.0 8.2.9.3.2 Transfer FW
800 */
cxl_mem_abort_fw_xfer(struct cxl_memdev_state * mds)801 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
802 {
803 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
804 struct cxl_mbox_transfer_fw *transfer;
805 struct cxl_mbox_cmd mbox_cmd;
806 int rc;
807
808 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL);
809 if (!transfer)
810 return -ENOMEM;
811
812 /* Set a 1s poll interval and a total wait time of 30s */
813 mbox_cmd = (struct cxl_mbox_cmd) {
814 .opcode = CXL_MBOX_OP_TRANSFER_FW,
815 .size_in = sizeof(*transfer),
816 .payload_in = transfer,
817 .poll_interval_ms = 1000,
818 .poll_count = 30,
819 };
820
821 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
822
823 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
824 kfree(transfer);
825 return rc;
826 }
827
cxl_fw_cleanup(struct fw_upload * fwl)828 static void cxl_fw_cleanup(struct fw_upload *fwl)
829 {
830 struct cxl_memdev_state *mds = fwl->dd_handle;
831
832 mds->fw.next_slot = 0;
833 }
834
cxl_fw_do_cancel(struct fw_upload * fwl)835 static int cxl_fw_do_cancel(struct fw_upload *fwl)
836 {
837 struct cxl_memdev_state *mds = fwl->dd_handle;
838 struct cxl_dev_state *cxlds = &mds->cxlds;
839 struct cxl_memdev *cxlmd = cxlds->cxlmd;
840 int rc;
841
842 rc = cxl_mem_abort_fw_xfer(mds);
843 if (rc < 0)
844 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
845
846 return FW_UPLOAD_ERR_CANCELED;
847 }
848
cxl_fw_prepare(struct fw_upload * fwl,const u8 * data,u32 size)849 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
850 u32 size)
851 {
852 struct cxl_memdev_state *mds = fwl->dd_handle;
853 struct cxl_mbox_transfer_fw *transfer;
854 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
855
856 if (!size)
857 return FW_UPLOAD_ERR_INVALID_SIZE;
858
859 mds->fw.oneshot = struct_size(transfer, data, size) <
860 cxl_mbox->payload_size;
861
862 if (cxl_mem_get_fw_info(mds))
863 return FW_UPLOAD_ERR_HW_ERROR;
864
865 /*
866 * So far no state has been changed, hence no other cleanup is
867 * necessary. Simply return the cancelled status.
868 */
869 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
870 return FW_UPLOAD_ERR_CANCELED;
871
872 return FW_UPLOAD_ERR_NONE;
873 }
874
cxl_fw_write(struct fw_upload * fwl,const u8 * data,u32 offset,u32 size,u32 * written)875 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
876 u32 offset, u32 size, u32 *written)
877 {
878 struct cxl_memdev_state *mds = fwl->dd_handle;
879 struct cxl_dev_state *cxlds = &mds->cxlds;
880 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
881 struct cxl_memdev *cxlmd = cxlds->cxlmd;
882 struct cxl_mbox_transfer_fw *transfer;
883 struct cxl_mbox_cmd mbox_cmd;
884 u32 cur_size, remaining;
885 size_t size_in;
886 int rc;
887
888 *written = 0;
889
890 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
891 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
892 dev_err(&cxlmd->dev,
893 "misaligned offset for FW transfer slice (%u)\n",
894 offset);
895 return FW_UPLOAD_ERR_RW_ERROR;
896 }
897
898 /*
899 * Pick transfer size based on mds->payload_size @size must bw 128-byte
900 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
901 * sizeof(*transfer) is 128. These constraints imply that @cur_size
902 * will always be 128b aligned.
903 */
904 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
905
906 remaining = size - cur_size;
907 size_in = struct_size(transfer, data, cur_size);
908
909 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
910 return cxl_fw_do_cancel(fwl);
911
912 /*
913 * Slot numbers are 1-indexed
914 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
915 * Check for rollover using modulo, and 1-index it by adding 1
916 */
917 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
918
919 /* Do the transfer via mailbox cmd */
920 transfer = kzalloc(size_in, GFP_KERNEL);
921 if (!transfer)
922 return FW_UPLOAD_ERR_RW_ERROR;
923
924 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
925 memcpy(transfer->data, data + offset, cur_size);
926 if (mds->fw.oneshot) {
927 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
928 transfer->slot = mds->fw.next_slot;
929 } else {
930 if (offset == 0) {
931 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
932 } else if (remaining == 0) {
933 transfer->action = CXL_FW_TRANSFER_ACTION_END;
934 transfer->slot = mds->fw.next_slot;
935 } else {
936 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
937 }
938 }
939
940 mbox_cmd = (struct cxl_mbox_cmd) {
941 .opcode = CXL_MBOX_OP_TRANSFER_FW,
942 .size_in = size_in,
943 .payload_in = transfer,
944 .poll_interval_ms = 1000,
945 .poll_count = 30,
946 };
947
948 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
949 if (rc < 0) {
950 rc = FW_UPLOAD_ERR_RW_ERROR;
951 goto out_free;
952 }
953
954 *written = cur_size;
955
956 /* Activate FW if oneshot or if the last slice was written */
957 if (mds->fw.oneshot || remaining == 0) {
958 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
959 mds->fw.next_slot);
960 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
961 if (rc < 0) {
962 dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
963 rc);
964 rc = FW_UPLOAD_ERR_HW_ERROR;
965 goto out_free;
966 }
967 }
968
969 rc = FW_UPLOAD_ERR_NONE;
970
971 out_free:
972 kfree(transfer);
973 return rc;
974 }
975
cxl_fw_poll_complete(struct fw_upload * fwl)976 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
977 {
978 struct cxl_memdev_state *mds = fwl->dd_handle;
979
980 /*
981 * cxl_internal_send_cmd() handles background operations synchronously.
982 * No need to wait for completions here - any errors would've been
983 * reported and handled during the ->write() call(s).
984 * Just check if a cancel request was received, and return success.
985 */
986 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
987 return cxl_fw_do_cancel(fwl);
988
989 return FW_UPLOAD_ERR_NONE;
990 }
991
cxl_fw_cancel(struct fw_upload * fwl)992 static void cxl_fw_cancel(struct fw_upload *fwl)
993 {
994 struct cxl_memdev_state *mds = fwl->dd_handle;
995
996 set_bit(CXL_FW_CANCEL, mds->fw.state);
997 }
998
999 static const struct fw_upload_ops cxl_memdev_fw_ops = {
1000 .prepare = cxl_fw_prepare,
1001 .write = cxl_fw_write,
1002 .poll_complete = cxl_fw_poll_complete,
1003 .cancel = cxl_fw_cancel,
1004 .cleanup = cxl_fw_cleanup,
1005 };
1006
cxl_remove_fw_upload(void * fwl)1007 static void cxl_remove_fw_upload(void *fwl)
1008 {
1009 firmware_upload_unregister(fwl);
1010 }
1011
devm_cxl_setup_fw_upload(struct device * host,struct cxl_memdev_state * mds)1012 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
1013 {
1014 struct cxl_dev_state *cxlds = &mds->cxlds;
1015 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
1016 struct device *dev = &cxlds->cxlmd->dev;
1017 struct fw_upload *fwl;
1018
1019 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds))
1020 return 0;
1021
1022 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
1023 &cxl_memdev_fw_ops, mds);
1024 if (IS_ERR(fwl))
1025 return PTR_ERR(fwl);
1026 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
1027 }
1028 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL");
1029
1030 static const struct file_operations cxl_memdev_fops = {
1031 .owner = THIS_MODULE,
1032 .unlocked_ioctl = cxl_memdev_ioctl,
1033 .open = cxl_memdev_open,
1034 .release = cxl_memdev_release_file,
1035 .compat_ioctl = compat_ptr_ioctl,
1036 .llseek = noop_llseek,
1037 };
1038
devm_cxl_add_memdev(struct device * host,struct cxl_dev_state * cxlds)1039 struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
1040 struct cxl_dev_state *cxlds)
1041 {
1042 struct cxl_memdev *cxlmd;
1043 struct device *dev;
1044 struct cdev *cdev;
1045 int rc;
1046
1047 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops);
1048 if (IS_ERR(cxlmd))
1049 return cxlmd;
1050
1051 dev = &cxlmd->dev;
1052 rc = dev_set_name(dev, "mem%d", cxlmd->id);
1053 if (rc)
1054 goto err;
1055
1056 /*
1057 * Activate ioctl operations, no cxl_memdev_rwsem manipulation
1058 * needed as this is ordered with cdev_add() publishing the device.
1059 */
1060 cxlmd->cxlds = cxlds;
1061 cxlds->cxlmd = cxlmd;
1062
1063 cdev = &cxlmd->cdev;
1064 rc = cdev_device_add(cdev, dev);
1065 if (rc)
1066 goto err;
1067
1068 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd);
1069 if (rc)
1070 return ERR_PTR(rc);
1071 return cxlmd;
1072
1073 err:
1074 /*
1075 * The cdev was briefly live, shutdown any ioctl operations that
1076 * saw that state.
1077 */
1078 cxl_memdev_shutdown(dev);
1079 put_device(dev);
1080 return ERR_PTR(rc);
1081 }
1082 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL");
1083
sanitize_teardown_notifier(void * data)1084 static void sanitize_teardown_notifier(void *data)
1085 {
1086 struct cxl_memdev_state *mds = data;
1087 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1088 struct kernfs_node *state;
1089
1090 /*
1091 * Prevent new irq triggered invocations of the workqueue and
1092 * flush inflight invocations.
1093 */
1094 mutex_lock(&cxl_mbox->mbox_mutex);
1095 state = mds->security.sanitize_node;
1096 mds->security.sanitize_node = NULL;
1097 mutex_unlock(&cxl_mbox->mbox_mutex);
1098
1099 cancel_delayed_work_sync(&mds->security.poll_dwork);
1100 sysfs_put(state);
1101 }
1102
devm_cxl_sanitize_setup_notifier(struct device * host,struct cxl_memdev * cxlmd)1103 int devm_cxl_sanitize_setup_notifier(struct device *host,
1104 struct cxl_memdev *cxlmd)
1105 {
1106 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1107 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1108 struct kernfs_node *sec;
1109
1110 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1111 return 0;
1112
1113 /*
1114 * Note, the expectation is that @cxlmd would have failed to be
1115 * created if these sysfs_get_dirent calls fail.
1116 */
1117 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
1118 if (!sec)
1119 return -ENOENT;
1120 mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1121 sysfs_put(sec);
1122 if (!mds->security.sanitize_node)
1123 return -ENOENT;
1124
1125 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1126 }
1127 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL");
1128
cxl_memdev_init(void)1129 __init int cxl_memdev_init(void)
1130 {
1131 dev_t devt;
1132 int rc;
1133
1134 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1135 if (rc)
1136 return rc;
1137
1138 cxl_mem_major = MAJOR(devt);
1139
1140 return 0;
1141 }
1142
cxl_memdev_exit(void)1143 void cxl_memdev_exit(void)
1144 {
1145 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1146 }
1147