1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
3  * Copyright 2014-2016 Freescale Semiconductor Inc.
4  * Copyright 2016-2019 NXP
9 #include <soc/fsl/dpaa2-io.h>
14 #include <linux/dma-mapping.h>
18 #include "qbman-portal.h"
34 	unsigned int max;  member
38 	unsigned int idx;      /* position of the next-to-be-returned entry */
58 	 * If cpu == -1, choose the current cpu, with no guarantees about  in service_select_by_cpu()
73 	d = service_select_by_cpu(d, -1);  in service_select()
79 	list_del(&d->node);  in service_select()
80 	list_add_tail(&d->node, &dpio_list);  in service_select()
87  * dpaa2_io_service_select() - return a dpaa2_io service affined to this cpu
104  * dpaa2_io_create() - create a dpaa2_io object.
121 	/* check if CPU is out of range (-1 means any cpu) */  in dpaa2_io_create()
122 	if (desc->cpu != DPAA2_IO_ANY_CPU && desc->cpu >= num_possible_cpus()) {  in dpaa2_io_create()
127 	obj->dpio_desc = *desc;  in dpaa2_io_create()
128 	obj->swp_desc.cena_bar = obj->dpio_desc.regs_cena;  in dpaa2_io_create()
129 	obj->swp_desc.cinh_bar = obj->dpio_desc.regs_cinh;  in dpaa2_io_create()
130 	obj->swp_desc.qman_version = obj->dpio_desc.qman_version;  in dpaa2_io_create()
131 	obj->swp = qbman_swp_init(&obj->swp_desc);  in dpaa2_io_create()
133 	if (!obj->swp) {  in dpaa2_io_create()
138 	INIT_LIST_HEAD(&obj->node);  in dpaa2_io_create()
139 	spin_lock_init(&obj->lock_mgmt_cmd);  in dpaa2_io_create()
140 	spin_lock_init(&obj->lock_notifications);  in dpaa2_io_create()
141 	INIT_LIST_HEAD(&obj->notifications);  in dpaa2_io_create()
144 	qbman_swp_interrupt_set_trigger(obj->swp,  in dpaa2_io_create()
146 	qbman_swp_interrupt_clear_status(obj->swp, 0xffffffff);  in dpaa2_io_create()
147 	if (obj->dpio_desc.receives_notifications)  in dpaa2_io_create()
148 		qbman_swp_push_set(obj->swp, 0, 1);  in dpaa2_io_create()
151 	list_add_tail(&obj->node, &dpio_list);  in dpaa2_io_create()
152 	if (desc->cpu >= 0 && !dpio_by_cpu[desc->cpu])  in dpaa2_io_create()
153 		dpio_by_cpu[desc->cpu] = obj;  in dpaa2_io_create()
156 	obj->dev = dev;  in dpaa2_io_create()
162  * dpaa2_io_down() - release the dpaa2_io object.
173 	dpio_by_cpu[d->dpio_desc.cpu] = NULL;  in dpaa2_io_down()
174 	list_del(&d->node);  in dpaa2_io_down()
183  * dpaa2_io_irq() - ISR for DPIO interrupts
193 	int max = 0;  in dpaa2_io_irq()  local
197 	swp = obj->swp;  in dpaa2_io_irq()
210 			ctx->cb(ctx);  in dpaa2_io_irq()
212 			pr_crit("fsl-mc-dpio: Unrecognised/ignored DQRR entry\n");  in dpaa2_io_irq()
215 		++max;  in dpaa2_io_irq()
216 		if (max > DPAA_POLL_MAX)  in dpaa2_io_irq()
227  * dpaa2_io_get_cpu() - get the cpu associated with a given DPIO object
235 	return d->dpio_desc.cpu;  in dpaa2_io_get_cpu()
240  * dpaa2_io_service_register() - Prepare for servicing of FQDAN or CDAN
256  * Return 0 for success, or -ENODEV for failure.
265 	d = service_select_by_cpu(d, ctx->desired_cpu);  in dpaa2_io_service_register()
267 		return -ENODEV;  in dpaa2_io_service_register()
269 	link = device_link_add(dev, d->dev, DL_FLAG_AUTOREMOVE_CONSUMER);  in dpaa2_io_service_register()
271 		return -EINVAL;  in dpaa2_io_service_register()
273 	ctx->dpio_id = d->dpio_desc.dpio_id;  in dpaa2_io_service_register()
274 	ctx->qman64 = (u64)(uintptr_t)ctx;  in dpaa2_io_service_register()
275 	ctx->dpio_private = d;  in dpaa2_io_service_register()
276 	spin_lock_irqsave(&d->lock_notifications, irqflags);  in dpaa2_io_service_register()
277 	list_add(&ctx->node, &d->notifications);  in dpaa2_io_service_register()
278 	spin_unlock_irqrestore(&d->lock_notifications, irqflags);  in dpaa2_io_service_register()
281 	if (ctx->is_cdan)  in dpaa2_io_service_register()
282 		return qbman_swp_CDAN_set_context_enable(d->swp,  in dpaa2_io_service_register()
283 							 (u16)ctx->id,  in dpaa2_io_service_register()
284 							 ctx->qman64);  in dpaa2_io_service_register()
290  * dpaa2_io_service_deregister - The opposite of 'register'.
296  * to detach the notification-producing device from the DPIO.
302 	struct dpaa2_io *d = ctx->dpio_private;  in dpaa2_io_service_deregister()
305 	if (ctx->is_cdan)  in dpaa2_io_service_deregister()
306 		qbman_swp_CDAN_disable(d->swp, (u16)ctx->id);  in dpaa2_io_service_deregister()
308 	spin_lock_irqsave(&d->lock_notifications, irqflags);  in dpaa2_io_service_deregister()
309 	list_del(&ctx->node);  in dpaa2_io_service_deregister()
310 	spin_unlock_irqrestore(&d->lock_notifications, irqflags);  in dpaa2_io_service_deregister()
316  * dpaa2_io_service_rearm() - Rearm the notification for the given DPIO service.
334 	d = service_select_by_cpu(d, ctx->desired_cpu);  in dpaa2_io_service_rearm()
336 		return -ENODEV;  in dpaa2_io_service_rearm()
338 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_rearm()
339 	if (ctx->is_cdan)  in dpaa2_io_service_rearm()
340 		err = qbman_swp_CDAN_enable(d->swp, (u16)ctx->id);  in dpaa2_io_service_rearm()
342 		err = qbman_swp_fq_schedule(d->swp, ctx->id);  in dpaa2_io_service_rearm()
343 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_rearm()
350  * dpaa2_io_service_pull_fq() - pull dequeue functions from a fq.
352  * @fqid: the given frame queue id.
364 	qbman_pull_desc_set_storage(&pd, s->vaddr, s->paddr, 1);  in dpaa2_io_service_pull_fq()
365 	qbman_pull_desc_set_numframes(&pd, (u8)s->max);  in dpaa2_io_service_pull_fq()
370 		return -ENODEV;  in dpaa2_io_service_pull_fq()
371 	s->swp = d->swp;  in dpaa2_io_service_pull_fq()
372 	err = qbman_swp_pull(d->swp, &pd);  in dpaa2_io_service_pull_fq()
374 		s->swp = NULL;  in dpaa2_io_service_pull_fq()
381  * dpaa2_io_service_pull_channel() - pull dequeue functions from a channel.
395 	qbman_pull_desc_set_storage(&pd, s->vaddr, s->paddr, 1);  in dpaa2_io_service_pull_channel()
396 	qbman_pull_desc_set_numframes(&pd, (u8)s->max);  in dpaa2_io_service_pull_channel()
401 		return -ENODEV;  in dpaa2_io_service_pull_channel()
403 	s->swp = d->swp;  in dpaa2_io_service_pull_channel()
404 	err = qbman_swp_pull(d->swp, &pd);  in dpaa2_io_service_pull_channel()
406 		s->swp = NULL;  in dpaa2_io_service_pull_channel()
413  * dpaa2_io_service_enqueue_fq() - Enqueue a frame to a frame queue.
415  * @fqid: the given frame queue id.
416  * @fd: the frame descriptor which is enqueued.
418  * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,
419  * or -ENODEV if there is no dpio service.
429 		return -ENODEV;  in dpaa2_io_service_enqueue_fq()
435 	return qbman_swp_enqueue(d->swp, &ed, fd);  in dpaa2_io_service_enqueue_fq()
440  * dpaa2_io_service_enqueue_multiple_fq() - Enqueue multiple frames
441  * to a frame queue using one fqid.
443  * @fqid: the given frame queue id.
444  * @fd: the frame descriptor which is enqueued.
447  * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,
448  * or -ENODEV if there is no dpio service.
459 		return -ENODEV;  in dpaa2_io_service_enqueue_multiple_fq()
465 	return qbman_swp_enqueue_multiple(d->swp, &ed, fd, 0, nb);  in dpaa2_io_service_enqueue_multiple_fq()
470  * dpaa2_io_service_enqueue_multiple_desc_fq() - Enqueue multiple frames
471  * to different frame queue using a list of fqids.
473  * @fqid: the given list of frame queue ids.
474  * @fd: the frame descriptor which is enqueued.
477  * Return 0 for successful enqueue, -EBUSY if the enqueue ring is not ready,
478  * or -ENODEV if there is no dpio service.
490 		return -ENOMEM;  in dpaa2_io_service_enqueue_multiple_desc_fq()
494 		ret = -ENODEV;  in dpaa2_io_service_enqueue_multiple_desc_fq()
504 	ret = qbman_swp_enqueue_multiple_desc(d->swp, &ed[0], fd, nb);  in dpaa2_io_service_enqueue_multiple_desc_fq()
512  * dpaa2_io_service_enqueue_qd() - Enqueue a frame to a QD.
517  * @fd: the frame descriptor which is enqueued.
519  * Return 0 for successful enqueue, or -EBUSY if the enqueue ring is not ready,
520  * or -ENODEV if there is no dpio service.
530 		return -ENODEV;  in dpaa2_io_service_enqueue_qd()
536 	return qbman_swp_enqueue(d->swp, &ed, fd);  in dpaa2_io_service_enqueue_qd()
541  * dpaa2_io_service_release() - Release buffers to a buffer pool.
558 		return -ENODEV;  in dpaa2_io_service_release()
563 	return qbman_swp_release(d->swp, &rd, buffers, num_buffers);  in dpaa2_io_service_release()
568  * dpaa2_io_service_acquire() - Acquire buffers from a buffer pool.
588 		return -ENODEV;  in dpaa2_io_service_acquire()
590 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_acquire()
591 	err = qbman_swp_acquire(d->swp, bpid, buffers, num_buffers);  in dpaa2_io_service_acquire()
592 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_service_acquire()
604  * dpaa2_io_store_create() - Create the dma memory storage for dequeue result.
608  * The size of the storage is "max_frames*sizeof(struct dpaa2_dq)".
618 	size_t size;  in dpaa2_io_store_create()  local
627 	ret->max = max_frames;  in dpaa2_io_store_create()
628 	size = max_frames * sizeof(struct dpaa2_dq) + 64;  in dpaa2_io_store_create()
629 	ret->alloced_addr = kzalloc(size, GFP_KERNEL);  in dpaa2_io_store_create()
630 	if (!ret->alloced_addr) {  in dpaa2_io_store_create()
635 	ret->vaddr = PTR_ALIGN(ret->alloced_addr, 64);  in dpaa2_io_store_create()
636 	ret->paddr = dma_map_single(dev, ret->vaddr,  in dpaa2_io_store_create()
639 	if (dma_mapping_error(dev, ret->paddr)) {  in dpaa2_io_store_create()
640 		kfree(ret->alloced_addr);  in dpaa2_io_store_create()
645 	ret->idx = 0;  in dpaa2_io_store_create()
646 	ret->dev = dev;  in dpaa2_io_store_create()
653  * dpaa2_io_store_destroy() - Frees the dma memory storage for dequeue
659 	dma_unmap_single(s->dev, s->paddr, sizeof(struct dpaa2_dq) * s->max,  in dpaa2_io_store_destroy()
661 	kfree(s->alloced_addr);  in dpaa2_io_store_destroy()
667  * dpaa2_io_store_next() - Determine when the next dequeue result is available.
669  * @is_last: indicate whether this is the last frame in the pull command.
672  * can be used to determine when the next frame result is available. Once
673  * this function returns non-NULL, a subsequent call to it will try to find
676  * Note that if a pull-dequeue has a NULL result because the target FQ/channel
679  * differentiate between "end-of-empty-dequeue" and "still-waiting".
686 	struct dpaa2_dq *ret = &s->vaddr[s->idx];  in dpaa2_io_store_next()
688 	match = qbman_result_has_new_result(s->swp, ret);  in dpaa2_io_store_next()
694 	s->idx++;  in dpaa2_io_store_next()
698 		s->idx = 0;  in dpaa2_io_store_next()
700 		 * If we get an empty dequeue result to terminate a zero-results  in dpaa2_io_store_next()
702 		 * check non-NULL results every time.  in dpaa2_io_store_next()
707 		prefetch(&s->vaddr[s->idx]);  in dpaa2_io_store_next()
716  * dpaa2_io_query_fq_count() - Get the frame and byte count for a given fq.
718  * @fqid: the id of frame queue to be queried.
719  * @fcnt: the queried frame count.
722  * Knowing the FQ count at run-time can be useful in debugging situations.
723  * The instantaneous frame- and byte-count are hereby returned.
737 		return -ENODEV;  in dpaa2_io_query_fq_count()
739 	swp = d->swp;  in dpaa2_io_query_fq_count()
740 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_fq_count()
742 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_fq_count()
753  * dpaa2_io_query_bp_count() - Query the number of buffers currently in a
770 		return -ENODEV;  in dpaa2_io_query_bp_count()
772 	swp = d->swp;  in dpaa2_io_query_bp_count()
773 	spin_lock_irqsave(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_bp_count()
775 	spin_unlock_irqrestore(&d->lock_mgmt_cmd, irqflags);  in dpaa2_io_query_bp_count()