1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (C) 2014-2018 Etnaviv Project
27 	u32 *vaddr = (u32 *)buffer->vaddr;  in OUT()
29 	BUG_ON(buffer->user_size >= buffer->size);  in OUT()
31 	vaddr[buffer->user_size / 4] = data;  in OUT()
32 	buffer->user_size += 4;  in OUT()
40 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_LOAD_STATE()
51 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_END()
59 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_WAIT()
67 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_LINK()
77 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_STALL()
95 	lockdep_assert_held(&gpu->lock);  in etnaviv_cmd_select_pipe()
101 	 * otherwise we need to flush the 2D pixel engine cache.  in etnaviv_cmd_select_pipe()
103 	if (gpu->exec_state == ETNA_PIPE_2D)  in etnaviv_cmd_select_pipe()
105 	else if (gpu->exec_state == ETNA_PIPE_3D)  in etnaviv_cmd_select_pipe()
119 	u32 size = buf->size;  in etnaviv_buffer_dump()
120 	u32 *ptr = buf->vaddr + off;  in etnaviv_buffer_dump()
122 	dev_info(gpu->dev, "virt %p phys 0x%08x free 0x%08x\n",  in etnaviv_buffer_dump()
124 			&gpu->mmu_context->cmdbuf_mapping) +  in etnaviv_buffer_dump()
125 			off, size - len * 4 - off);  in etnaviv_buffer_dump()
127 	print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,  in etnaviv_buffer_dump()
128 			ptr, len * 4, 0);  in etnaviv_buffer_dump()
140 	u32 *lw = buffer->vaddr + wl_offset;  in etnaviv_buffer_replace_wait()
150  * 'cmd_dwords' 64-bit words into the buffer, wrapping if necessary.
155 	if (buffer->user_size + cmd_dwords * sizeof(u64) > buffer->size)  in etnaviv_buffer_reserve()
156 		buffer->user_size = 0;  in etnaviv_buffer_reserve()
159 				     &gpu->mmu_context->cmdbuf_mapping) +  in etnaviv_buffer_reserve()
160 	       buffer->user_size;  in etnaviv_buffer_reserve()
165 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_init()
167 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_init()
170 	buffer->user_size = 0;  in etnaviv_buffer_init()
172 	CMD_WAIT(buffer, gpu->fe_waitcycles);  in etnaviv_buffer_init()
174 		 etnaviv_cmdbuf_get_va(buffer, &gpu->mmu_context->cmdbuf_mapping)  in etnaviv_buffer_init()
175 		 + buffer->user_size - 4);  in etnaviv_buffer_init()
177 	return buffer->user_size / 8;  in etnaviv_buffer_init()
182 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_config_mmuv2()
184 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_config_mmuv2()
186 	buffer->user_size = 0;  in etnaviv_buffer_config_mmuv2()
188 	if (gpu->identity.features & chipFeatures_PIPE_3D) {  in etnaviv_buffer_config_mmuv2()
198 	if (gpu->identity.features & chipFeatures_PIPE_2D) {  in etnaviv_buffer_config_mmuv2()
210 	buffer->user_size = ALIGN(buffer->user_size, 8);  in etnaviv_buffer_config_mmuv2()
212 	return buffer->user_size / 8;  in etnaviv_buffer_config_mmuv2()
217 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_config_pta()
219 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_config_pta()
221 	buffer->user_size = 0;  in etnaviv_buffer_config_pta()
228 	buffer->user_size = ALIGN(buffer->user_size, 8);  in etnaviv_buffer_config_pta()
230 	return buffer->user_size / 8;  in etnaviv_buffer_config_pta()
235 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_end()
236 	unsigned int waitlink_offset = buffer->user_size - 16;  in etnaviv_buffer_end()
238 	bool has_blt = !!(gpu->identity.minor_features5 &  in etnaviv_buffer_end()
241 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_end()
243 	if (gpu->exec_state == ETNA_PIPE_2D)  in etnaviv_buffer_end()
245 	else if (gpu->exec_state == ETNA_PIPE_3D)  in etnaviv_buffer_end()
269 		if (gpu->exec_state == ETNA_PIPE_3D) {  in etnaviv_buffer_end()
294 		/* Replace the last link-wait with an "END" command */  in etnaviv_buffer_end()
303 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_sync_point_queue()
304 	unsigned int waitlink_offset = buffer->user_size - 16;  in etnaviv_sync_point_queue()
307 	lockdep_assert_held(&gpu->lock);  in etnaviv_sync_point_queue()
313 	dwords = 4;  in etnaviv_sync_point_queue()
324 	CMD_WAIT(buffer, gpu->fe_waitcycles);  in etnaviv_sync_point_queue()
326 		 etnaviv_cmdbuf_get_va(buffer, &gpu->mmu_context->cmdbuf_mapping)  in etnaviv_sync_point_queue()
327 		 + buffer->user_size - 4);  in etnaviv_sync_point_queue()
344 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_queue()
345 	unsigned int waitlink_offset = buffer->user_size - 16;  in etnaviv_buffer_queue()
348 	bool switch_context = gpu->exec_state != exec_state;  in etnaviv_buffer_queue()
349 	bool switch_mmu_context = gpu->mmu_context != mmu_context;  in etnaviv_buffer_queue()
350 	unsigned int new_flush_seq = READ_ONCE(gpu->mmu_context->flush_seq);  in etnaviv_buffer_queue()
351 	bool need_flush = switch_mmu_context || gpu->flush_seq != new_flush_seq;  in etnaviv_buffer_queue()
352 	bool has_blt = !!(gpu->identity.minor_features5 &  in etnaviv_buffer_queue()
355 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_queue()
361 					    &gpu->mmu_context->cmdbuf_mapping);  in etnaviv_buffer_queue()
362 	link_dwords = cmdbuf->size / 8;  in etnaviv_buffer_queue()
367 	 * link to this buffer - a total of four additional words.  in etnaviv_buffer_queue()
377 			if (gpu->mmu_context->global->version == ETNAVIV_IOMMU_V1)  in etnaviv_buffer_queue()
385 			extra_dwords += 4;  in etnaviv_buffer_queue()
388 		if (switch_mmu_context && gpu->sec_mode == ETNA_SEC_KERNEL)  in etnaviv_buffer_queue()
399 			struct etnaviv_iommu_context *old_context = gpu->mmu_context;  in etnaviv_buffer_queue()
401 			gpu->mmu_context = etnaviv_iommu_context_get(mmu_context);  in etnaviv_buffer_queue()
407 			if (gpu->mmu_context->global->version == ETNAVIV_IOMMU_V1) {  in etnaviv_buffer_queue()
419 				    gpu->sec_mode == ETNA_SEC_KERNEL) {  in etnaviv_buffer_queue()
421 						etnaviv_iommuv2_get_pta_id(gpu->mmu_context);  in etnaviv_buffer_queue()
427 				if (gpu->sec_mode == ETNA_SEC_NONE)  in etnaviv_buffer_queue()
428 					flush |= etnaviv_iommuv2_get_mtlb_addr(gpu->mmu_context);  in etnaviv_buffer_queue()
438 			gpu->flush_seq = new_flush_seq;  in etnaviv_buffer_queue()
443 			gpu->exec_state = exec_state;  in etnaviv_buffer_queue()
448 					&gpu->mmu_context->cmdbuf_mapping);  in etnaviv_buffer_queue()
466 	 * target: 3 enable/flush/disable + 4 enable/semaphore stall/disable,  in etnaviv_buffer_queue()
479 	if (gpu->exec_state == ETNA_PIPE_2D) {  in etnaviv_buffer_queue()
507 	CMD_WAIT(buffer, gpu->fe_waitcycles);  in etnaviv_buffer_queue()
509 		 etnaviv_cmdbuf_get_va(buffer, &gpu->mmu_context->cmdbuf_mapping)  in etnaviv_buffer_queue()
510 		 + buffer->user_size - 4);  in etnaviv_buffer_queue()
515 			etnaviv_cmdbuf_get_va(cmdbuf, &gpu->mmu_context->cmdbuf_mapping),  in etnaviv_buffer_queue()
516 			cmdbuf->vaddr);  in etnaviv_buffer_queue()
519 		print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,  in etnaviv_buffer_queue()
520 			       cmdbuf->vaddr, cmdbuf->size, 0);  in etnaviv_buffer_queue()
522 		pr_info("link op: %p\n", buffer->vaddr + waitlink_offset);  in etnaviv_buffer_queue()