63 			u8  fault;  member
65 		} **fault;  member
155 	 * page fault) and maybe some other commands.  in nouveau_svmm_bind()
369 /* Issue fault replay for GPU to retry accesses that faulted previously. */
380 /* Cancel a replayable fault that could not be handled.
382  * Cancelling the fault will trigger recovery to reset the engine
402 			       struct nouveau_svm_fault *fault)  in nouveau_svm_fault_cancel_fault()  argument
404 	nouveau_svm_fault_cancel(svm, fault->inst,  in nouveau_svm_fault_cancel_fault()
405 				      fault->hub,  in nouveau_svm_fault_cancel_fault()
406 				      fault->gpc,  in nouveau_svm_fault_cancel_fault()
407 				      fault->client);  in nouveau_svm_fault_cancel_fault()
442 	struct nouveau_svm_fault *fault;  in nouveau_svm_fault_cache()  local
450 	if (!buffer->fault[buffer->fault_nr]) {  in nouveau_svm_fault_cache()
451 		fault = kmalloc(sizeof(*fault), GFP_KERNEL);  in nouveau_svm_fault_cache()
452 		if (WARN_ON(!fault)) {  in nouveau_svm_fault_cache()
456 		buffer->fault[buffer->fault_nr] = fault;  in nouveau_svm_fault_cache()
459 	fault = buffer->fault[buffer->fault_nr++];  in nouveau_svm_fault_cache()
460 	fault->inst   = inst;  in nouveau_svm_fault_cache()
461 	fault->addr   = (u64)addrhi << 32 | addrlo;  in nouveau_svm_fault_cache()
462 	fault->time   = (u64)timehi << 32 | timelo;  in nouveau_svm_fault_cache()
463 	fault->engine = engine;  in nouveau_svm_fault_cache()
464 	fault->gpc    = gpc;  in nouveau_svm_fault_cache()
465 	fault->hub    = hub;  in nouveau_svm_fault_cache()
466 	fault->access = (info & 0x000f0000) >> 16;  in nouveau_svm_fault_cache()
467 	fault->client = client;  in nouveau_svm_fault_cache()
468 	fault->fault  = (info & 0x0000001f);  in nouveau_svm_fault_cache()
470 	SVM_DBG(svm, "fault %016llx %016llx %02x",  in nouveau_svm_fault_cache()
471 		fault->inst, fault->addr, fault->access);  in nouveau_svm_fault_cache()
562 	/* Have HMM fault pages within the fault window to the GPU. */  in nouveau_range_fault()
626 	/* Parse available fault buffer entries into a cache, and update  in nouveau_svm_fault()
629 	SVM_DBG(svm, "fault handler");  in nouveau_svm_fault()
645 	SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr);  in nouveau_svm_fault()
651 	sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault),  in nouveau_svm_fault()
657 		if (!svmm || buffer->fault[fi]->inst != inst) {  in nouveau_svm_fault()
659 				nouveau_ivmm_find(svm, buffer->fault[fi]->inst);  in nouveau_svm_fault()
661 			inst = buffer->fault[fi]->inst;  in nouveau_svm_fault()
664 		buffer->fault[fi]->svmm = svmm;  in nouveau_svm_fault()
680 		if (!(svmm = buffer->fault[fi]->svmm)) {  in nouveau_svm_fault()
681 			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);  in nouveau_svm_fault()
684 		SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr);  in nouveau_svm_fault()
689 		start = buffer->fault[fi]->addr;  in nouveau_svm_fault()
696 		 * fault window, determining required pages and access  in nouveau_svm_fault()
703 		 * Determine required permissions based on GPU fault  in nouveau_svm_fault()
707 		switch (buffer->fault[fi]->access) {  in nouveau_svm_fault()
721 			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);  in nouveau_svm_fault()
745 			 * fault addresses have sufficient access permission.  in nouveau_svm_fault()
747 			if (buffer->fault[fn]->svmm != svmm ||  in nouveau_svm_fault()
748 			    buffer->fault[fn]->addr >= limit ||  in nouveau_svm_fault()
749 			    (buffer->fault[fi]->access == 0 /* READ. */ &&  in nouveau_svm_fault()
751 			    (buffer->fault[fi]->access != 0 /* READ. */ &&  in nouveau_svm_fault()
752 			     buffer->fault[fi]->access != 3 /* PREFETCH. */ &&  in nouveau_svm_fault()
760 				struct nouveau_svm_fault *fault =  in nouveau_svm_fault()  local
761 					buffer->fault[fi++];  in nouveau_svm_fault()
763 				nouveau_svm_fault_cancel_fault(svm, fault);  in nouveau_svm_fault()
769 	/* Issue fault replay to the GPU. */  in nouveau_svm_fault()
849 	if (buffer->fault) {  in nouveau_svm_fault_buffer_dtor()
850 		for (i = 0; buffer->fault[i] && i < buffer->entries; i++)  in nouveau_svm_fault_buffer_dtor()
851 			kfree(buffer->fault[i]);  in nouveau_svm_fault_buffer_dtor()
852 		kvfree(buffer->fault);  in nouveau_svm_fault_buffer_dtor()
875 		SVM_ERR(svm, "Fault buffer allocation failed: %d", ret);  in nouveau_svm_fault_buffer_ctor()
890 	buffer->fault = kvzalloc(sizeof(*buffer->fault) * buffer->entries, GFP_KERNEL);  in nouveau_svm_fault_buffer_ctor()
891 	if (!buffer->fault)  in nouveau_svm_fault_buffer_ctor()
951 		SVM_DBG(svm, "No supported fault buffer class");  in nouveau_svm_init()