3  * Adjunct processor matrix VFIO device driver callbacks.
157 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&  in get_update_locks_by_apqn()
158 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {  in get_update_locks_by_apqn()
175  *			       KVM guest to which the matrix mdev linked to a
199  *			    hash table of queues assigned to a matrix mdev
200  * @matrix_mdev: the matrix mdev
634 				struct ap_matrix *matrix)  in vfio_ap_matrix_init()  argument
636 	matrix->apm_max = info->apxa ? info->na : 63;  in vfio_ap_matrix_init()
637 	matrix->aqm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
638 	matrix->adm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
655 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,  in vfio_ap_mdev_filter_cdoms()
663  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
668  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
674  *	 driver, its APID will be filtered from the guest's APCB. The matrix
699 	 * from the matrix mdev, but only those that are assigned to the host's  in vfio_ap_mdev_filter_matrix()
702 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_filter_matrix()
704 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_filter_matrix()
748 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_init_dev()
813 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_fr_queues()
814 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_unlink_fr_queues()
857  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
863  * AP adapter IDs and AP queue indexes is not configured for any matrix
880 		if (mdev_apm == matrix_mdev->matrix.apm &&  in vfio_ap_mdev_verify_no_sharing()
881 		    mdev_aqm == matrix_mdev->matrix.aqm)  in vfio_ap_mdev_verify_no_sharing()
891 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_verify_no_sharing()
895 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_verify_no_sharing()
924 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
925 					       matrix_mdev->matrix.aqm))  in vfio_ap_mdev_validate_masks()
928 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
929 					      matrix_mdev->matrix.aqm);  in vfio_ap_mdev_validate_masks()
937 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)  in vfio_ap_mdev_link_adapter()
985  * corresponding bit in the mediated matrix device's APM
987  * @dev:	the matrix device
988  * @attr:	the mediated matrix device's assign_adapter attribute
1011  *	   matrix device
1033 	if (apid > matrix_mdev->matrix.apm_max) {  in assign_adapter_store()
1038 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in assign_adapter_store()
1043 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1047 		clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1074 	/* If the queue is assigned to the matrix mdev, unlink it. */  in vfio_ap_unlink_apqn_fr_mdev()
1083  *				 adapter from the matrix mdev to which the
1085  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1097 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in vfio_ap_mdev_unlink_adapter()
1132  * corresponding bit in the mediated matrix device's APM
1134  * @dev:	the matrix device
1135  * @attr:	the mediated matrix device's unassign_adapter attribute
1159 	if (apid > matrix_mdev->matrix.apm_max) {  in unassign_adapter_store()
1164 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in unassign_adapter_store()
1169 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
1183 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)  in vfio_ap_mdev_link_domain()
1190  * corresponding bit in the mediated matrix device's AQM
1192  * @dev:	the matrix device
1193  * @attr:	the mediated matrix device's assign_domain attribute
1216  *	   matrix device
1238 	if (apqi > matrix_mdev->matrix.aqm_max) {  in assign_domain_store()
1243 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in assign_domain_store()
1248 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1252 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1279 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_domain()
1314  * corresponding bit in the mediated matrix device's AQM
1316  * @dev:	the matrix device
1317  * @attr:	the mediated matrix device's unassign_domain attribute
1341 	if (apqi > matrix_mdev->matrix.aqm_max) {  in unassign_domain_store()
1346 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in unassign_domain_store()
1351 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
1363  * the corresponding bit in the mediated matrix device's ADM
1365  * @dev:	the matrix device
1366  * @attr:	the mediated matrix device's assign_control_domain attribute
1389 	if (id > matrix_mdev->matrix.adm_max) {  in assign_control_domain_store()
1394 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {  in assign_control_domain_store()
1404 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
1417  * clears the corresponding bit in the mediated matrix device's ADM
1419  * @dev:	the matrix device
1420  * @attr:	the mediated matrix device's unassign_control_domain attribute
1443 	if (domid > matrix_mdev->matrix.adm_max) {  in unassign_control_domain_store()
1448 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {  in unassign_control_domain_store()
1453 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
1476 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
1479 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {  in control_domains_show()
1490 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)  in vfio_ap_mdev_matrix_show()  argument
1497 	unsigned long napm_bits = matrix->apm_max + 1;  in vfio_ap_mdev_matrix_show()
1498 	unsigned long naqm_bits = matrix->aqm_max + 1;  in vfio_ap_mdev_matrix_show()
1502 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);  in vfio_ap_mdev_matrix_show()
1503 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);  in vfio_ap_mdev_matrix_show()
1506 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1507 			for_each_set_bit_inv(apqi, matrix->aqm,  in vfio_ap_mdev_matrix_show()
1516 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1522 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {  in vfio_ap_mdev_matrix_show()
1539 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);  in matrix_show()
1544 static DEVICE_ATTR_RO(matrix);
1586  * @matrix_mdev: a mediated matrix device
1589  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1650  * @matrix_mdev: a matrix mediated device
2036 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&  in vfio_ap_mdev_for_queue()
2037 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))  in vfio_ap_mdev_for_queue()
2058 	/* If the queue is assigned to the matrix mediated device, then  in status_show()
2068 		 * merely assigned to a matrix mediated device.  in status_show()
2184 		 * matrix and update the guest's AP configuration after all of  in vfio_ap_mdev_probe_queue()
2226 			 * Since the queues are defined via a matrix of adapters  in vfio_ap_mdev_remove_queue()
2351 					  matrix_mdev->matrix.apm,  in vfio_ap_mdev_cfg_remove()
2354 					  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_cfg_remove()
2357 					     matrix_mdev->matrix.adm,  in vfio_ap_mdev_cfg_remove()
2469  *			  AP configuration for each matrix mdev to which they
2491 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2493 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);  in vfio_ap_mdev_cfg_add()
2495 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2565 	filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm,  in vfio_ap_mdev_hot_plug_cfg()
2567 	filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm,  in vfio_ap_mdev_hot_plug_cfg()
2569 	filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm,  in vfio_ap_mdev_hot_plug_cfg()