3  * Adjunct processor matrix VFIO device driver callbacks.
157 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&  in get_update_locks_by_apqn()
158 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {  in get_update_locks_by_apqn()
175  *			       KVM guest to which the matrix mdev linked to a
199  *			    hash table of queues assigned to a matrix mdev
200  * @matrix_mdev: the matrix mdev
646 				struct ap_matrix *matrix)  in vfio_ap_matrix_init()  argument
648 	matrix->apm_max = info->apxa ? info->na : 63;  in vfio_ap_matrix_init()
649 	matrix->aqm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
650 	matrix->adm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
676 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,  in vfio_ap_mdev_filter_cdoms()
699  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
704  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
710  *	 driver, its APID will be filtered from the guest's APCB. The matrix
734 	 * from the matrix mdev, but only those that are assigned to the host's  in vfio_ap_mdev_filter_matrix()
737 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_filter_matrix()
739 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_filter_matrix()
782 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_init_dev()
849 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_fr_queues()
850 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_unlink_fr_queues()
906  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
908  * @assignee: the matrix mdev to which @mdev_apm and @mdev_aqm are being
945 		if (!bitmap_and(apm, mdev_apm, assigned_to->matrix.apm,	AP_DEVICES))  in vfio_ap_mdev_verify_no_sharing()
948 		if (!bitmap_and(aqm, mdev_aqm, assigned_to->matrix.aqm,	AP_DOMAINS))  in vfio_ap_mdev_verify_no_sharing()
979 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
980 					       matrix_mdev->matrix.aqm))  in vfio_ap_mdev_validate_masks()
984 					      matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
985 					      matrix_mdev->matrix.aqm);  in vfio_ap_mdev_validate_masks()
993 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)  in vfio_ap_mdev_link_adapter()
1041  * corresponding bit in the mediated matrix device's APM
1043  * @dev:	the matrix device
1044  * @attr:	the mediated matrix device's assign_adapter attribute
1067  *	   matrix device
1089 	if (apid > matrix_mdev->matrix.apm_max) {  in assign_adapter_store()
1094 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in assign_adapter_store()
1099 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1103 		clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
1130 	/* If the queue is assigned to the matrix mdev, unlink it. */  in vfio_ap_unlink_apqn_fr_mdev()
1139  *				 adapter from the matrix mdev to which the
1141  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1153 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in vfio_ap_mdev_unlink_adapter()
1207  * corresponding bit in the mediated matrix device's APM
1209  * @dev:	the matrix device
1210  * @attr:	the mediated matrix device's unassign_adapter attribute
1234 	if (apid > matrix_mdev->matrix.apm_max) {  in unassign_adapter_store()
1239 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in unassign_adapter_store()
1244 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
1258 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)  in vfio_ap_mdev_link_domain()
1265  * corresponding bit in the mediated matrix device's AQM
1267  * @dev:	the matrix device
1268  * @attr:	the mediated matrix device's assign_domain attribute
1291  *	   matrix device
1313 	if (apqi > matrix_mdev->matrix.aqm_max) {  in assign_domain_store()
1318 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in assign_domain_store()
1323 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1327 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1354 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_domain()
1408  * corresponding bit in the mediated matrix device's AQM
1410  * @dev:	the matrix device
1411  * @attr:	the mediated matrix device's unassign_domain attribute
1435 	if (apqi > matrix_mdev->matrix.aqm_max) {  in unassign_domain_store()
1440 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in unassign_domain_store()
1445 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
1457  * the corresponding bit in the mediated matrix device's ADM
1459  * @dev:	the matrix device
1460  * @attr:	the mediated matrix device's assign_control_domain attribute
1483 	if (id > matrix_mdev->matrix.adm_max) {  in assign_control_domain_store()
1488 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {  in assign_control_domain_store()
1498 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
1511  * clears the corresponding bit in the mediated matrix device's ADM
1513  * @dev:	the matrix device
1514  * @attr:	the mediated matrix device's unassign_control_domain attribute
1537 	if (domid > matrix_mdev->matrix.adm_max) {  in unassign_control_domain_store()
1542 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {  in unassign_control_domain_store()
1547 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
1567 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
1571 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1)  in control_domains_show()
1579 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)  in vfio_ap_mdev_matrix_show()  argument
1585 	unsigned long napm_bits = matrix->apm_max + 1;  in vfio_ap_mdev_matrix_show()
1586 	unsigned long naqm_bits = matrix->aqm_max + 1;  in vfio_ap_mdev_matrix_show()
1589 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);  in vfio_ap_mdev_matrix_show()
1590 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);  in vfio_ap_mdev_matrix_show()
1593 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1594 			for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)  in vfio_ap_mdev_matrix_show()
1598 		for_each_set_bit_inv(apid, matrix->apm, napm_bits)  in vfio_ap_mdev_matrix_show()
1601 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits)  in vfio_ap_mdev_matrix_show()
1615 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);  in matrix_show()
1620 static DEVICE_ATTR_RO(matrix);
1648 	idx += write_ap_bitmap(matrix_mdev->matrix.apm, buf, idx, ',');  in ap_config_show()
1649 	idx += write_ap_bitmap(matrix_mdev->matrix.aqm, buf, idx, ',');  in ap_config_show()
1650 	idx += write_ap_bitmap(matrix_mdev->matrix.adm, buf, idx, '\n');  in ap_config_show()
1676 	for_each_set_bit_inv(bit, matrix_mdev->matrix.apm, AP_DEVICES) {  in ap_matrix_overflow_check()
1677 		if (bit > matrix_mdev->matrix.apm_max)  in ap_matrix_overflow_check()
1681 	for_each_set_bit_inv(bit, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in ap_matrix_overflow_check()
1682 		if (bit > matrix_mdev->matrix.aqm_max)  in ap_matrix_overflow_check()
1686 	for_each_set_bit_inv(bit, matrix_mdev->matrix.adm, AP_DOMAINS) {  in ap_matrix_overflow_check()
1687 		if (bit > matrix_mdev->matrix.adm_max)  in ap_matrix_overflow_check()
1725 	ap_matrix_copy(&m_old, &matrix_mdev->matrix);  in ap_config_store()
1739 	ap_matrix_copy(&matrix_mdev->matrix, &m_new);  in ap_config_store()
1744 		ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1749 		ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1755 	ap_matrix_copy(&matrix_mdev->matrix, &m_old);  in ap_config_store()
1762 	ap_matrix_copy(&matrix_mdev->matrix, &m_new);  in ap_config_store()
1815  * @matrix_mdev: a mediated matrix device
1818  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1879  * @matrix_mdev: a matrix mediated device
2301 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&  in vfio_ap_mdev_for_queue()
2302 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))  in vfio_ap_mdev_for_queue()
2323 	/* If the queue is assigned to the matrix mediated device, then  in status_show()
2333 		 * merely assigned to a matrix mediated device.  in status_show()
2446 		 * matrix and update the guest's AP configuration after all of  in vfio_ap_mdev_probe_queue()
2488 			 * Since the queues are defined via a matrix of adapters  in vfio_ap_mdev_remove_queue()
2613 					  matrix_mdev->matrix.apm,  in vfio_ap_mdev_cfg_remove()
2616 					  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_cfg_remove()
2619 					     matrix_mdev->matrix.adm,  in vfio_ap_mdev_cfg_remove()
2731  *			  AP configuration for each matrix mdev to which they
2753 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2755 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);  in vfio_ap_mdev_cfg_add()
2757 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2827 	filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm,  in vfio_ap_mdev_hot_plug_cfg()
2829 	filter_domains = bitmap_intersects(matrix_mdev->matrix.aqm,  in vfio_ap_mdev_hot_plug_cfg()
2831 	filter_cdoms = bitmap_intersects(matrix_mdev->matrix.adm,  in vfio_ap_mdev_hot_plug_cfg()