39  * Return 0 on success. If counter mode wasn't changed then it is considered
82 static void auto_mode_init_counter(struct rdma_counter *counter,  in auto_mode_init_counter()  argument
86 	struct auto_mode_param *param = &counter->mode.param;  in auto_mode_init_counter()
88 	counter->mode.mode = RDMA_COUNTER_MODE_AUTO;  in auto_mode_init_counter()
89 	counter->mode.mask = new_mask;  in auto_mode_init_counter()
95 static int __rdma_counter_bind_qp(struct rdma_counter *counter,  in __rdma_counter_bind_qp()  argument
100 	if (qp->counter)  in __rdma_counter_bind_qp()
106 	mutex_lock(&counter->lock);  in __rdma_counter_bind_qp()
107 	ret = qp->device->ops.counter_bind_qp(counter, qp, port);  in __rdma_counter_bind_qp()
108 	mutex_unlock(&counter->lock);  in __rdma_counter_bind_qp()
151 	struct rdma_counter *counter;  in alloc_and_bind()  local
157 	counter = rdma_zalloc_drv_obj(dev, rdma_counter);  in alloc_and_bind()
158 	if (!counter)  in alloc_and_bind()
161 	counter->device    = dev;  in alloc_and_bind()
162 	counter->port      = port;  in alloc_and_bind()
164 	dev->ops.counter_init(counter);  in alloc_and_bind()
166 	rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER);  in alloc_and_bind()
167 	counter->stats = dev->ops.counter_alloc_stats(counter);  in alloc_and_bind()
168 	if (!counter->stats)  in alloc_and_bind()
183 		auto_mode_init_counter(counter, qp, port_counter->mode.mask);  in alloc_and_bind()
194 	counter->mode.mode = mode;  in alloc_and_bind()
195 	counter->mode.bind_opcnt = bind_opcnt;  in alloc_and_bind()
196 	kref_init(&counter->kref);  in alloc_and_bind()
197 	mutex_init(&counter->lock);  in alloc_and_bind()
199 	ret = __rdma_counter_bind_qp(counter, qp, port);  in alloc_and_bind()
203 	rdma_restrack_parent_name(&counter->res, &qp->res);  in alloc_and_bind()
204 	rdma_restrack_add(&counter->res);  in alloc_and_bind()
205 	return counter;  in alloc_and_bind()
208 	rdma_free_hw_stats_struct(counter->stats);  in alloc_and_bind()
210 	rdma_restrack_put(&counter->res);  in alloc_and_bind()
211 	kfree(counter);  in alloc_and_bind()
215 static void rdma_counter_free(struct rdma_counter *counter)  in rdma_counter_free()  argument
219 	port_counter = &counter->device->port_data[counter->port].port_counter;  in rdma_counter_free()
229 	rdma_restrack_del(&counter->res);  in rdma_counter_free()
230 	rdma_free_hw_stats_struct(counter->stats);  in rdma_counter_free()
231 	kfree(counter);  in rdma_counter_free()
234 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,  in auto_mode_match()  argument
237 	struct auto_mode_param *param = &counter->mode.param;  in auto_mode_match()
244 		match &= (task_pid_nr(counter->res.task) ==  in auto_mode_match()
252 	struct rdma_counter *counter = qp->counter;  in __rdma_counter_unbind_qp()  local
258 	mutex_lock(&counter->lock);  in __rdma_counter_unbind_qp()
260 	mutex_unlock(&counter->lock);  in __rdma_counter_unbind_qp()
265 static void counter_history_stat_update(struct rdma_counter *counter)  in counter_history_stat_update()  argument
267 	struct ib_device *dev = counter->device;  in counter_history_stat_update()
271 	port_counter = &dev->port_data[counter->port].port_counter;  in counter_history_stat_update()
275 	rdma_counter_query_stats(counter);  in counter_history_stat_update()
277 	for (i = 0; i < counter->stats->num_counters; i++)  in counter_history_stat_update()
278 		port_counter->hstats->value[i] += counter->stats->value[i];  in counter_history_stat_update()
282  * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
285  * Return: The counter (with ref-count increased) if found
291 	struct rdma_counter *counter = NULL;  in rdma_get_counter_auto_mode()  local
301 		counter = container_of(res, struct rdma_counter, res);  in rdma_get_counter_auto_mode()
302 		if ((counter->device != qp->device) || (counter->port != port))  in rdma_get_counter_auto_mode()
305 		if (auto_mode_match(qp, counter, port_counter->mode.mask))  in rdma_get_counter_auto_mode()
308 		counter = NULL;  in rdma_get_counter_auto_mode()
311 	if (counter && !kref_get_unless_zero(&counter->kref))  in rdma_get_counter_auto_mode()
312 		counter = NULL;  in rdma_get_counter_auto_mode()
315 	return counter;  in rdma_get_counter_auto_mode()
320 	struct rdma_counter *counter;  in counter_release()  local
322 	counter = container_of(kref, struct rdma_counter, kref);  in counter_release()
323 	counter_history_stat_update(counter);  in counter_release()
324 	counter->device->ops.counter_dealloc(counter);  in counter_release()
325 	rdma_counter_free(counter);  in counter_release()
329  * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
336 	struct rdma_counter *counter;  in rdma_counter_bind_qp_auto()  local
349 	counter = rdma_get_counter_auto_mode(qp, port);  in rdma_counter_bind_qp_auto()
350 	if (counter) {  in rdma_counter_bind_qp_auto()
351 		ret = __rdma_counter_bind_qp(counter, qp, port);  in rdma_counter_bind_qp_auto()
353 			kref_put(&counter->kref, counter_release);  in rdma_counter_bind_qp_auto()
357 		counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO,  in rdma_counter_bind_qp_auto()
359 		if (!counter)  in rdma_counter_bind_qp_auto()
367  * rdma_counter_unbind_qp - Unbind a qp from a counter
369  *   true - Decrease the counter ref-count anyway (e.g., qp destroy)
373 	struct rdma_counter *counter = qp->counter;  in rdma_counter_unbind_qp()  local
376 	if (!counter)  in rdma_counter_unbind_qp()
383 	kref_put(&counter->kref, counter_release);  in rdma_counter_unbind_qp()
387 int rdma_counter_query_stats(struct rdma_counter *counter)  in rdma_counter_query_stats()  argument
389 	struct ib_device *dev = counter->device;  in rdma_counter_query_stats()
395 	mutex_lock(&counter->lock);  in rdma_counter_query_stats()
396 	ret = dev->ops.counter_update_stats(counter);  in rdma_counter_query_stats()
397 	mutex_unlock(&counter->lock);  in rdma_counter_query_stats()
407 	struct rdma_counter *counter;  in get_running_counters_hwstat_sum()  local
419 		counter = container_of(res, struct rdma_counter, res);  in get_running_counters_hwstat_sum()
420 		if ((counter->device != dev) || (counter->port != port) ||  in get_running_counters_hwstat_sum()
421 		    rdma_counter_query_stats(counter))  in get_running_counters_hwstat_sum()
424 		sum += counter->stats->value[index];  in get_running_counters_hwstat_sum()
478 	struct rdma_counter *counter;  in rdma_get_counter_by_id()  local
484 	counter = container_of(res, struct rdma_counter, res);  in rdma_get_counter_by_id()
485 	kref_get(&counter->kref);  in rdma_get_counter_by_id()
488 	return counter;  in rdma_get_counter_by_id()
492  * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
498 	struct rdma_counter *counter;  in rdma_counter_bind_qpn()  local
510 	counter = rdma_get_counter_by_id(dev, counter_id);  in rdma_counter_bind_qpn()
511 	if (!counter) {  in rdma_counter_bind_qpn()
516 	if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) {  in rdma_counter_bind_qpn()
521 	if ((counter->device != qp->device) || (counter->port != qp->port)) {  in rdma_counter_bind_qpn()
526 	ret = __rdma_counter_bind_qp(counter, qp, port);  in rdma_counter_bind_qpn()
534 	kref_put(&counter->kref, counter_release);  in rdma_counter_bind_qpn()
541  * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
542  *   The id of new counter is returned in @counter_id
548 	struct rdma_counter *counter;  in rdma_counter_bind_qpn_alloc()  local
571 	counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL, true);  in rdma_counter_bind_qpn_alloc()
572 	if (!counter) {  in rdma_counter_bind_qpn_alloc()
578 		*counter_id = counter->id;  in rdma_counter_bind_qpn_alloc()
589  * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
611 	if (!qp->counter || qp->counter->id != counter_id ||  in rdma_counter_unbind_qpn()