Lines Matching +full:unit +full:- +full:address
15 * - Redistributions of source code must retain the above
19 * - Redistributions in binary form must reproduce the above
54 hop_num = hr_dev->caps.qpc_hop_num; in hns_roce_check_whether_mhop()
57 hop_num = hr_dev->caps.mpt_hop_num; in hns_roce_check_whether_mhop()
60 hop_num = hr_dev->caps.cqc_hop_num; in hns_roce_check_whether_mhop()
63 hop_num = hr_dev->caps.srqc_hop_num; in hns_roce_check_whether_mhop()
66 hop_num = hr_dev->caps.sccc_hop_num; in hns_roce_check_whether_mhop()
69 hop_num = hr_dev->caps.qpc_timer_hop_num; in hns_roce_check_whether_mhop()
72 hop_num = hr_dev->caps.cqc_timer_hop_num; in hns_roce_check_whether_mhop()
75 hop_num = hr_dev->caps.gmv_hop_num; in hns_roce_check_whether_mhop()
126 struct device *dev = hr_dev->dev; in get_hem_table_config()
130 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz in get_hem_table_config()
132 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz in get_hem_table_config()
134 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; in get_hem_table_config()
135 mhop->hop_num = hr_dev->caps.qpc_hop_num; in get_hem_table_config()
138 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz in get_hem_table_config()
140 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz in get_hem_table_config()
142 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; in get_hem_table_config()
143 mhop->hop_num = hr_dev->caps.mpt_hop_num; in get_hem_table_config()
146 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz in get_hem_table_config()
148 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz in get_hem_table_config()
150 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; in get_hem_table_config()
151 mhop->hop_num = hr_dev->caps.cqc_hop_num; in get_hem_table_config()
154 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz in get_hem_table_config()
156 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz in get_hem_table_config()
158 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; in get_hem_table_config()
159 mhop->hop_num = hr_dev->caps.sccc_hop_num; in get_hem_table_config()
162 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz in get_hem_table_config()
164 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz in get_hem_table_config()
166 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; in get_hem_table_config()
167 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; in get_hem_table_config()
170 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz in get_hem_table_config()
172 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz in get_hem_table_config()
174 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; in get_hem_table_config()
175 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; in get_hem_table_config()
178 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz in get_hem_table_config()
180 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz in get_hem_table_config()
182 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; in get_hem_table_config()
183 mhop->hop_num = hr_dev->caps.srqc_hop_num; in get_hem_table_config()
186 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + in get_hem_table_config()
188 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz + in get_hem_table_config()
190 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num; in get_hem_table_config()
191 mhop->hop_num = hr_dev->caps.gmv_hop_num; in get_hem_table_config()
194 dev_err(dev, "table %u not support multi-hop addressing!\n", in get_hem_table_config()
196 return -EINVAL; in get_hem_table_config()
206 struct device *dev = hr_dev->dev; in hns_roce_calc_hem_mhop()
212 if (get_hem_table_config(hr_dev, mhop, table->type)) in hns_roce_calc_hem_mhop()
213 return -EINVAL; in hns_roce_calc_hem_mhop()
222 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
223 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; in hns_roce_calc_hem_mhop()
224 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : in hns_roce_calc_hem_mhop()
225 mhop->bt_chunk_size; in hns_roce_calc_hem_mhop()
226 table_idx = *obj / (chunk_size / table->obj_size); in hns_roce_calc_hem_mhop()
229 mhop->l2_idx = table_idx & (chunk_ba_num - 1); in hns_roce_calc_hem_mhop()
230 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); in hns_roce_calc_hem_mhop()
231 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; in hns_roce_calc_hem_mhop()
234 mhop->l1_idx = table_idx & (chunk_ba_num - 1); in hns_roce_calc_hem_mhop()
235 mhop->l0_idx = table_idx / chunk_ba_num; in hns_roce_calc_hem_mhop()
238 mhop->l0_idx = table_idx; in hns_roce_calc_hem_mhop()
242 table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
243 return -EINVAL; in hns_roce_calc_hem_mhop()
245 if (mhop->l0_idx >= mhop->ba_l0_num) in hns_roce_calc_hem_mhop()
246 mhop->l0_idx %= mhop->ba_l0_num; in hns_roce_calc_hem_mhop()
269 INIT_LIST_HEAD(&hem->chunk_list); in hns_roce_alloc_hem()
280 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); in hns_roce_alloc_hem()
281 chunk->npages = 0; in hns_roce_alloc_hem()
282 chunk->nsg = 0; in hns_roce_alloc_hem()
283 memset(chunk->buf, 0, sizeof(chunk->buf)); in hns_roce_alloc_hem()
284 list_add_tail(&chunk->list, &hem->chunk_list); in hns_roce_alloc_hem()
288 --order; in hns_roce_alloc_hem()
294 mem = &chunk->mem[chunk->npages]; in hns_roce_alloc_hem()
295 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, in hns_roce_alloc_hem()
300 chunk->buf[chunk->npages] = buf; in hns_roce_alloc_hem()
303 ++chunk->npages; in hns_roce_alloc_hem()
304 ++chunk->nsg; in hns_roce_alloc_hem()
305 npages -= 1 << order; in hns_roce_alloc_hem()
323 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { in hns_roce_free_hem()
324 for (i = 0; i < chunk->npages; ++i) in hns_roce_free_hem()
325 dma_free_coherent(hr_dev->dev, in hns_roce_free_hem()
326 sg_dma_len(&chunk->mem[i]), in hns_roce_free_hem()
327 chunk->buf[i], in hns_roce_free_hem()
328 sg_dma_address(&chunk->mem[i])); in hns_roce_free_hem()
340 struct ib_device *ibdev = &hr_dev->ib_dev; in calc_hem_config()
351 l0_idx = mhop->l0_idx; in calc_hem_config()
352 l1_idx = mhop->l1_idx; in calc_hem_config()
353 l2_idx = mhop->l2_idx; in calc_hem_config()
354 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; in calc_hem_config()
355 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in calc_hem_config()
358 index->l1 = l0_idx * chunk_ba_num + l1_idx; in calc_hem_config()
359 index->l0 = l0_idx; in calc_hem_config()
360 index->buf = l0_idx * chunk_ba_num * chunk_ba_num + in calc_hem_config()
364 index->l0 = l0_idx; in calc_hem_config()
365 index->buf = l0_idx * chunk_ba_num + l1_idx; in calc_hem_config()
368 index->buf = l0_idx; in calc_hem_config()
372 table->type, mhop->hop_num); in calc_hem_config()
373 return -EINVAL; in calc_hem_config()
376 if (unlikely(index->buf >= table->num_hem)) { in calc_hem_config()
378 table->type, index->buf, table->num_hem); in calc_hem_config()
379 return -EINVAL; in calc_hem_config()
390 u32 bt_size = mhop->bt_chunk_size; in free_mhop_hem()
391 struct device *dev = hr_dev->dev; in free_mhop_hem()
393 if (index->inited & HEM_INDEX_BUF) { in free_mhop_hem()
394 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
395 table->hem[index->buf] = NULL; in free_mhop_hem()
398 if (index->inited & HEM_INDEX_L1) { in free_mhop_hem()
399 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], in free_mhop_hem()
400 table->bt_l1_dma_addr[index->l1]); in free_mhop_hem()
401 table->bt_l1[index->l1] = NULL; in free_mhop_hem()
404 if (index->inited & HEM_INDEX_L0) { in free_mhop_hem()
405 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], in free_mhop_hem()
406 table->bt_l0_dma_addr[index->l0]); in free_mhop_hem()
407 table->bt_l0[index->l0] = NULL; in free_mhop_hem()
416 u32 bt_size = mhop->bt_chunk_size; in alloc_mhop_hem()
417 struct device *dev = hr_dev->dev; in alloc_mhop_hem()
425 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || in alloc_mhop_hem()
426 check_whether_bt_num_2(table->type, mhop->hop_num)) && in alloc_mhop_hem()
427 !table->bt_l0[index->l0]) { in alloc_mhop_hem()
428 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
429 &table->bt_l0_dma_addr[index->l0], in alloc_mhop_hem()
431 if (!table->bt_l0[index->l0]) { in alloc_mhop_hem()
432 ret = -ENOMEM; in alloc_mhop_hem()
435 index->inited |= HEM_INDEX_L0; in alloc_mhop_hem()
439 if (check_whether_bt_num_3(table->type, mhop->hop_num) && in alloc_mhop_hem()
440 !table->bt_l1[index->l1]) { in alloc_mhop_hem()
441 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
442 &table->bt_l1_dma_addr[index->l1], in alloc_mhop_hem()
444 if (!table->bt_l1[index->l1]) { in alloc_mhop_hem()
445 ret = -ENOMEM; in alloc_mhop_hem()
448 index->inited |= HEM_INDEX_L1; in alloc_mhop_hem()
449 *(table->bt_l0[index->l0] + mhop->l1_idx) = in alloc_mhop_hem()
450 table->bt_l1_dma_addr[index->l1]; in alloc_mhop_hem()
457 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; in alloc_mhop_hem()
459 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, in alloc_mhop_hem()
461 if (!table->hem[index->buf]) { in alloc_mhop_hem()
462 ret = -ENOMEM; in alloc_mhop_hem()
466 index->inited |= HEM_INDEX_BUF; in alloc_mhop_hem()
467 hns_roce_hem_first(table->hem[index->buf], &iter); in alloc_mhop_hem()
469 if (table->type < HEM_TYPE_MTT) { in alloc_mhop_hem()
470 if (mhop->hop_num == 2) in alloc_mhop_hem()
471 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; in alloc_mhop_hem()
472 else if (mhop->hop_num == 1) in alloc_mhop_hem()
473 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
474 } else if (mhop->hop_num == 2) { in alloc_mhop_hem()
475 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
490 struct ib_device *ibdev = &hr_dev->ib_dev; in set_mhop_hem()
494 if (index->inited & HEM_INDEX_L0) { in set_mhop_hem()
495 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); in set_mhop_hem()
502 if (index->inited & HEM_INDEX_L1) { in set_mhop_hem()
503 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); in set_mhop_hem()
510 if (index->inited & HEM_INDEX_BUF) { in set_mhop_hem()
511 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) in set_mhop_hem()
514 step_idx = mhop->hop_num; in set_mhop_hem()
515 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); in set_mhop_hem()
527 struct ib_device *ibdev = &hr_dev->ib_dev; in hns_roce_table_mhop_get()
538 mutex_lock(&table->mutex); in hns_roce_table_mhop_get()
539 if (table->hem[index.buf]) { in hns_roce_table_mhop_get()
540 refcount_inc(&table->hem[index.buf]->refcount); in hns_roce_table_mhop_get()
550 /* set HEM base address to hardware */ in hns_roce_table_mhop_get()
551 if (table->type < HEM_TYPE_MTT) { in hns_roce_table_mhop_get()
554 ibdev_err(ibdev, "set HEM address to HW failed!\n"); in hns_roce_table_mhop_get()
559 refcount_set(&table->hem[index.buf]->refcount, 1); in hns_roce_table_mhop_get()
565 mutex_unlock(&table->mutex); in hns_roce_table_mhop_get()
572 struct device *dev = hr_dev->dev; in hns_roce_table_get()
576 if (hns_roce_check_whether_mhop(hr_dev, table->type)) in hns_roce_table_get()
579 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_get()
581 mutex_lock(&table->mutex); in hns_roce_table_get()
583 if (table->hem[i]) { in hns_roce_table_get()
584 refcount_inc(&table->hem[i]->refcount); in hns_roce_table_get()
588 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
589 table->table_chunk_size >> PAGE_SHIFT, in hns_roce_table_get()
590 table->table_chunk_size, in hns_roce_table_get()
592 if (!table->hem[i]) { in hns_roce_table_get()
593 ret = -ENOMEM; in hns_roce_table_get()
597 /* Set HEM base address(128K/page, pa) to Hardware */ in hns_roce_table_get()
598 ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); in hns_roce_table_get()
600 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
601 table->hem[i] = NULL; in hns_roce_table_get()
602 dev_err(dev, "set HEM base address to HW failed, ret = %d.\n", in hns_roce_table_get()
607 refcount_set(&table->hem[i]->refcount, 1); in hns_roce_table_get()
609 mutex_unlock(&table->mutex); in hns_roce_table_get()
618 struct ib_device *ibdev = &hr_dev->ib_dev; in clear_mhop_hem()
619 u32 hop_num = mhop->hop_num; in clear_mhop_hem()
624 index->inited = HEM_INDEX_BUF; in clear_mhop_hem()
625 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; in clear_mhop_hem()
626 if (check_whether_bt_num_2(table->type, hop_num)) { in clear_mhop_hem()
627 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
628 chunk_ba_num, table->num_hem)) in clear_mhop_hem()
629 index->inited |= HEM_INDEX_L0; in clear_mhop_hem()
630 } else if (check_whether_bt_num_3(table->type, hop_num)) { in clear_mhop_hem()
631 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
632 chunk_ba_num, table->num_hem)) { in clear_mhop_hem()
633 index->inited |= HEM_INDEX_L1; in clear_mhop_hem()
634 if (hns_roce_check_bt_null(table->bt_l1, index->l1, in clear_mhop_hem()
636 index->inited |= HEM_INDEX_L0; in clear_mhop_hem()
640 if (table->type < HEM_TYPE_MTT) { in clear_mhop_hem()
646 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx); in clear_mhop_hem()
651 if (index->inited & HEM_INDEX_L1) { in clear_mhop_hem()
652 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1); in clear_mhop_hem()
658 if (index->inited & HEM_INDEX_L0) { in clear_mhop_hem()
659 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0); in clear_mhop_hem()
672 struct ib_device *ibdev = &hr_dev->ib_dev; in hns_roce_table_mhop_put()
684 mutex_lock(&table->mutex); in hns_roce_table_mhop_put()
685 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, in hns_roce_table_mhop_put()
686 &table->mutex)) in hns_roce_table_mhop_put()
692 mutex_unlock(&table->mutex); in hns_roce_table_mhop_put()
698 struct device *dev = hr_dev->dev; in hns_roce_table_put()
702 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_put()
707 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_put()
709 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, in hns_roce_table_put()
710 &table->mutex)) in hns_roce_table_put()
713 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT); in hns_roce_table_put()
715 dev_warn(dev, "failed to clear HEM base address, ret = %d.\n", in hns_roce_table_put()
718 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
719 table->hem[i] = NULL; in hns_roce_table_put()
721 mutex_unlock(&table->mutex); in hns_roce_table_put()
740 mutex_lock(&table->mutex); in hns_roce_table_find()
742 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_find()
743 obj_per_chunk = table->table_chunk_size / table->obj_size; in hns_roce_table_find()
744 hem = table->hem[obj / obj_per_chunk]; in hns_roce_table_find()
746 dma_offset = offset = idx_offset * table->obj_size; in hns_roce_table_find()
761 hem = table->hem[hem_idx]; in hns_roce_table_find()
770 list_for_each_entry(chunk, &hem->chunk_list, list) { in hns_roce_table_find()
771 for (i = 0; i < chunk->npages; ++i) { in hns_roce_table_find()
772 length = sg_dma_len(&chunk->mem[i]); in hns_roce_table_find()
776 &chunk->mem[i]) + dma_offset; in hns_roce_table_find()
777 dma_offset -= length; in hns_roce_table_find()
781 addr = chunk->buf[i] + offset; in hns_roce_table_find()
784 offset -= length; in hns_roce_table_find()
789 mutex_unlock(&table->mutex); in hns_roce_table_find()
801 table->table_chunk_size = hr_dev->caps.chunk_sz; in hns_roce_init_hem_table()
802 obj_per_chunk = table->table_chunk_size / obj_size; in hns_roce_init_hem_table()
805 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); in hns_roce_init_hem_table()
806 if (!table->hem) in hns_roce_init_hem_table()
807 return -ENOMEM; in hns_roce_init_hem_table()
817 return -EINVAL; in hns_roce_init_hem_table()
831 table->hem = kcalloc(num_hem, sizeof(*table->hem), in hns_roce_init_hem_table()
833 if (!table->hem) in hns_roce_init_hem_table()
840 table->bt_l1 = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
841 sizeof(*table->bt_l1), in hns_roce_init_hem_table()
843 if (!table->bt_l1) in hns_roce_init_hem_table()
846 table->bt_l1_dma_addr = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
847 sizeof(*table->bt_l1_dma_addr), in hns_roce_init_hem_table()
850 if (!table->bt_l1_dma_addr) in hns_roce_init_hem_table()
856 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), in hns_roce_init_hem_table()
858 if (!table->bt_l0) in hns_roce_init_hem_table()
861 table->bt_l0_dma_addr = kcalloc(num_bt_l0, in hns_roce_init_hem_table()
862 sizeof(*table->bt_l0_dma_addr), in hns_roce_init_hem_table()
864 if (!table->bt_l0_dma_addr) in hns_roce_init_hem_table()
869 table->type = type; in hns_roce_init_hem_table()
870 table->num_hem = num_hem; in hns_roce_init_hem_table()
871 table->obj_size = obj_size; in hns_roce_init_hem_table()
872 mutex_init(&table->mutex); in hns_roce_init_hem_table()
877 kfree(table->bt_l0); in hns_roce_init_hem_table()
878 table->bt_l0 = NULL; in hns_roce_init_hem_table()
881 kfree(table->bt_l1_dma_addr); in hns_roce_init_hem_table()
882 table->bt_l1_dma_addr = NULL; in hns_roce_init_hem_table()
885 kfree(table->bt_l1); in hns_roce_init_hem_table()
886 table->bt_l1 = NULL; in hns_roce_init_hem_table()
889 kfree(table->hem); in hns_roce_init_hem_table()
890 table->hem = NULL; in hns_roce_init_hem_table()
893 return -ENOMEM; in hns_roce_init_hem_table()
906 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : in hns_roce_cleanup_mhop_hem_table()
909 for (i = 0; i < table->num_hem; ++i) { in hns_roce_cleanup_mhop_hem_table()
910 obj = i * buf_chunk_size / table->obj_size; in hns_roce_cleanup_mhop_hem_table()
911 if (table->hem[i]) in hns_roce_cleanup_mhop_hem_table()
915 kfree(table->hem); in hns_roce_cleanup_mhop_hem_table()
916 table->hem = NULL; in hns_roce_cleanup_mhop_hem_table()
917 kfree(table->bt_l1); in hns_roce_cleanup_mhop_hem_table()
918 table->bt_l1 = NULL; in hns_roce_cleanup_mhop_hem_table()
919 kfree(table->bt_l1_dma_addr); in hns_roce_cleanup_mhop_hem_table()
920 table->bt_l1_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
921 kfree(table->bt_l0); in hns_roce_cleanup_mhop_hem_table()
922 table->bt_l0 = NULL; in hns_roce_cleanup_mhop_hem_table()
923 kfree(table->bt_l0_dma_addr); in hns_roce_cleanup_mhop_hem_table()
924 table->bt_l0_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
930 struct device *dev = hr_dev->dev; in hns_roce_cleanup_hem_table()
935 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_cleanup_hem_table()
940 for (i = 0; i < table->num_hem; ++i) in hns_roce_cleanup_hem_table()
941 if (table->hem[i]) { in hns_roce_cleanup_hem_table()
942 obj = i * table->table_chunk_size / table->obj_size; in hns_roce_cleanup_hem_table()
943 ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0); in hns_roce_cleanup_hem_table()
945 dev_err(dev, "clear HEM base address failed, ret = %d.\n", in hns_roce_cleanup_hem_table()
948 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
951 kfree(table->hem); in hns_roce_cleanup_hem_table()
956 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) in hns_roce_cleanup_hem()
958 &hr_dev->srq_table.table); in hns_roce_cleanup_hem()
959 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); in hns_roce_cleanup_hem()
960 if (hr_dev->caps.qpc_timer_entry_sz) in hns_roce_cleanup_hem()
962 &hr_dev->qpc_timer_table); in hns_roce_cleanup_hem()
963 if (hr_dev->caps.cqc_timer_entry_sz) in hns_roce_cleanup_hem()
965 &hr_dev->cqc_timer_table); in hns_roce_cleanup_hem()
966 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) in hns_roce_cleanup_hem()
968 &hr_dev->qp_table.sccc_table); in hns_roce_cleanup_hem()
969 if (hr_dev->caps.trrl_entry_sz) in hns_roce_cleanup_hem()
971 &hr_dev->qp_table.trrl_table); in hns_roce_cleanup_hem()
973 if (hr_dev->caps.gmv_entry_sz) in hns_roce_cleanup_hem()
974 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table); in hns_roce_cleanup_hem()
976 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); in hns_roce_cleanup_hem()
977 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); in hns_roce_cleanup_hem()
978 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); in hns_roce_cleanup_hem()
1009 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, in hem_list_alloc_item()
1010 &hem->dma_addr, GFP_KERNEL); in hem_list_alloc_item()
1011 if (!hem->addr) { in hem_list_alloc_item()
1017 hem->count = count; in hem_list_alloc_item()
1018 hem->start = start; in hem_list_alloc_item()
1019 hem->end = end; in hem_list_alloc_item()
1020 INIT_LIST_HEAD(&hem->list); in hem_list_alloc_item()
1021 INIT_LIST_HEAD(&hem->sibling); in hem_list_alloc_item()
1030 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, in hem_list_free_item()
1031 hem->addr, hem->dma_addr); in hem_list_free_item()
1041 list_del(&hem->list); in hem_list_free_all()
1052 /* assign L0 table address to hem from root bt */
1057 hem->addr = cpu_addr; in hem_list_assign_bt()
1058 hem->dma_addr = (dma_addr_t)phy_addr; in hem_list_assign_bt()
1064 return (hem->start <= offset && offset <= hem->end); in hem_list_page_is_in_range()
1086 * hopnum base address table levels in hem_list_is_bottom_bt()
1088 * 1 L0 -> buf in hem_list_is_bottom_bt()
1089 * 2 L0 -> L1 -> buf in hem_list_is_bottom_bt()
1090 * 3 L0 -> L1 -> L2 -> buf in hem_list_is_bottom_bt()
1092 return bt_level >= (hopnum ? hopnum - 1 : hopnum); in hem_list_is_bottom_bt()
1096 * calc base address entries num
1098 * @bt_level: base address table level
1099 * @unit: ba entries per bt page
1101 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) in hem_list_calc_ba_range() argument
1111 * 1 0 unit in hem_list_calc_ba_range()
1112 * ------------ in hem_list_calc_ba_range()
1113 * 2 0 unit * unit in hem_list_calc_ba_range()
1114 * 2 1 unit in hem_list_calc_ba_range()
1115 * ------------ in hem_list_calc_ba_range()
1116 * 3 0 unit * unit * unit in hem_list_calc_ba_range()
1117 * 3 1 unit * unit in hem_list_calc_ba_range()
1118 * 3 2 unit in hem_list_calc_ba_range()
1121 max = hopnum - bt_level; in hem_list_calc_ba_range()
1123 step = step * unit; in hem_list_calc_ba_range()
1132 * @unit: ba entries per bt page
1135 int region_cnt, int unit) in hns_roce_hem_list_calc_root_ba() argument
1144 if (r->hopnum > 1) { in hns_roce_hem_list_calc_root_ba()
1145 step = hem_list_calc_ba_range(r->hopnum, 1, unit); in hns_roce_hem_list_calc_root_ba()
1147 total += (r->count + step - 1) / step; in hns_roce_hem_list_calc_root_ba()
1149 total += r->count; in hns_roce_hem_list_calc_root_ba()
1157 const struct hns_roce_buf_region *r, int unit, in hem_list_alloc_mid_bt() argument
1164 const int hopnum = r->hopnum; in hem_list_alloc_mid_bt()
1177 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); in hem_list_alloc_mid_bt()
1178 return -EINVAL; in hem_list_alloc_mid_bt()
1181 if (offset < r->offset) { in hem_list_alloc_mid_bt()
1182 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", in hem_list_alloc_mid_bt()
1183 offset, r->offset); in hem_list_alloc_mid_bt()
1184 return -EINVAL; in hem_list_alloc_mid_bt()
1187 distance = offset - r->offset; in hem_list_alloc_mid_bt()
1188 max_ofs = r->offset + r->count - 1; in hem_list_alloc_mid_bt()
1200 step = hem_list_calc_ba_range(hopnum, level, unit); in hem_list_alloc_mid_bt()
1202 ret = -EINVAL; in hem_list_alloc_mid_bt()
1206 start_aligned = (distance / step) * step + r->offset; in hem_list_alloc_mid_bt()
1207 end = min_t(int, start_aligned + step - 1, max_ofs); in hem_list_alloc_mid_bt()
1208 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, in hem_list_alloc_mid_bt()
1211 ret = -ENOMEM; in hem_list_alloc_mid_bt()
1215 list_add(&cur->list, &temp_list[level]); in hem_list_alloc_mid_bt()
1217 list_add(&cur->sibling, &temp_list[0]); in hem_list_alloc_mid_bt()
1221 pre = hem_ptrs[level - 1]; in hem_list_alloc_mid_bt()
1222 step = (cur->start - pre->start) / step * BA_BYTE_LEN; in hem_list_alloc_mid_bt()
1223 hem_list_link_bt(hr_dev, pre->addr + step, in hem_list_alloc_mid_bt()
1224 cur->dma_addr); in hem_list_alloc_mid_bt()
1242 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, in alloc_root_hem() argument
1250 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); in alloc_root_hem()
1252 return ERR_PTR(-ENOMEM); in alloc_root_hem()
1254 if (ba_num > unit) in alloc_root_hem()
1255 return ERR_PTR(-ENOBUFS); in alloc_root_hem()
1259 r = ®ions[region_cnt - 1]; in alloc_root_hem()
1260 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, in alloc_root_hem()
1263 return ERR_PTR(-ENOMEM); in alloc_root_hem()
1277 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, in alloc_fake_root_bt()
1278 r->count, false); in alloc_fake_root_bt()
1280 return -ENOMEM; in alloc_fake_root_bt()
1283 list_add(&hem->list, branch_head); in alloc_fake_root_bt()
1284 list_add(&hem->sibling, leaf_head); in alloc_fake_root_bt()
1286 return r->count; in alloc_fake_root_bt()
1290 int unit, const struct hns_roce_buf_region *r, in setup_middle_bt() argument
1298 step = hem_list_calc_ba_range(r->hopnum, 1, unit); in setup_middle_bt()
1300 return -EINVAL; in setup_middle_bt()
1304 offset = (hem->start - r->offset) / step * BA_BYTE_LEN; in setup_middle_bt()
1305 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); in setup_middle_bt()
1314 int unit, int max_ba_num, struct hns_roce_hem_head *head, in setup_root_hem() argument
1324 root_hem = list_first_entry(&head->root, in setup_root_hem()
1327 return -ENOMEM; in setup_root_hem()
1332 if (!r->count) in setup_root_hem()
1336 cpu_base = root_hem->addr + total * BA_BYTE_LEN; in setup_root_hem()
1337 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; in setup_root_hem()
1340 * which's address share to all regions. in setup_root_hem()
1342 if (hem_list_is_bottom_bt(r->hopnum, 0)) in setup_root_hem()
1344 &head->branch[i], &head->leaf); in setup_root_hem()
1346 ret = setup_middle_bt(hr_dev, cpu_base, unit, r, in setup_root_hem()
1347 &hem_list->mid_bt[i][1]); in setup_root_hem()
1355 list_splice(&head->leaf, &hem_list->btm_bt); in setup_root_hem()
1356 list_splice(&head->root, &hem_list->root_bt); in setup_root_hem()
1358 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); in setup_root_hem()
1364 struct hns_roce_hem_list *hem_list, int unit, in hem_list_alloc_root_bt() argument
1374 root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); in hem_list_alloc_root_bt()
1379 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, in hem_list_alloc_root_bt()
1390 hem_list->root_ba = root_hem->dma_addr; in hem_list_alloc_root_bt()
1391 list_add(&root_hem->list, &head.root); in hem_list_alloc_root_bt()
1392 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, in hem_list_alloc_root_bt()
1404 /* construct the base address table and link them by address hop config */
1412 int unit; in hns_roce_hem_list_request() local
1417 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", in hns_roce_hem_list_request()
1419 return -EINVAL; in hns_roce_hem_list_request()
1422 unit = (1 << bt_pg_shift) / BA_BYTE_LEN; in hns_roce_hem_list_request()
1425 if (!r->count) in hns_roce_hem_list_request()
1428 end = r->offset + r->count; in hns_roce_hem_list_request()
1429 for (ofs = r->offset; ofs < end; ofs += unit) { in hns_roce_hem_list_request()
1430 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, in hns_roce_hem_list_request()
1431 hem_list->mid_bt[i], in hns_roce_hem_list_request()
1432 &hem_list->btm_bt); in hns_roce_hem_list_request()
1434 dev_err(hr_dev->dev, in hns_roce_hem_list_request()
1441 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, in hns_roce_hem_list_request()
1444 dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret); in hns_roce_hem_list_request()
1461 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], in hns_roce_hem_list_release()
1464 hem_list_free_all(hr_dev, &hem_list->root_bt, true); in hns_roce_hem_list_release()
1465 INIT_LIST_HEAD(&hem_list->btm_bt); in hns_roce_hem_list_release()
1466 hem_list->root_ba = 0; in hns_roce_hem_list_release()
1473 INIT_LIST_HEAD(&hem_list->root_bt); in hns_roce_hem_list_init()
1474 INIT_LIST_HEAD(&hem_list->btm_bt); in hns_roce_hem_list_init()
1477 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); in hns_roce_hem_list_init()
1484 struct list_head *head = &hem_list->btm_bt; in hns_roce_hem_list_find_mtt()
1491 nr = offset - hem->start; in hns_roce_hem_list_find_mtt()
1492 cpu_base = hem->addr + nr * BA_BYTE_LEN; in hns_roce_hem_list_find_mtt()
1493 nr = hem->end + 1 - offset; in hns_roce_hem_list_find_mtt()