16  *      - Redistributions of source code must retain the above
20  *      - Redistributions in binary form must reproduce the above
66 	int i;  in mthca_free_icm_pages()  local
68 	if (chunk->nsg > 0)  in mthca_free_icm_pages()
69 		pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,  in mthca_free_icm_pages()
72 	for (i = 0; i < chunk->npages; ++i)  in mthca_free_icm_pages()
73 		__free_pages(sg_page(&chunk->mem[i]),  in mthca_free_icm_pages()
74 			     get_order(chunk->mem[i].length));  in mthca_free_icm_pages()
79 	int i;  in mthca_free_icm_coherent()  local
81 	for (i = 0; i < chunk->npages; ++i) {  in mthca_free_icm_coherent()
82 		dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length,  in mthca_free_icm_coherent()
83 				  lowmem_page_address(sg_page(&chunk->mem[i])),  in mthca_free_icm_coherent()
84 				  sg_dma_address(&chunk->mem[i]));  in mthca_free_icm_coherent()
95 	list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {  in mthca_free_icm()
117 		return -ENOMEM;  in mthca_alloc_icm_pages()
129 		return -ENOMEM;  in mthca_alloc_icm_coherent()
132 	BUG_ON(mem->offset);  in mthca_alloc_icm_coherent()
152 	icm->refcount = 0;  in mthca_alloc_icm()
153 	INIT_LIST_HEAD(&icm->chunk_list);  in mthca_alloc_icm()
164 			sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN);  in mthca_alloc_icm()
165 			chunk->npages = 0;  in mthca_alloc_icm()
166 			chunk->nsg    = 0;  in mthca_alloc_icm()
167 			list_add_tail(&chunk->list, &icm->chunk_list);  in mthca_alloc_icm()
171 			--cur_order;  in mthca_alloc_icm()
174 			ret = mthca_alloc_icm_coherent(&dev->pdev->dev,  in mthca_alloc_icm()
175 						       &chunk->mem[chunk->npages],  in mthca_alloc_icm()
178 			ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages],  in mthca_alloc_icm()
182 			++chunk->npages;  in mthca_alloc_icm()
185 				++chunk->nsg;  in mthca_alloc_icm()
186 			else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) {  in mthca_alloc_icm()
187 				chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,  in mthca_alloc_icm()
188 							chunk->npages,  in mthca_alloc_icm()
191 				if (chunk->nsg <= 0)  in mthca_alloc_icm()
195 			if (chunk->npages == MTHCA_ICM_CHUNK_LEN)  in mthca_alloc_icm()
198 			npages -= 1 << cur_order;  in mthca_alloc_icm()
200 			--cur_order;  in mthca_alloc_icm()
207 		chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,  in mthca_alloc_icm()
208 					chunk->npages,  in mthca_alloc_icm()
211 		if (chunk->nsg <= 0)  in mthca_alloc_icm()
224 	int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;  in mthca_table_get()  local
227 	mutex_lock(&table->mutex);  in mthca_table_get()
229 	if (table->icm[i]) {  in mthca_table_get()
230 		++table->icm[i]->refcount;  in mthca_table_get()
234 	table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT,  in mthca_table_get()
235 					(table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |  in mthca_table_get()
236 					__GFP_NOWARN, table->coherent);  in mthca_table_get()
237 	if (!table->icm[i]) {  in mthca_table_get()
238 		ret = -ENOMEM;  in mthca_table_get()
242 	if (mthca_MAP_ICM(dev, table->icm[i],  in mthca_table_get()
243 			  table->virt + i * MTHCA_TABLE_CHUNK_SIZE)) {  in mthca_table_get()
244 		mthca_free_icm(dev, table->icm[i], table->coherent);  in mthca_table_get()
245 		table->icm[i] = NULL;  in mthca_table_get()
246 		ret = -ENOMEM;  in mthca_table_get()
250 	++table->icm[i]->refcount;  in mthca_table_get()
253 	mutex_unlock(&table->mutex);  in mthca_table_get()
259 	int i;  in mthca_table_put()  local
264 	i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;  in mthca_table_put()
266 	mutex_lock(&table->mutex);  in mthca_table_put()
268 	if (--table->icm[i]->refcount == 0) {  in mthca_table_put()
269 		mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE,  in mthca_table_put()
271 		mthca_free_icm(dev, table->icm[i], table->coherent);  in mthca_table_put()
272 		table->icm[i] = NULL;  in mthca_table_put()
275 	mutex_unlock(&table->mutex);  in mthca_table_put()
280 	int idx, offset, dma_offset, i;  in mthca_table_find()  local
285 	if (!table->lowmem)  in mthca_table_find()
288 	mutex_lock(&table->mutex);  in mthca_table_find()
290 	idx = (obj & (table->num_obj - 1)) * table->obj_size;  in mthca_table_find()
291 	icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE];  in mthca_table_find()
297 	list_for_each_entry(chunk, &icm->chunk_list, list) {  in mthca_table_find()
298 		for (i = 0; i < chunk->npages; ++i) {  in mthca_table_find()
300 				if (sg_dma_len(&chunk->mem[i]) > dma_offset)  in mthca_table_find()
301 					*dma_handle = sg_dma_address(&chunk->mem[i]) +  in mthca_table_find()
303 				dma_offset -= sg_dma_len(&chunk->mem[i]);  in mthca_table_find()
308 			if (chunk->mem[i].length > offset) {  in mthca_table_find()
309 				page = sg_page(&chunk->mem[i]);  in mthca_table_find()
312 			offset -= chunk->mem[i].length;  in mthca_table_find()
317 	mutex_unlock(&table->mutex);  in mthca_table_find()
324 	int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size;  in mthca_table_get_range()
325 	int i, err;  in mthca_table_get_range()  local
327 	for (i = start; i <= end; i += inc) {  in mthca_table_get_range()
328 		err = mthca_table_get(dev, table, i);  in mthca_table_get_range()
336 	while (i > start) {  in mthca_table_get_range()
337 		i -= inc;  in mthca_table_get_range()
338 		mthca_table_put(dev, table, i);  in mthca_table_get_range()
347 	int i;  in mthca_table_put_range()  local
352 	for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size)  in mthca_table_put_range()
353 		mthca_table_put(dev, table, i);  in mthca_table_put_range()
365 	int i;  in mthca_alloc_icm_table()  local
370 	table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL);  in mthca_alloc_icm_table()
374 	table->virt     = virt;  in mthca_alloc_icm_table()
375 	table->num_icm  = num_icm;  in mthca_alloc_icm_table()
376 	table->num_obj  = nobj;  in mthca_alloc_icm_table()
377 	table->obj_size = obj_size;  in mthca_alloc_icm_table()
378 	table->lowmem   = use_lowmem;  in mthca_alloc_icm_table()
379 	table->coherent = use_coherent;  in mthca_alloc_icm_table()
380 	mutex_init(&table->mutex);  in mthca_alloc_icm_table()
382 	for (i = 0; i < num_icm; ++i)  in mthca_alloc_icm_table()
383 		table->icm[i] = NULL;  in mthca_alloc_icm_table()
385 	for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {  in mthca_alloc_icm_table()
387 		if ((i + 1) * MTHCA_TABLE_CHUNK_SIZE > nobj * obj_size)  in mthca_alloc_icm_table()
388 			chunk_size = nobj * obj_size - i * MTHCA_TABLE_CHUNK_SIZE;  in mthca_alloc_icm_table()
390 		table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT,  in mthca_alloc_icm_table()
393 		if (!table->icm[i])  in mthca_alloc_icm_table()
395 		if (mthca_MAP_ICM(dev, table->icm[i],  in mthca_alloc_icm_table()
396 				  virt + i * MTHCA_TABLE_CHUNK_SIZE)) {  in mthca_alloc_icm_table()
397 			mthca_free_icm(dev, table->icm[i], table->coherent);  in mthca_alloc_icm_table()
398 			table->icm[i] = NULL;  in mthca_alloc_icm_table()
406 		++table->icm[i]->refcount;  in mthca_alloc_icm_table()
412 	for (i = 0; i < num_icm; ++i)  in mthca_alloc_icm_table()
413 		if (table->icm[i]) {  in mthca_alloc_icm_table()
414 			mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE,  in mthca_alloc_icm_table()
416 			mthca_free_icm(dev, table->icm[i], table->coherent);  in mthca_alloc_icm_table()
426 	int i;  in mthca_free_icm_table()  local
428 	for (i = 0; i < table->num_icm; ++i)  in mthca_free_icm_table()
429 		if (table->icm[i]) {  in mthca_free_icm_table()
431 					table->virt + i * MTHCA_TABLE_CHUNK_SIZE,  in mthca_free_icm_table()
433 			mthca_free_icm(dev, table->icm[i], table->coherent);  in mthca_free_icm_table()
441 	return dev->uar_table.uarc_base +  in mthca_uarc_virt()
442 		uar->index * dev->uar_table.uarc_size +  in mthca_uarc_virt()
451 	int i;  in mthca_map_user_db()  local
456 	if (index < 0 || index > dev->uar_table.uarc_size / 8)  in mthca_map_user_db()
457 		return -EINVAL;  in mthca_map_user_db()
459 	mutex_lock(&db_tab->mutex);  in mthca_map_user_db()
461 	i = index / MTHCA_DB_REC_PER_PAGE;  in mthca_map_user_db()
463 	if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE)       ||  in mthca_map_user_db()
464 	    (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) ||  in mthca_map_user_db()
466 		ret = -EINVAL;  in mthca_map_user_db()
470 	if (db_tab->page[i].refcount) {  in mthca_map_user_db()
471 		++db_tab->page[i].refcount;  in mthca_map_user_db()
475 	ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0,  in mthca_map_user_db()
480 	sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE,  in mthca_map_user_db()
483 	ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);  in mthca_map_user_db()
489 	ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem),  in mthca_map_user_db()
490 				 mthca_uarc_virt(dev, uar, i));  in mthca_map_user_db()
492 		pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);  in mthca_map_user_db()
493 		put_page(sg_page(&db_tab->page[i].mem));  in mthca_map_user_db()
497 	db_tab->page[i].uvirt    = uaddr;  in mthca_map_user_db()
498 	db_tab->page[i].refcount = 1;  in mthca_map_user_db()
501 	mutex_unlock(&db_tab->mutex);  in mthca_map_user_db()
516 	mutex_lock(&db_tab->mutex);  in mthca_unmap_user_db()
518 	--db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount;  in mthca_unmap_user_db()
520 	mutex_unlock(&db_tab->mutex);  in mthca_unmap_user_db()
527 	int i;  in mthca_init_user_db_tab()  local
532 	npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE;  in mthca_init_user_db_tab()
533 	db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL);  in mthca_init_user_db_tab()
535 		return ERR_PTR(-ENOMEM);  in mthca_init_user_db_tab()
537 	mutex_init(&db_tab->mutex);  in mthca_init_user_db_tab()
538 	for (i = 0; i < npages; ++i) {  in mthca_init_user_db_tab()
539 		db_tab->page[i].refcount = 0;  in mthca_init_user_db_tab()
540 		db_tab->page[i].uvirt    = 0;  in mthca_init_user_db_tab()
541 		sg_init_table(&db_tab->page[i].mem, 1);  in mthca_init_user_db_tab()
550 	int i;  in mthca_cleanup_user_db_tab()  local
555 	for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) {  in mthca_cleanup_user_db_tab()
556 		if (db_tab->page[i].uvirt) {  in mthca_cleanup_user_db_tab()
557 			mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1);  in mthca_cleanup_user_db_tab()
558 			pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);  in mthca_cleanup_user_db_tab()
559 			put_page(sg_page(&db_tab->page[i].mem));  in mthca_cleanup_user_db_tab()
571 	int i, j;  in mthca_alloc_db()  local
575 	mutex_lock(&dev->db_tab->mutex);  in mthca_alloc_db()
582 		end   = dev->db_tab->max_group1;  in mthca_alloc_db()
590 		start = dev->db_tab->npages - 1;  in mthca_alloc_db()
591 		end   = dev->db_tab->min_group2;  in mthca_alloc_db()
592 		dir   = -1;  in mthca_alloc_db()
596 		ret = -EINVAL;  in mthca_alloc_db()
600 	for (i = start; i != end; i += dir)  in mthca_alloc_db()
601 		if (dev->db_tab->page[i].db_rec &&  in mthca_alloc_db()
602 		    !bitmap_full(dev->db_tab->page[i].used,  in mthca_alloc_db()
604 			page = dev->db_tab->page + i;  in mthca_alloc_db()
608 	for (i = start; i != end; i += dir)  in mthca_alloc_db()
609 		if (!dev->db_tab->page[i].db_rec) {  in mthca_alloc_db()
610 			page = dev->db_tab->page + i;  in mthca_alloc_db()
614 	if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) {  in mthca_alloc_db()
615 		ret = -ENOMEM;  in mthca_alloc_db()
620 		++dev->db_tab->max_group1;  in mthca_alloc_db()
622 		--dev->db_tab->min_group2;  in mthca_alloc_db()
624 	page = dev->db_tab->page + end;  in mthca_alloc_db()
627 	page->db_rec = dma_alloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,  in mthca_alloc_db()
628 					  &page->mapping, GFP_KERNEL);  in mthca_alloc_db()
629 	if (!page->db_rec) {  in mthca_alloc_db()
630 		ret = -ENOMEM;  in mthca_alloc_db()
633 	memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE);  in mthca_alloc_db()
635 	ret = mthca_MAP_ICM_page(dev, page->mapping,  in mthca_alloc_db()
636 				 mthca_uarc_virt(dev, &dev->driver_uar, i));  in mthca_alloc_db()
638 		dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,  in mthca_alloc_db()
639 				  page->db_rec, page->mapping);  in mthca_alloc_db()
643 	bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE);  in mthca_alloc_db()
646 	j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE);  in mthca_alloc_db()
647 	set_bit(j, page->used);  in mthca_alloc_db()
650 		j = MTHCA_DB_REC_PER_PAGE - 1 - j;  in mthca_alloc_db()
652 	ret = i * MTHCA_DB_REC_PER_PAGE + j;  in mthca_alloc_db()
654 	page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5));  in mthca_alloc_db()
656 	*db = (__be32 *) &page->db_rec[j];  in mthca_alloc_db()
659 	mutex_unlock(&dev->db_tab->mutex);  in mthca_alloc_db()
666 	int i, j;  in mthca_free_db()  local
669 	i = db_index / MTHCA_DB_REC_PER_PAGE;  in mthca_free_db()
672 	page = dev->db_tab->page + i;  in mthca_free_db()
674 	mutex_lock(&dev->db_tab->mutex);  in mthca_free_db()
676 	page->db_rec[j] = 0;  in mthca_free_db()
677 	if (i >= dev->db_tab->min_group2)  in mthca_free_db()
678 		j = MTHCA_DB_REC_PER_PAGE - 1 - j;  in mthca_free_db()
679 	clear_bit(j, page->used);  in mthca_free_db()
681 	if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) &&  in mthca_free_db()
682 	    i >= dev->db_tab->max_group1 - 1) {  in mthca_free_db()
683 		mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1);  in mthca_free_db()
685 		dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,  in mthca_free_db()
686 				  page->db_rec, page->mapping);  in mthca_free_db()
687 		page->db_rec = NULL;  in mthca_free_db()
689 		if (i == dev->db_tab->max_group1) {  in mthca_free_db()
690 			--dev->db_tab->max_group1;  in mthca_free_db()
693 		if (i == dev->db_tab->min_group2)  in mthca_free_db()
694 			++dev->db_tab->min_group2;  in mthca_free_db()
697 	mutex_unlock(&dev->db_tab->mutex);  in mthca_free_db()
702 	int i;  in mthca_init_db_tab()  local
707 	dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL);  in mthca_init_db_tab()
708 	if (!dev->db_tab)  in mthca_init_db_tab()
709 		return -ENOMEM;  in mthca_init_db_tab()
711 	mutex_init(&dev->db_tab->mutex);  in mthca_init_db_tab()
713 	dev->db_tab->npages     = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE;  in mthca_init_db_tab()
714 	dev->db_tab->max_group1 = 0;  in mthca_init_db_tab()
715 	dev->db_tab->min_group2 = dev->db_tab->npages - 1;  in mthca_init_db_tab()
717 	dev->db_tab->page = kmalloc(dev->db_tab->npages *  in mthca_init_db_tab()
718 				    sizeof *dev->db_tab->page,  in mthca_init_db_tab()
720 	if (!dev->db_tab->page) {  in mthca_init_db_tab()
721 		kfree(dev->db_tab);  in mthca_init_db_tab()
722 		return -ENOMEM;  in mthca_init_db_tab()
725 	for (i = 0; i < dev->db_tab->npages; ++i)  in mthca_init_db_tab()
726 		dev->db_tab->page[i].db_rec = NULL;  in mthca_init_db_tab()
733 	int i;  in mthca_cleanup_db_tab()  local
744 	for (i = 0; i < dev->db_tab->npages; ++i) {  in mthca_cleanup_db_tab()
745 		if (!dev->db_tab->page[i].db_rec)  in mthca_cleanup_db_tab()
748 		if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE))  in mthca_cleanup_db_tab()
749 			mthca_warn(dev, "Kernel UARC page %d not empty\n", i);  in mthca_cleanup_db_tab()
751 		mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1);  in mthca_cleanup_db_tab()
753 		dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,  in mthca_cleanup_db_tab()
754 				  dev->db_tab->page[i].db_rec,  in mthca_cleanup_db_tab()
755 				  dev->db_tab->page[i].mapping);  in mthca_cleanup_db_tab()
758 	kfree(dev->db_tab->page);  in mthca_cleanup_db_tab()
759 	kfree(dev->db_tab);  in mthca_cleanup_db_tab()