15  *      - Redistributions of source code must retain the above
19  *      - Redistributions in binary form must reproduce the above
56 	int i;  in mlx4_free_icm_pages()  local
58 	if (chunk->nsg > 0)  in mlx4_free_icm_pages()
59 		pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,  in mlx4_free_icm_pages()
62 	for (i = 0; i < chunk->npages; ++i)  in mlx4_free_icm_pages()
63 		__free_pages(sg_page(&chunk->mem[i]),  in mlx4_free_icm_pages()
64 			     get_order(chunk->mem[i].length));  in mlx4_free_icm_pages()
69 	int i;  in mlx4_free_icm_coherent()  local
71 	for (i = 0; i < chunk->npages; ++i)  in mlx4_free_icm_coherent()
72 		dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length,  in mlx4_free_icm_coherent()
73 				  lowmem_page_address(sg_page(&chunk->mem[i])),  in mlx4_free_icm_coherent()
74 				  sg_dma_address(&chunk->mem[i]));  in mlx4_free_icm_coherent()
84 	list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {  in mlx4_free_icm()
102 		return -ENOMEM;  in mlx4_alloc_icm_pages()
114 		return -ENOMEM;  in mlx4_alloc_icm_coherent()
117 	BUG_ON(mem->offset);  in mlx4_alloc_icm_coherent()
137 	icm->refcount = 0;  in mlx4_alloc_icm()
138 	INIT_LIST_HEAD(&icm->chunk_list);  in mlx4_alloc_icm()
149 			sg_init_table(chunk->mem, MLX4_ICM_CHUNK_LEN);  in mlx4_alloc_icm()
150 			chunk->npages = 0;  in mlx4_alloc_icm()
151 			chunk->nsg    = 0;  in mlx4_alloc_icm()
152 			list_add_tail(&chunk->list, &icm->chunk_list);  in mlx4_alloc_icm()
156 			--cur_order;  in mlx4_alloc_icm()
159 			ret = mlx4_alloc_icm_coherent(&dev->pdev->dev,  in mlx4_alloc_icm()
160 						      &chunk->mem[chunk->npages],  in mlx4_alloc_icm()
163 			ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages],  in mlx4_alloc_icm()
167 			if (--cur_order < 0)  in mlx4_alloc_icm()
173 		++chunk->npages;  in mlx4_alloc_icm()
176 			++chunk->nsg;  in mlx4_alloc_icm()
177 		else if (chunk->npages == MLX4_ICM_CHUNK_LEN) {  in mlx4_alloc_icm()
178 			chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,  in mlx4_alloc_icm()
179 						chunk->npages,  in mlx4_alloc_icm()
182 			if (chunk->nsg <= 0)  in mlx4_alloc_icm()
186 		if (chunk->npages == MLX4_ICM_CHUNK_LEN)  in mlx4_alloc_icm()
189 		npages -= 1 << cur_order;  in mlx4_alloc_icm()
193 		chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,  in mlx4_alloc_icm()
194 					chunk->npages,  in mlx4_alloc_icm()
197 		if (chunk->nsg <= 0)  in mlx4_alloc_icm()
221 	return mlx4_map_cmd(dev, MLX4_CMD_MAP_ICM_AUX, icm, -1);  in mlx4_MAP_ICM_AUX()
232 	int i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size);  in mlx4_table_get()  local
235 	mutex_lock(&table->mutex);  in mlx4_table_get()
237 	if (table->icm[i]) {  in mlx4_table_get()
238 		++table->icm[i]->refcount;  in mlx4_table_get()
242 	table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT,  in mlx4_table_get()
243 				       (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |  in mlx4_table_get()
244 				       __GFP_NOWARN, table->coherent);  in mlx4_table_get()
245 	if (!table->icm[i]) {  in mlx4_table_get()
246 		ret = -ENOMEM;  in mlx4_table_get()
250 	if (mlx4_MAP_ICM(dev, table->icm[i], table->virt +  in mlx4_table_get()
251 			 (u64) i * MLX4_TABLE_CHUNK_SIZE)) {  in mlx4_table_get()
252 		mlx4_free_icm(dev, table->icm[i], table->coherent);  in mlx4_table_get()
253 		table->icm[i] = NULL;  in mlx4_table_get()
254 		ret = -ENOMEM;  in mlx4_table_get()
258 	++table->icm[i]->refcount;  in mlx4_table_get()
261 	mutex_unlock(&table->mutex);  in mlx4_table_get()
267 	int i;  in mlx4_table_put()  local
269 	i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size);  in mlx4_table_put()
271 	mutex_lock(&table->mutex);  in mlx4_table_put()
273 	if (--table->icm[i]->refcount == 0) {  in mlx4_table_put()
274 		mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,  in mlx4_table_put()
276 		mlx4_free_icm(dev, table->icm[i], table->coherent);  in mlx4_table_put()
277 		table->icm[i] = NULL;  in mlx4_table_put()
280 	mutex_unlock(&table->mutex);  in mlx4_table_put()
285 	int idx, offset, dma_offset, i;  in mlx4_table_find()  local
290 	if (!table->lowmem)  in mlx4_table_find()
293 	mutex_lock(&table->mutex);  in mlx4_table_find()
295 	idx = (obj & (table->num_obj - 1)) * table->obj_size;  in mlx4_table_find()
296 	icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE];  in mlx4_table_find()
302 	list_for_each_entry(chunk, &icm->chunk_list, list) {  in mlx4_table_find()
303 		for (i = 0; i < chunk->npages; ++i) {  in mlx4_table_find()
305 				if (sg_dma_len(&chunk->mem[i]) > dma_offset)  in mlx4_table_find()
306 					*dma_handle = sg_dma_address(&chunk->mem[i]) +  in mlx4_table_find()
308 				dma_offset -= sg_dma_len(&chunk->mem[i]);  in mlx4_table_find()
315 			if (chunk->mem[i].length > offset) {  in mlx4_table_find()
316 				page = sg_page(&chunk->mem[i]);  in mlx4_table_find()
319 			offset -= chunk->mem[i].length;  in mlx4_table_find()
324 	mutex_unlock(&table->mutex);  in mlx4_table_find()
331 	int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size;  in mlx4_table_get_range()
332 	int i, err;  in mlx4_table_get_range()  local
334 	for (i = start; i <= end; i += inc) {  in mlx4_table_get_range()
335 		err = mlx4_table_get(dev, table, i);  in mlx4_table_get_range()
343 	while (i > start) {  in mlx4_table_get_range()
344 		i -= inc;  in mlx4_table_get_range()
345 		mlx4_table_put(dev, table, i);  in mlx4_table_get_range()
354 	int i;  in mlx4_table_put_range()  local
356 	for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size)  in mlx4_table_put_range()
357 		mlx4_table_put(dev, table, i);  in mlx4_table_put_range()
367 	int i;  in mlx4_init_icm_table()  local
370 	num_icm = (nobj + obj_per_chunk - 1) / obj_per_chunk;  in mlx4_init_icm_table()
372 	table->icm      = kcalloc(num_icm, sizeof *table->icm, GFP_KERNEL);  in mlx4_init_icm_table()
373 	if (!table->icm)  in mlx4_init_icm_table()
374 		return -ENOMEM;  in mlx4_init_icm_table()
375 	table->virt     = virt;  in mlx4_init_icm_table()
376 	table->num_icm  = num_icm;  in mlx4_init_icm_table()
377 	table->num_obj  = nobj;  in mlx4_init_icm_table()
378 	table->obj_size = obj_size;  in mlx4_init_icm_table()
379 	table->lowmem   = use_lowmem;  in mlx4_init_icm_table()
380 	table->coherent = use_coherent;  in mlx4_init_icm_table()
381 	mutex_init(&table->mutex);  in mlx4_init_icm_table()
383 	for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {  in mlx4_init_icm_table()
385 		if ((i + 1) * MLX4_TABLE_CHUNK_SIZE > nobj * obj_size)  in mlx4_init_icm_table()
386 			chunk_size = PAGE_ALIGN(nobj * obj_size - i * MLX4_TABLE_CHUNK_SIZE);  in mlx4_init_icm_table()
388 		table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT,  in mlx4_init_icm_table()
391 		if (!table->icm[i])  in mlx4_init_icm_table()
393 		if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) {  in mlx4_init_icm_table()
394 			mlx4_free_icm(dev, table->icm[i], use_coherent);  in mlx4_init_icm_table()
395 			table->icm[i] = NULL;  in mlx4_init_icm_table()
403 		++table->icm[i]->refcount;  in mlx4_init_icm_table()
409 	for (i = 0; i < num_icm; ++i)  in mlx4_init_icm_table()
410 		if (table->icm[i]) {  in mlx4_init_icm_table()
411 			mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE,  in mlx4_init_icm_table()
413 			mlx4_free_icm(dev, table->icm[i], use_coherent);  in mlx4_init_icm_table()
416 	return -ENOMEM;  in mlx4_init_icm_table()
421 	int i;  in mlx4_cleanup_icm_table()  local
423 	for (i = 0; i < table->num_icm; ++i)  in mlx4_cleanup_icm_table()
424 		if (table->icm[i]) {  in mlx4_cleanup_icm_table()
425 			mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,  in mlx4_cleanup_icm_table()
427 			mlx4_free_icm(dev, table->icm[i], table->coherent);  in mlx4_cleanup_icm_table()
430 	kfree(table->icm);  in mlx4_cleanup_icm_table()