Lines Matching +full:cpu +full:- +full:capacity
1 // SPDX-License-Identifier: GPL-2.0
12 * objpool: ring-array based lockless MPMC/FIFO queues
24 void *obj = (void *)&slot->entries[pool->capacity]; in objpool_init_percpu_slot()
28 slot->mask = pool->capacity - 1; in objpool_init_percpu_slot()
36 slot->entries[slot->tail & slot->mask] = obj; in objpool_init_percpu_slot()
37 obj = obj + pool->obj_size; in objpool_init_percpu_slot()
38 slot->tail++; in objpool_init_percpu_slot()
39 slot->last = slot->tail; in objpool_init_percpu_slot()
40 pool->nr_objs++; in objpool_init_percpu_slot()
53 for (i = 0; i < pool->nr_cpus; i++) { in objpool_init_percpu_slots()
58 /* skip the cpu node which could never be present */ in objpool_init_percpu_slots()
68 size = struct_size(slot, entries, pool->capacity) + in objpool_init_percpu_slots()
69 pool->obj_size * nodes; in objpool_init_percpu_slots()
72 * here we allocate percpu-slot & objs together in a single in objpool_init_percpu_slots()
79 if (pool->gfp & GFP_ATOMIC) in objpool_init_percpu_slots()
80 slot = kmalloc_node(size, pool->gfp, cpu_to_node(i)); in objpool_init_percpu_slots()
82 slot = __vmalloc_node(size, sizeof(void *), pool->gfp, in objpool_init_percpu_slots()
85 return -ENOMEM; in objpool_init_percpu_slots()
87 pool->cpu_slots[i] = slot; in objpool_init_percpu_slots()
89 /* initialize the objpool_slot of cpu node i */ in objpool_init_percpu_slots()
103 if (!pool->cpu_slots) in objpool_fini_percpu_slots()
106 for (i = 0; i < pool->nr_cpus; i++) in objpool_fini_percpu_slots()
107 kvfree(pool->cpu_slots[i]); in objpool_fini_percpu_slots()
108 kfree(pool->cpu_slots); in objpool_fini_percpu_slots()
111 /* initialize object pool and pre-allocate objects */
116 int rc, capacity, slot_size; in objpool_init() local
121 return -EINVAL; in objpool_init()
126 /* calculate capacity of percpu objpool_slot */ in objpool_init()
127 capacity = roundup_pow_of_two(nr_objs); in objpool_init()
128 if (!capacity) in objpool_init()
129 return -EINVAL; in objpool_init()
133 pool->nr_cpus = nr_cpu_ids; in objpool_init()
134 pool->obj_size = object_size; in objpool_init()
135 pool->capacity = capacity; in objpool_init()
136 pool->gfp = gfp & ~__GFP_ZERO; in objpool_init()
137 pool->context = context; in objpool_init()
138 pool->release = release; in objpool_init()
139 slot_size = pool->nr_cpus * sizeof(struct objpool_slot); in objpool_init()
140 pool->cpu_slots = kzalloc(slot_size, pool->gfp); in objpool_init()
141 if (!pool->cpu_slots) in objpool_init()
142 return -ENOMEM; in objpool_init()
144 /* initialize per-cpu slots */ in objpool_init()
149 refcount_set(&pool->ref, pool->nr_objs + 1); in objpool_init()
157 objpool_try_add_slot(void *obj, struct objpool_head *pool, int cpu) in objpool_try_add_slot() argument
159 struct objpool_slot *slot = pool->cpu_slots[cpu]; in objpool_try_add_slot()
163 tail = READ_ONCE(slot->tail); in objpool_try_add_slot()
166 head = READ_ONCE(slot->head); in objpool_try_add_slot()
168 WARN_ON_ONCE(tail - head > pool->nr_objs); in objpool_try_add_slot()
169 } while (!try_cmpxchg_acquire(&slot->tail, &tail, tail + 1)); in objpool_try_add_slot()
172 WRITE_ONCE(slot->entries[tail & slot->mask], obj); in objpool_try_add_slot()
174 smp_store_release(&slot->last, tail + 1); in objpool_try_add_slot()
195 static inline void *objpool_try_get_slot(struct objpool_head *pool, int cpu) in objpool_try_get_slot() argument
197 struct objpool_slot *slot = pool->cpu_slots[cpu]; in objpool_try_get_slot()
199 uint32_t head = smp_load_acquire(&slot->head); in objpool_try_get_slot()
201 while (head != READ_ONCE(slot->last)) { in objpool_try_get_slot()
212 * by condition 'last != head && last - head <= nr_objs' in objpool_try_get_slot()
213 * that is equivalent to 'last - head - 1 < nr_objs' as in objpool_try_get_slot()
216 if (READ_ONCE(slot->last) - head - 1 >= pool->nr_objs) { in objpool_try_get_slot()
217 head = READ_ONCE(slot->head); in objpool_try_get_slot()
222 obj = READ_ONCE(slot->entries[head & slot->mask]); in objpool_try_get_slot()
225 if (try_cmpxchg_release(&slot->head, &head, head + 1)) in objpool_try_get_slot()
237 int i, cpu; in objpool_pop() local
242 cpu = raw_smp_processor_id(); in objpool_pop()
244 obj = objpool_try_get_slot(pool, cpu); in objpool_pop()
247 cpu = cpumask_next_wrap(cpu, cpu_possible_mask, -1, 1); in objpool_pop()
258 if (!pool->cpu_slots) in objpool_free()
265 if (pool->release) in objpool_free()
266 pool->release(pool, pool->context); in objpool_free()
274 return -EINVAL; in objpool_drop()
276 if (refcount_dec_and_test(&pool->ref)) { in objpool_drop()
281 return -EAGAIN; in objpool_drop()
294 if (refcount_sub_and_test(count, &pool->ref)) in objpool_fini()