1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
3
4 #include "i40e.h"
5 #include "i40e_osdep.h"
6 #include "i40e_register.h"
7 #include "i40e_alloc.h"
8 #include "i40e_hmc.h"
9 #include "i40e_type.h"
10
11 /**
12 * i40e_add_sd_table_entry - Adds a segment descriptor to the table
13 * @hw: pointer to our hw struct
14 * @hmc_info: pointer to the HMC configuration information struct
15 * @sd_index: segment descriptor index to manipulate
16 * @type: what type of segment descriptor we're manipulating
17 * @direct_mode_sz: size to alloc in direct mode
18 **/
i40e_add_sd_table_entry(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 sd_index,enum i40e_sd_entry_type type,u64 direct_mode_sz)19 int i40e_add_sd_table_entry(struct i40e_hw *hw,
20 struct i40e_hmc_info *hmc_info,
21 u32 sd_index,
22 enum i40e_sd_entry_type type,
23 u64 direct_mode_sz)
24 {
25 enum i40e_memory_type mem_type __attribute__((unused));
26 struct i40e_hmc_sd_entry *sd_entry;
27 bool dma_mem_alloc_done = false;
28 struct i40e_dma_mem mem;
29 int ret_code = 0;
30 u64 alloc_len;
31
32 if (NULL == hmc_info->sd_table.sd_entry) {
33 ret_code = -EINVAL;
34 hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_entry\n");
35 goto exit;
36 }
37
38 if (sd_index >= hmc_info->sd_table.sd_cnt) {
39 ret_code = -EINVAL;
40 hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_index\n");
41 goto exit;
42 }
43
44 sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
45 if (!sd_entry->valid) {
46 if (I40E_SD_TYPE_PAGED == type) {
47 mem_type = i40e_mem_pd;
48 alloc_len = I40E_HMC_PAGED_BP_SIZE;
49 } else {
50 mem_type = i40e_mem_bp_jumbo;
51 alloc_len = direct_mode_sz;
52 }
53
54 /* allocate a 4K pd page or 2M backing page */
55 ret_code = i40e_allocate_dma_mem(hw, &mem, mem_type, alloc_len,
56 I40E_HMC_PD_BP_BUF_ALIGNMENT);
57 if (ret_code)
58 goto exit;
59 dma_mem_alloc_done = true;
60 if (I40E_SD_TYPE_PAGED == type) {
61 ret_code = i40e_allocate_virt_mem(hw,
62 &sd_entry->u.pd_table.pd_entry_virt_mem,
63 sizeof(struct i40e_hmc_pd_entry) * 512);
64 if (ret_code)
65 goto exit;
66 sd_entry->u.pd_table.pd_entry =
67 (struct i40e_hmc_pd_entry *)
68 sd_entry->u.pd_table.pd_entry_virt_mem.va;
69 sd_entry->u.pd_table.pd_page_addr = mem;
70 } else {
71 sd_entry->u.bp.addr = mem;
72 sd_entry->u.bp.sd_pd_index = sd_index;
73 }
74 /* initialize the sd entry */
75 hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
76
77 /* increment the ref count */
78 I40E_INC_SD_REFCNT(&hmc_info->sd_table);
79 }
80 /* Increment backing page reference count */
81 if (I40E_SD_TYPE_DIRECT == sd_entry->entry_type)
82 I40E_INC_BP_REFCNT(&sd_entry->u.bp);
83 exit:
84 if (ret_code)
85 if (dma_mem_alloc_done)
86 i40e_free_dma_mem(hw, &mem);
87
88 return ret_code;
89 }
90
91 /**
92 * i40e_add_pd_table_entry - Adds page descriptor to the specified table
93 * @hw: pointer to our HW structure
94 * @hmc_info: pointer to the HMC configuration information structure
95 * @pd_index: which page descriptor index to manipulate
96 * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
97 *
98 * This function:
99 * 1. Initializes the pd entry
100 * 2. Adds pd_entry in the pd_table
101 * 3. Mark the entry valid in i40e_hmc_pd_entry structure
102 * 4. Initializes the pd_entry's ref count to 1
103 * assumptions:
104 * 1. The memory for pd should be pinned down, physically contiguous and
105 * aligned on 4K boundary and zeroed memory.
106 * 2. It should be 4K in size.
107 **/
i40e_add_pd_table_entry(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 pd_index,struct i40e_dma_mem * rsrc_pg)108 int i40e_add_pd_table_entry(struct i40e_hw *hw,
109 struct i40e_hmc_info *hmc_info,
110 u32 pd_index,
111 struct i40e_dma_mem *rsrc_pg)
112 {
113 struct i40e_hmc_pd_table *pd_table;
114 struct i40e_hmc_pd_entry *pd_entry;
115 struct i40e_dma_mem mem;
116 struct i40e_dma_mem *page = &mem;
117 u32 sd_idx, rel_pd_idx;
118 int ret_code = 0;
119 u64 page_desc;
120 u64 *pd_addr;
121
122 if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) {
123 ret_code = -EINVAL;
124 hw_dbg(hw, "i40e_add_pd_table_entry: bad pd_index\n");
125 goto exit;
126 }
127
128 /* find corresponding sd */
129 sd_idx = (pd_index / I40E_HMC_PD_CNT_IN_SD);
130 if (I40E_SD_TYPE_PAGED !=
131 hmc_info->sd_table.sd_entry[sd_idx].entry_type)
132 goto exit;
133
134 rel_pd_idx = (pd_index % I40E_HMC_PD_CNT_IN_SD);
135 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
136 pd_entry = &pd_table->pd_entry[rel_pd_idx];
137 if (!pd_entry->valid) {
138 if (rsrc_pg) {
139 pd_entry->rsrc_pg = true;
140 page = rsrc_pg;
141 } else {
142 /* allocate a 4K backing page */
143 ret_code = i40e_allocate_dma_mem(hw, page, i40e_mem_bp,
144 I40E_HMC_PAGED_BP_SIZE,
145 I40E_HMC_PD_BP_BUF_ALIGNMENT);
146 if (ret_code)
147 goto exit;
148 pd_entry->rsrc_pg = false;
149 }
150
151 pd_entry->bp.addr = *page;
152 pd_entry->bp.sd_pd_index = pd_index;
153 pd_entry->bp.entry_type = I40E_SD_TYPE_PAGED;
154 /* Set page address and valid bit */
155 page_desc = page->pa | 0x1;
156
157 pd_addr = (u64 *)pd_table->pd_page_addr.va;
158 pd_addr += rel_pd_idx;
159
160 /* Add the backing page physical address in the pd entry */
161 memcpy(pd_addr, &page_desc, sizeof(u64));
162
163 pd_entry->sd_index = sd_idx;
164 pd_entry->valid = true;
165 I40E_INC_PD_REFCNT(pd_table);
166 }
167 I40E_INC_BP_REFCNT(&pd_entry->bp);
168 exit:
169 return ret_code;
170 }
171
172 /**
173 * i40e_remove_pd_bp - remove a backing page from a page descriptor
174 * @hw: pointer to our HW structure
175 * @hmc_info: pointer to the HMC configuration information structure
176 * @idx: the page index
177 *
178 * This function:
179 * 1. Marks the entry in pd tabe (for paged address mode) or in sd table
180 * (for direct address mode) invalid.
181 * 2. Write to register PMPDINV to invalidate the backing page in FV cache
182 * 3. Decrement the ref count for the pd _entry
183 * assumptions:
184 * 1. Caller can deallocate the memory used by backing storage after this
185 * function returns.
186 **/
i40e_remove_pd_bp(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 idx)187 int i40e_remove_pd_bp(struct i40e_hw *hw,
188 struct i40e_hmc_info *hmc_info,
189 u32 idx)
190 {
191 struct i40e_hmc_pd_entry *pd_entry;
192 struct i40e_hmc_pd_table *pd_table;
193 struct i40e_hmc_sd_entry *sd_entry;
194 u32 sd_idx, rel_pd_idx;
195 int ret_code = 0;
196 u64 *pd_addr;
197
198 /* calculate index */
199 sd_idx = idx / I40E_HMC_PD_CNT_IN_SD;
200 rel_pd_idx = idx % I40E_HMC_PD_CNT_IN_SD;
201 if (sd_idx >= hmc_info->sd_table.sd_cnt) {
202 ret_code = -EINVAL;
203 hw_dbg(hw, "i40e_remove_pd_bp: bad idx\n");
204 goto exit;
205 }
206 sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
207 if (I40E_SD_TYPE_PAGED != sd_entry->entry_type) {
208 ret_code = -EINVAL;
209 hw_dbg(hw, "i40e_remove_pd_bp: wrong sd_entry type\n");
210 goto exit;
211 }
212 /* get the entry and decrease its ref counter */
213 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
214 pd_entry = &pd_table->pd_entry[rel_pd_idx];
215 I40E_DEC_BP_REFCNT(&pd_entry->bp);
216 if (pd_entry->bp.ref_cnt)
217 goto exit;
218
219 /* mark the entry invalid */
220 pd_entry->valid = false;
221 I40E_DEC_PD_REFCNT(pd_table);
222 pd_addr = (u64 *)pd_table->pd_page_addr.va;
223 pd_addr += rel_pd_idx;
224 memset(pd_addr, 0, sizeof(u64));
225 I40E_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx);
226
227 /* free memory here */
228 if (!pd_entry->rsrc_pg)
229 ret_code = i40e_free_dma_mem(hw, &pd_entry->bp.addr);
230 if (ret_code)
231 goto exit;
232 if (!pd_table->ref_cnt)
233 i40e_free_virt_mem(hw, &pd_table->pd_entry_virt_mem);
234 exit:
235 return ret_code;
236 }
237
238 /**
239 * i40e_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
240 * @hmc_info: pointer to the HMC configuration information structure
241 * @idx: the page index
242 **/
i40e_prep_remove_sd_bp(struct i40e_hmc_info * hmc_info,u32 idx)243 int i40e_prep_remove_sd_bp(struct i40e_hmc_info *hmc_info,
244 u32 idx)
245 {
246 struct i40e_hmc_sd_entry *sd_entry;
247 int ret_code = 0;
248
249 /* get the entry and decrease its ref counter */
250 sd_entry = &hmc_info->sd_table.sd_entry[idx];
251 I40E_DEC_BP_REFCNT(&sd_entry->u.bp);
252 if (sd_entry->u.bp.ref_cnt) {
253 ret_code = -EBUSY;
254 goto exit;
255 }
256 I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
257
258 /* mark the entry invalid */
259 sd_entry->valid = false;
260 exit:
261 return ret_code;
262 }
263
264 /**
265 * i40e_remove_sd_bp_new - Removes a backing page from a segment descriptor
266 * @hw: pointer to our hw struct
267 * @hmc_info: pointer to the HMC configuration information structure
268 * @idx: the page index
269 * @is_pf: used to distinguish between VF and PF
270 **/
i40e_remove_sd_bp_new(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 idx,bool is_pf)271 int i40e_remove_sd_bp_new(struct i40e_hw *hw,
272 struct i40e_hmc_info *hmc_info,
273 u32 idx, bool is_pf)
274 {
275 struct i40e_hmc_sd_entry *sd_entry;
276
277 if (!is_pf)
278 return -EOPNOTSUPP;
279
280 /* get the entry and decrease its ref counter */
281 sd_entry = &hmc_info->sd_table.sd_entry[idx];
282 I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_DIRECT);
283
284 return i40e_free_dma_mem(hw, &sd_entry->u.bp.addr);
285 }
286
287 /**
288 * i40e_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
289 * @hmc_info: pointer to the HMC configuration information structure
290 * @idx: segment descriptor index to find the relevant page descriptor
291 **/
i40e_prep_remove_pd_page(struct i40e_hmc_info * hmc_info,u32 idx)292 int i40e_prep_remove_pd_page(struct i40e_hmc_info *hmc_info,
293 u32 idx)
294 {
295 struct i40e_hmc_sd_entry *sd_entry;
296 int ret_code = 0;
297
298 sd_entry = &hmc_info->sd_table.sd_entry[idx];
299
300 if (sd_entry->u.pd_table.ref_cnt) {
301 ret_code = -EBUSY;
302 goto exit;
303 }
304
305 /* mark the entry invalid */
306 sd_entry->valid = false;
307
308 I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
309 exit:
310 return ret_code;
311 }
312
313 /**
314 * i40e_remove_pd_page_new - Removes a PD page from sd entry.
315 * @hw: pointer to our hw struct
316 * @hmc_info: pointer to the HMC configuration information structure
317 * @idx: segment descriptor index to find the relevant page descriptor
318 * @is_pf: used to distinguish between VF and PF
319 **/
i40e_remove_pd_page_new(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 idx,bool is_pf)320 int i40e_remove_pd_page_new(struct i40e_hw *hw,
321 struct i40e_hmc_info *hmc_info,
322 u32 idx, bool is_pf)
323 {
324 struct i40e_hmc_sd_entry *sd_entry;
325
326 if (!is_pf)
327 return -EOPNOTSUPP;
328
329 sd_entry = &hmc_info->sd_table.sd_entry[idx];
330 I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_PAGED);
331
332 return i40e_free_dma_mem(hw, &sd_entry->u.pd_table.pd_page_addr);
333 }
334