1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2023 Intel Corporation */
3
4 #include <linux/dev_printk.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/export.h>
7 #include <linux/kernel.h>
8 #include <linux/kstrtox.h>
9 #include <linux/overflow.h>
10 #include <linux/string.h>
11 #include <linux/slab.h>
12 #include <linux/types.h>
13 #include <asm/errno.h>
14 #include "adf_accel_devices.h"
15 #include "adf_admin.h"
16 #include "adf_cfg.h"
17 #include "adf_cfg_strings.h"
18 #include "adf_clock.h"
19 #include "adf_common_drv.h"
20 #include "adf_heartbeat.h"
21 #include "adf_transport_internal.h"
22 #include "icp_qat_fw_init_admin.h"
23
24 #define ADF_HB_EMPTY_SIG 0xA5A5A5A5
25
26 /* Heartbeat counter pair */
27 struct hb_cnt_pair {
28 __u16 resp_heartbeat_cnt;
29 __u16 req_heartbeat_cnt;
30 };
31
adf_hb_check_polling_freq(struct adf_accel_dev * accel_dev)32 static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev)
33 {
34 u64 curr_time = adf_clock_get_current_time();
35 u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time;
36
37 if (polling_time < accel_dev->heartbeat->hb_timer) {
38 dev_warn(&GET_DEV(accel_dev),
39 "HB polling too frequent. Configured HB timer %d ms\n",
40 accel_dev->heartbeat->hb_timer);
41 return -EINVAL;
42 }
43
44 accel_dev->heartbeat->last_hb_check_time = curr_time;
45 return 0;
46 }
47
48 /**
49 * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should
50 * be updated by one to support the currently loaded firmware.
51 * @accel_dev: Pointer to acceleration device.
52 *
53 * Return:
54 * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND
55 * * false - no changes needed
56 */
validate_hb_ctrs_cnt(struct adf_accel_dev * accel_dev)57 static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev)
58 {
59 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
60 const size_t max_aes = accel_dev->hw_device->num_engines;
61 const size_t hb_struct_size = sizeof(struct hb_cnt_pair);
62 const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes,
63 hb_struct_size);
64 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
65 const size_t stats_size = size_mul(dev_ctrs, hb_struct_size);
66 const u32 exp_diff_cnt = exp_diff_size / sizeof(u32);
67 const u32 stats_el_cnt = stats_size / sizeof(u32);
68 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
69 const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs);
70 u32 el_diff_cnt = 0;
71 int i;
72
73 /* count how many bytes are different from pattern */
74 for (i = 0; i < stats_el_cnt; i++) {
75 if (mem_to_chk[i] == ADF_HB_EMPTY_SIG)
76 break;
77
78 el_diff_cnt++;
79 }
80
81 return el_diff_cnt && el_diff_cnt == exp_diff_cnt;
82 }
83
adf_heartbeat_check_ctrs(struct adf_accel_dev * accel_dev)84 void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev)
85 {
86 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
87 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
88 const size_t max_aes = accel_dev->hw_device->num_engines;
89 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
90 const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair));
91 const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32);
92
93 /* fill hb stats memory with pattern */
94 memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill);
95 accel_dev->heartbeat->ctrs_cnt_checked = false;
96 }
97 EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs);
98
get_timer_ticks(struct adf_accel_dev * accel_dev,unsigned int * value)99 static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value)
100 {
101 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { };
102 u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS;
103 int cfg_read_status;
104 u32 ticks;
105 int ret;
106
107 cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
108 ADF_HEARTBEAT_TIMER, timer_str);
109 if (cfg_read_status == 0) {
110 if (kstrtouint(timer_str, 10, &timer_ms))
111 dev_dbg(&GET_DEV(accel_dev),
112 "kstrtouint failed to parse the %s, param value",
113 ADF_HEARTBEAT_TIMER);
114 }
115
116 if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) {
117 dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n",
118 ADF_CFG_HB_TIMER_MIN_MS);
119 return -EINVAL;
120 }
121
122 /*
123 * On 4xxx devices adf_timer is responsible for HB updates and
124 * its period is fixed to 200ms
125 */
126 if (accel_dev->timer)
127 timer_ms = ADF_CFG_HB_TIMER_MIN_MS;
128
129 ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks);
130 if (ret)
131 return ret;
132
133 adf_heartbeat_save_cfg_param(accel_dev, timer_ms);
134
135 accel_dev->heartbeat->hb_timer = timer_ms;
136 *value = ticks;
137
138 return 0;
139 }
140
check_ae(struct hb_cnt_pair * curr,struct hb_cnt_pair * prev,u16 * count,const size_t hb_ctrs)141 static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev,
142 u16 *count, const size_t hb_ctrs)
143 {
144 size_t thr;
145
146 /* loop through all threads in AE */
147 for (thr = 0; thr < hb_ctrs; thr++) {
148 u16 req = curr[thr].req_heartbeat_cnt;
149 u16 resp = curr[thr].resp_heartbeat_cnt;
150 u16 last = prev[thr].resp_heartbeat_cnt;
151
152 if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) {
153 u16 retry = ++count[thr];
154
155 if (retry >= ADF_CFG_HB_COUNT_THRESHOLD)
156 return -EIO;
157
158 } else {
159 count[thr] = 0;
160 }
161 }
162 return 0;
163 }
164
adf_hb_get_status(struct adf_accel_dev * accel_dev)165 static int adf_hb_get_status(struct adf_accel_dev *accel_dev)
166 {
167 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
168 struct hb_cnt_pair *live_stats, *last_stats, *curr_stats;
169 const size_t hb_ctrs = hw_device->num_hb_ctrs;
170 const unsigned long ae_mask = hw_device->ae_mask;
171 const size_t max_aes = hw_device->num_engines;
172 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
173 const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats));
174 struct hb_cnt_pair *ae_curr_p, *ae_prev_p;
175 u16 *count_fails, *ae_count_p;
176 size_t ae_offset;
177 size_t ae = 0;
178 int ret = 0;
179
180 if (!accel_dev->heartbeat->ctrs_cnt_checked) {
181 if (validate_hb_ctrs_cnt(accel_dev))
182 hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND;
183
184 accel_dev->heartbeat->ctrs_cnt_checked = true;
185 }
186
187 live_stats = accel_dev->heartbeat->dma.virt_addr;
188 last_stats = live_stats + dev_ctrs;
189 count_fails = (u16 *)(last_stats + dev_ctrs);
190
191 curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL);
192 if (!curr_stats)
193 return -ENOMEM;
194
195 /* loop through active AEs */
196 for_each_set_bit(ae, &ae_mask, max_aes) {
197 ae_offset = size_mul(ae, hb_ctrs);
198 ae_curr_p = curr_stats + ae_offset;
199 ae_prev_p = last_stats + ae_offset;
200 ae_count_p = count_fails + ae_offset;
201
202 ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs);
203 if (ret)
204 break;
205 }
206
207 /* Copy current stats for the next iteration */
208 memcpy(last_stats, curr_stats, stats_size);
209 kfree(curr_stats);
210
211 return ret;
212 }
213
adf_heartbeat_status(struct adf_accel_dev * accel_dev,enum adf_device_heartbeat_status * hb_status)214 void adf_heartbeat_status(struct adf_accel_dev *accel_dev,
215 enum adf_device_heartbeat_status *hb_status)
216 {
217 struct adf_heartbeat *hb;
218
219 if (!adf_dev_started(accel_dev) ||
220 test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
221 *hb_status = HB_DEV_UNRESPONSIVE;
222 return;
223 }
224
225 if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) {
226 *hb_status = HB_DEV_UNSUPPORTED;
227 return;
228 }
229
230 hb = accel_dev->heartbeat;
231 hb->hb_sent_counter++;
232
233 if (adf_hb_get_status(accel_dev)) {
234 dev_err(&GET_DEV(accel_dev),
235 "Heartbeat ERROR: QAT is not responding.\n");
236 *hb_status = HB_DEV_UNRESPONSIVE;
237 hb->hb_failed_counter++;
238 return;
239 }
240
241 *hb_status = HB_DEV_ALIVE;
242 }
243
adf_heartbeat_ms_to_ticks(struct adf_accel_dev * accel_dev,unsigned int time_ms,u32 * value)244 int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms,
245 u32 *value)
246 {
247 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
248 u32 clk_per_sec;
249
250 /* HB clock may be different than AE clock */
251 if (!hw_data->get_hb_clock)
252 return -EINVAL;
253
254 clk_per_sec = hw_data->get_hb_clock(hw_data);
255 *value = time_ms * (clk_per_sec / MSEC_PER_SEC);
256
257 return 0;
258 }
259
adf_heartbeat_save_cfg_param(struct adf_accel_dev * accel_dev,unsigned int timer_ms)260 int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev,
261 unsigned int timer_ms)
262 {
263 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
264
265 snprintf(timer_str, sizeof(timer_str), "%u", timer_ms);
266 return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
267 ADF_HEARTBEAT_TIMER, timer_str,
268 ADF_STR);
269 }
270 EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param);
271
adf_heartbeat_init(struct adf_accel_dev * accel_dev)272 int adf_heartbeat_init(struct adf_accel_dev *accel_dev)
273 {
274 struct adf_heartbeat *hb;
275
276 hb = kzalloc(sizeof(*hb), GFP_KERNEL);
277 if (!hb)
278 goto err_ret;
279
280 hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
281 &hb->dma.phy_addr, GFP_KERNEL);
282 if (!hb->dma.virt_addr)
283 goto err_free;
284
285 /*
286 * Default set this flag as true to avoid unnecessary checks,
287 * it will be reset on platforms that need such a check
288 */
289 hb->ctrs_cnt_checked = true;
290 accel_dev->heartbeat = hb;
291
292 return 0;
293
294 err_free:
295 kfree(hb);
296 err_ret:
297 return -ENOMEM;
298 }
299
adf_heartbeat_start(struct adf_accel_dev * accel_dev)300 int adf_heartbeat_start(struct adf_accel_dev *accel_dev)
301 {
302 unsigned int timer_ticks;
303 int ret;
304
305 if (!accel_dev->heartbeat) {
306 dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!");
307 return -EFAULT;
308 }
309
310 if (accel_dev->hw_device->check_hb_ctrs)
311 accel_dev->hw_device->check_hb_ctrs(accel_dev);
312
313 ret = get_timer_ticks(accel_dev, &timer_ticks);
314 if (ret)
315 return ret;
316
317 ret = adf_send_admin_hb_timer(accel_dev, timer_ticks);
318 if (ret)
319 dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!");
320
321 return ret;
322 }
323
adf_heartbeat_shutdown(struct adf_accel_dev * accel_dev)324 void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev)
325 {
326 struct adf_heartbeat *hb = accel_dev->heartbeat;
327
328 if (!hb)
329 return;
330
331 if (hb->dma.virt_addr)
332 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
333 hb->dma.virt_addr, hb->dma.phy_addr);
334
335 kfree(hb);
336 accel_dev->heartbeat = NULL;
337 }
338