1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2025 Intel Corporation */
3
4 #define pr_fmt(fmt) "QAT: " fmt
5
6 #include <linux/bits.h>
7 #include <linux/dev_printk.h>
8 #include <linux/printk.h>
9 #include "adf_accel_devices.h"
10 #include "adf_bank_state.h"
11 #include "adf_common_drv.h"
12
13 /* Ring interrupt masks */
14 #define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0)
15 #define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0)
16 #define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4
17
check_stat(u32 (* op)(void __iomem *,u32),u32 expect_val,const char * name,void __iomem * base,u32 bank)18 static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val,
19 const char *name, void __iomem *base, u32 bank)
20 {
21 u32 actual_val = op(base, bank);
22
23 if (expect_val == actual_val)
24 return 0;
25
26 pr_err("Fail to restore %s register. Expected %#x, actual %#x\n",
27 name, expect_val, actual_val);
28
29 return -EINVAL;
30 }
31
bank_state_save(struct adf_hw_csr_ops * ops,void __iomem * base,u32 bank,struct adf_bank_state * state,u32 num_rings)32 static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base,
33 u32 bank, struct adf_bank_state *state, u32 num_rings)
34 {
35 u32 i;
36
37 state->ringstat0 = ops->read_csr_stat(base, bank);
38 state->ringuostat = ops->read_csr_uo_stat(base, bank);
39 state->ringestat = ops->read_csr_e_stat(base, bank);
40 state->ringnestat = ops->read_csr_ne_stat(base, bank);
41 state->ringnfstat = ops->read_csr_nf_stat(base, bank);
42 state->ringfstat = ops->read_csr_f_stat(base, bank);
43 state->ringcstat0 = ops->read_csr_c_stat(base, bank);
44 state->iaintflagen = ops->read_csr_int_en(base, bank);
45 state->iaintflagreg = ops->read_csr_int_flag(base, bank);
46 state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank);
47 state->iaintcolen = ops->read_csr_int_col_en(base, bank);
48 state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank);
49 state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank);
50 state->ringexpstat = ops->read_csr_exp_stat(base, bank);
51 state->ringexpintenable = ops->read_csr_exp_int_en(base, bank);
52 state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank);
53
54 for (i = 0; i < num_rings; i++) {
55 state->rings[i].head = ops->read_csr_ring_head(base, bank, i);
56 state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i);
57 state->rings[i].config = ops->read_csr_ring_config(base, bank, i);
58 state->rings[i].base = ops->read_csr_ring_base(base, bank, i);
59 }
60 }
61
bank_state_restore(struct adf_hw_csr_ops * ops,void __iomem * base,u32 bank,struct adf_bank_state * state,u32 num_rings,int tx_rx_gap)62 static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base,
63 u32 bank, struct adf_bank_state *state, u32 num_rings,
64 int tx_rx_gap)
65 {
66 u32 val, tmp_val, i;
67 int ret;
68
69 for (i = 0; i < num_rings; i++)
70 ops->write_csr_ring_base(base, bank, i, state->rings[i].base);
71
72 for (i = 0; i < num_rings; i++)
73 ops->write_csr_ring_config(base, bank, i, state->rings[i].config);
74
75 for (i = 0; i < num_rings / 2; i++) {
76 int tx = i * (tx_rx_gap + 1);
77 int rx = tx + tx_rx_gap;
78
79 ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
80 ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail);
81
82 /*
83 * The TX ring head needs to be updated again to make sure that
84 * the HW will not consider the ring as full when it is empty
85 * and the correct state flags are set to match the recovered state.
86 */
87 if (state->ringestat & BIT(tx)) {
88 val = ops->read_csr_int_srcsel(base, bank);
89 val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK;
90 ops->write_csr_int_srcsel_w_val(base, bank, val);
91 ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
92 }
93
94 ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
95 val = ops->read_csr_int_srcsel(base, bank);
96 val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
97 ops->write_csr_int_srcsel_w_val(base, bank, val);
98
99 ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head);
100 val = ops->read_csr_int_srcsel(base, bank);
101 val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
102 ops->write_csr_int_srcsel_w_val(base, bank, val);
103
104 /*
105 * The RX ring tail needs to be updated again to make sure that
106 * the HW will not consider the ring as empty when it is full
107 * and the correct state flags are set to match the recovered state.
108 */
109 if (state->ringfstat & BIT(rx))
110 ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
111 }
112
113 ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen);
114 ops->write_csr_int_en(base, bank, state->iaintflagen);
115 ops->write_csr_int_col_en(base, bank, state->iaintcolen);
116 ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0);
117 ops->write_csr_exp_int_en(base, bank, state->ringexpintenable);
118 ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl);
119
120 /*
121 * Verify whether any exceptions were raised during the bank save process.
122 * If exceptions occurred, the status and exception registers cannot
123 * be directly restored. Consequently, further restoration is not
124 * feasible, and the current state of the ring should be maintained.
125 */
126 val = state->ringexpstat;
127 if (val) {
128 pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n",
129 bank, val);
130 return 0;
131 }
132
133 /* Ensure that the restoration process completed without exceptions */
134 tmp_val = ops->read_csr_exp_stat(base, bank);
135 if (tmp_val) {
136 pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val);
137 return -EFAULT;
138 }
139
140 ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben);
141
142 /* Check that all ring statuses match the saved state. */
143 ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat",
144 base, bank);
145 if (ret)
146 return ret;
147
148 ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat",
149 base, bank);
150 if (ret)
151 return ret;
152
153 ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat",
154 base, bank);
155 if (ret)
156 return ret;
157
158 ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat",
159 base, bank);
160 if (ret)
161 return ret;
162
163 ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat",
164 base, bank);
165 if (ret)
166 return ret;
167
168 ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat",
169 base, bank);
170 if (ret)
171 return ret;
172
173 return 0;
174 }
175
176 /**
177 * adf_bank_state_save() - save state of bank-related registers
178 * @accel_dev: Pointer to the device structure
179 * @bank_number: Bank number
180 * @state: Pointer to bank state structure
181 *
182 * This function saves the state of a bank by reading the bank CSRs and
183 * writing them in the @state structure.
184 *
185 * Returns 0 on success, error code otherwise
186 */
adf_bank_state_save(struct adf_accel_dev * accel_dev,u32 bank_number,struct adf_bank_state * state)187 int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
188 struct adf_bank_state *state)
189 {
190 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
191 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
192 void __iomem *csr_base = adf_get_etr_base(accel_dev);
193
194 if (bank_number >= hw_data->num_banks || !state)
195 return -EINVAL;
196
197 dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number);
198
199 bank_state_save(csr_ops, csr_base, bank_number, state,
200 hw_data->num_rings_per_bank);
201
202 return 0;
203 }
204 EXPORT_SYMBOL_GPL(adf_bank_state_save);
205
206 /**
207 * adf_bank_state_restore() - restore state of bank-related registers
208 * @accel_dev: Pointer to the device structure
209 * @bank_number: Bank number
210 * @state: Pointer to bank state structure
211 *
212 * This function attempts to restore the state of a bank by writing the
213 * bank CSRs to the values in the state structure.
214 *
215 * Returns 0 on success, error code otherwise
216 */
adf_bank_state_restore(struct adf_accel_dev * accel_dev,u32 bank_number,struct adf_bank_state * state)217 int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
218 struct adf_bank_state *state)
219 {
220 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
221 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
222 void __iomem *csr_base = adf_get_etr_base(accel_dev);
223 int ret;
224
225 if (bank_number >= hw_data->num_banks || !state)
226 return -EINVAL;
227
228 dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number);
229
230 ret = bank_state_restore(csr_ops, csr_base, bank_number, state,
231 hw_data->num_rings_per_bank, hw_data->tx_rx_gap);
232 if (ret)
233 dev_err(&GET_DEV(accel_dev),
234 "Unable to restore state of bank %d\n", bank_number);
235
236 return ret;
237 }
238 EXPORT_SYMBOL_GPL(adf_bank_state_restore);
239