1 /*
2 * zfcp device driver
3 *
4 * Setup and helper functions to access QDIO.
5 *
6 * Copyright IBM Corporation 2002, 2010
7 */
8
9 #define KMSG_COMPONENT "zfcp"
10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11
12 #include <linux/slab.h>
13 #include <linux/module.h>
14 #include "zfcp_ext.h"
15 #include "zfcp_qdio.h"
16
17 #define QBUFF_PER_PAGE (PAGE_SIZE / sizeof(struct qdio_buffer))
18
19 static bool enable_multibuffer;
20 module_param_named(datarouter, enable_multibuffer, bool, 0400);
21 MODULE_PARM_DESC(datarouter, "Enable hardware data router support");
22
zfcp_qdio_buffers_enqueue(struct qdio_buffer ** sbal)23 static int zfcp_qdio_buffers_enqueue(struct qdio_buffer **sbal)
24 {
25 int pos;
26
27 for (pos = 0; pos < QDIO_MAX_BUFFERS_PER_Q; pos += QBUFF_PER_PAGE) {
28 sbal[pos] = (struct qdio_buffer *) get_zeroed_page(GFP_KERNEL);
29 if (!sbal[pos])
30 return -ENOMEM;
31 }
32 for (pos = 0; pos < QDIO_MAX_BUFFERS_PER_Q; pos++)
33 if (pos % QBUFF_PER_PAGE)
34 sbal[pos] = sbal[pos - 1] + 1;
35 return 0;
36 }
37
zfcp_qdio_handler_error(struct zfcp_qdio * qdio,char * id,unsigned int qdio_err)38 static void zfcp_qdio_handler_error(struct zfcp_qdio *qdio, char *id,
39 unsigned int qdio_err)
40 {
41 struct zfcp_adapter *adapter = qdio->adapter;
42
43 dev_warn(&adapter->ccw_device->dev, "A QDIO problem occurred\n");
44
45 if (qdio_err & QDIO_ERROR_SLSB_STATE) {
46 zfcp_qdio_siosl(adapter);
47 zfcp_erp_adapter_shutdown(adapter, 0, id);
48 return;
49 }
50 zfcp_erp_adapter_reopen(adapter,
51 ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED |
52 ZFCP_STATUS_COMMON_ERP_FAILED, id);
53 }
54
zfcp_qdio_zero_sbals(struct qdio_buffer * sbal[],int first,int cnt)55 static void zfcp_qdio_zero_sbals(struct qdio_buffer *sbal[], int first, int cnt)
56 {
57 int i, sbal_idx;
58
59 for (i = first; i < first + cnt; i++) {
60 sbal_idx = i % QDIO_MAX_BUFFERS_PER_Q;
61 memset(sbal[sbal_idx], 0, sizeof(struct qdio_buffer));
62 }
63 }
64
65 /* this needs to be called prior to updating the queue fill level */
zfcp_qdio_account(struct zfcp_qdio * qdio)66 static inline void zfcp_qdio_account(struct zfcp_qdio *qdio)
67 {
68 unsigned long long now, span;
69 int used;
70
71 now = get_clock_monotonic();
72 span = (now - qdio->req_q_time) >> 12;
73 used = QDIO_MAX_BUFFERS_PER_Q - atomic_read(&qdio->req_q_free);
74 qdio->req_q_util += used * span;
75 qdio->req_q_time = now;
76 }
77
zfcp_qdio_int_req(struct ccw_device * cdev,unsigned int qdio_err,int queue_no,int idx,int count,unsigned long parm)78 static void zfcp_qdio_int_req(struct ccw_device *cdev, unsigned int qdio_err,
79 int queue_no, int idx, int count,
80 unsigned long parm)
81 {
82 struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
83
84 if (unlikely(qdio_err)) {
85 zfcp_qdio_handler_error(qdio, "qdireq1", qdio_err);
86 return;
87 }
88
89 /* cleanup all SBALs being program-owned now */
90 zfcp_qdio_zero_sbals(qdio->req_q, idx, count);
91
92 spin_lock_irq(&qdio->stat_lock);
93 zfcp_qdio_account(qdio);
94 spin_unlock_irq(&qdio->stat_lock);
95 atomic_add(count, &qdio->req_q_free);
96 wake_up(&qdio->req_q_wq);
97 }
98
zfcp_qdio_int_resp(struct ccw_device * cdev,unsigned int qdio_err,int queue_no,int idx,int count,unsigned long parm)99 static void zfcp_qdio_int_resp(struct ccw_device *cdev, unsigned int qdio_err,
100 int queue_no, int idx, int count,
101 unsigned long parm)
102 {
103 struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
104 struct zfcp_adapter *adapter = qdio->adapter;
105 struct qdio_buffer_element *sbale;
106 int sbal_no, sbal_idx;
107 void *pl[ZFCP_QDIO_MAX_SBALS_PER_REQ + 1];
108 u64 req_id;
109 u8 scount;
110
111 if (unlikely(qdio_err)) {
112 memset(pl, 0, ZFCP_QDIO_MAX_SBALS_PER_REQ * sizeof(void *));
113 if (zfcp_adapter_multi_buffer_active(adapter)) {
114 sbale = qdio->res_q[idx]->element;
115 req_id = (u64) sbale->addr;
116 scount = sbale->scount + 1; /* incl. signaling SBAL */
117
118 for (sbal_no = 0; sbal_no < scount; sbal_no++) {
119 sbal_idx = (idx + sbal_no) %
120 QDIO_MAX_BUFFERS_PER_Q;
121 pl[sbal_no] = qdio->res_q[sbal_idx];
122 }
123 zfcp_dbf_hba_def_err(adapter, req_id, scount, pl);
124 }
125 zfcp_qdio_handler_error(qdio, "qdires1", qdio_err);
126 return;
127 }
128
129 /*
130 * go through all SBALs from input queue currently
131 * returned by QDIO layer
132 */
133 for (sbal_no = 0; sbal_no < count; sbal_no++) {
134 sbal_idx = (idx + sbal_no) % QDIO_MAX_BUFFERS_PER_Q;
135 /* go through all SBALEs of SBAL */
136 zfcp_fsf_reqid_check(qdio, sbal_idx);
137 }
138
139 /*
140 * put SBALs back to response queue
141 */
142 if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, idx, count))
143 zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdires2");
144 }
145
146 static struct qdio_buffer_element *
zfcp_qdio_sbal_chain(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req)147 zfcp_qdio_sbal_chain(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
148 {
149 struct qdio_buffer_element *sbale;
150
151 /* set last entry flag in current SBALE of current SBAL */
152 sbale = zfcp_qdio_sbale_curr(qdio, q_req);
153 sbale->eflags |= SBAL_EFLAGS_LAST_ENTRY;
154
155 /* don't exceed last allowed SBAL */
156 if (q_req->sbal_last == q_req->sbal_limit)
157 return NULL;
158
159 /* set chaining flag in first SBALE of current SBAL */
160 sbale = zfcp_qdio_sbale_req(qdio, q_req);
161 sbale->sflags |= SBAL_SFLAGS0_MORE_SBALS;
162
163 /* calculate index of next SBAL */
164 q_req->sbal_last++;
165 q_req->sbal_last %= QDIO_MAX_BUFFERS_PER_Q;
166
167 /* keep this requests number of SBALs up-to-date */
168 q_req->sbal_number++;
169 BUG_ON(q_req->sbal_number > ZFCP_QDIO_MAX_SBALS_PER_REQ);
170
171 /* start at first SBALE of new SBAL */
172 q_req->sbale_curr = 0;
173
174 /* set storage-block type for new SBAL */
175 sbale = zfcp_qdio_sbale_curr(qdio, q_req);
176 sbale->sflags |= q_req->sbtype;
177
178 return sbale;
179 }
180
181 static struct qdio_buffer_element *
zfcp_qdio_sbale_next(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req)182 zfcp_qdio_sbale_next(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
183 {
184 if (q_req->sbale_curr == qdio->max_sbale_per_sbal - 1)
185 return zfcp_qdio_sbal_chain(qdio, q_req);
186 q_req->sbale_curr++;
187 return zfcp_qdio_sbale_curr(qdio, q_req);
188 }
189
190 /**
191 * zfcp_qdio_sbals_from_sg - fill SBALs from scatter-gather list
192 * @qdio: pointer to struct zfcp_qdio
193 * @q_req: pointer to struct zfcp_qdio_req
194 * @sg: scatter-gather list
195 * @max_sbals: upper bound for number of SBALs to be used
196 * Returns: zero or -EINVAL on error
197 */
zfcp_qdio_sbals_from_sg(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req,struct scatterlist * sg)198 int zfcp_qdio_sbals_from_sg(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req,
199 struct scatterlist *sg)
200 {
201 struct qdio_buffer_element *sbale;
202
203 /* set storage-block type for this request */
204 sbale = zfcp_qdio_sbale_req(qdio, q_req);
205 sbale->sflags |= q_req->sbtype;
206
207 for (; sg; sg = sg_next(sg)) {
208 sbale = zfcp_qdio_sbale_next(qdio, q_req);
209 if (!sbale) {
210 atomic_inc(&qdio->req_q_full);
211 zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
212 q_req->sbal_number);
213 return -EINVAL;
214 }
215 sbale->addr = sg_virt(sg);
216 sbale->length = sg->length;
217 }
218 return 0;
219 }
220
zfcp_qdio_sbal_check(struct zfcp_qdio * qdio)221 static int zfcp_qdio_sbal_check(struct zfcp_qdio *qdio)
222 {
223 spin_lock_irq(&qdio->req_q_lock);
224 if (atomic_read(&qdio->req_q_free) ||
225 !(atomic_read(&qdio->adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
226 return 1;
227 spin_unlock_irq(&qdio->req_q_lock);
228 return 0;
229 }
230
231 /**
232 * zfcp_qdio_sbal_get - get free sbal in request queue, wait if necessary
233 * @qdio: pointer to struct zfcp_qdio
234 *
235 * The req_q_lock must be held by the caller of this function, and
236 * this function may only be called from process context; it will
237 * sleep when waiting for a free sbal.
238 *
239 * Returns: 0 on success, -EIO if there is no free sbal after waiting.
240 */
zfcp_qdio_sbal_get(struct zfcp_qdio * qdio)241 int zfcp_qdio_sbal_get(struct zfcp_qdio *qdio)
242 {
243 long ret;
244
245 spin_unlock_irq(&qdio->req_q_lock);
246 ret = wait_event_interruptible_timeout(qdio->req_q_wq,
247 zfcp_qdio_sbal_check(qdio), 5 * HZ);
248
249 if (!(atomic_read(&qdio->adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
250 return -EIO;
251
252 if (ret > 0)
253 return 0;
254
255 if (!ret) {
256 atomic_inc(&qdio->req_q_full);
257 /* assume hanging outbound queue, try queue recovery */
258 zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdsbg_1");
259 }
260
261 spin_lock_irq(&qdio->req_q_lock);
262 return -EIO;
263 }
264
265 /**
266 * zfcp_qdio_send - set PCI flag in first SBALE and send req to QDIO
267 * @qdio: pointer to struct zfcp_qdio
268 * @q_req: pointer to struct zfcp_qdio_req
269 * Returns: 0 on success, error otherwise
270 */
zfcp_qdio_send(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req)271 int zfcp_qdio_send(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
272 {
273 int retval;
274 u8 sbal_number = q_req->sbal_number;
275
276 spin_lock(&qdio->stat_lock);
277 zfcp_qdio_account(qdio);
278 spin_unlock(&qdio->stat_lock);
279
280 retval = do_QDIO(qdio->adapter->ccw_device, QDIO_FLAG_SYNC_OUTPUT, 0,
281 q_req->sbal_first, sbal_number);
282
283 if (unlikely(retval)) {
284 zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
285 sbal_number);
286 return retval;
287 }
288
289 /* account for transferred buffers */
290 atomic_sub(sbal_number, &qdio->req_q_free);
291 qdio->req_q_idx += sbal_number;
292 qdio->req_q_idx %= QDIO_MAX_BUFFERS_PER_Q;
293
294 return 0;
295 }
296
297
zfcp_qdio_setup_init_data(struct qdio_initialize * id,struct zfcp_qdio * qdio)298 static void zfcp_qdio_setup_init_data(struct qdio_initialize *id,
299 struct zfcp_qdio *qdio)
300 {
301 memset(id, 0, sizeof(*id));
302 id->cdev = qdio->adapter->ccw_device;
303 id->q_format = QDIO_ZFCP_QFMT;
304 memcpy(id->adapter_name, dev_name(&id->cdev->dev), 8);
305 ASCEBC(id->adapter_name, 8);
306 id->qib_rflags = QIB_RFLAGS_ENABLE_DATA_DIV;
307 if (enable_multibuffer)
308 id->qdr_ac |= QDR_AC_MULTI_BUFFER_ENABLE;
309 id->no_input_qs = 1;
310 id->no_output_qs = 1;
311 id->input_handler = zfcp_qdio_int_resp;
312 id->output_handler = zfcp_qdio_int_req;
313 id->int_parm = (unsigned long) qdio;
314 id->input_sbal_addr_array = (void **) (qdio->res_q);
315 id->output_sbal_addr_array = (void **) (qdio->req_q);
316 id->scan_threshold =
317 QDIO_MAX_BUFFERS_PER_Q - ZFCP_QDIO_MAX_SBALS_PER_REQ * 2;
318 }
319
320 /**
321 * zfcp_qdio_allocate - allocate queue memory and initialize QDIO data
322 * @adapter: pointer to struct zfcp_adapter
323 * Returns: -ENOMEM on memory allocation error or return value from
324 * qdio_allocate
325 */
zfcp_qdio_allocate(struct zfcp_qdio * qdio)326 static int zfcp_qdio_allocate(struct zfcp_qdio *qdio)
327 {
328 struct qdio_initialize init_data;
329
330 if (zfcp_qdio_buffers_enqueue(qdio->req_q) ||
331 zfcp_qdio_buffers_enqueue(qdio->res_q))
332 return -ENOMEM;
333
334 zfcp_qdio_setup_init_data(&init_data, qdio);
335 init_waitqueue_head(&qdio->req_q_wq);
336
337 return qdio_allocate(&init_data);
338 }
339
340 /**
341 * zfcp_close_qdio - close qdio queues for an adapter
342 * @qdio: pointer to structure zfcp_qdio
343 */
zfcp_qdio_close(struct zfcp_qdio * qdio)344 void zfcp_qdio_close(struct zfcp_qdio *qdio)
345 {
346 struct zfcp_adapter *adapter = qdio->adapter;
347 int idx, count;
348
349 if (!(atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
350 return;
351
352 /* clear QDIOUP flag, thus do_QDIO is not called during qdio_shutdown */
353 spin_lock_irq(&qdio->req_q_lock);
354 atomic_clear_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status);
355 spin_unlock_irq(&qdio->req_q_lock);
356
357 wake_up(&qdio->req_q_wq);
358
359 qdio_shutdown(adapter->ccw_device, QDIO_FLAG_CLEANUP_USING_CLEAR);
360
361 /* cleanup used outbound sbals */
362 count = atomic_read(&qdio->req_q_free);
363 if (count < QDIO_MAX_BUFFERS_PER_Q) {
364 idx = (qdio->req_q_idx + count) % QDIO_MAX_BUFFERS_PER_Q;
365 count = QDIO_MAX_BUFFERS_PER_Q - count;
366 zfcp_qdio_zero_sbals(qdio->req_q, idx, count);
367 }
368 qdio->req_q_idx = 0;
369 atomic_set(&qdio->req_q_free, 0);
370 }
371
372 /**
373 * zfcp_qdio_open - prepare and initialize response queue
374 * @qdio: pointer to struct zfcp_qdio
375 * Returns: 0 on success, otherwise -EIO
376 */
zfcp_qdio_open(struct zfcp_qdio * qdio)377 int zfcp_qdio_open(struct zfcp_qdio *qdio)
378 {
379 struct qdio_buffer_element *sbale;
380 struct qdio_initialize init_data;
381 struct zfcp_adapter *adapter = qdio->adapter;
382 struct ccw_device *cdev = adapter->ccw_device;
383 struct qdio_ssqd_desc ssqd;
384 int cc;
385
386 if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP)
387 return -EIO;
388
389 atomic_clear_mask(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED,
390 &qdio->adapter->status);
391
392 zfcp_qdio_setup_init_data(&init_data, qdio);
393
394 if (qdio_establish(&init_data))
395 goto failed_establish;
396
397 if (qdio_get_ssqd_desc(init_data.cdev, &ssqd))
398 goto failed_qdio;
399
400 if (ssqd.qdioac2 & CHSC_AC2_DATA_DIV_ENABLED)
401 atomic_set_mask(ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED,
402 &qdio->adapter->status);
403
404 if (ssqd.qdioac2 & CHSC_AC2_MULTI_BUFFER_ENABLED) {
405 atomic_set_mask(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status);
406 qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER;
407 } else {
408 atomic_clear_mask(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status);
409 qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER - 1;
410 }
411
412 qdio->max_sbale_per_req =
413 ZFCP_QDIO_MAX_SBALS_PER_REQ * qdio->max_sbale_per_sbal
414 - 2;
415 if (qdio_activate(cdev))
416 goto failed_qdio;
417
418 for (cc = 0; cc < QDIO_MAX_BUFFERS_PER_Q; cc++) {
419 sbale = &(qdio->res_q[cc]->element[0]);
420 sbale->length = 0;
421 sbale->eflags = SBAL_EFLAGS_LAST_ENTRY;
422 sbale->sflags = 0;
423 sbale->addr = NULL;
424 }
425
426 if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, 0, QDIO_MAX_BUFFERS_PER_Q))
427 goto failed_qdio;
428
429 /* set index of first available SBALS / number of available SBALS */
430 qdio->req_q_idx = 0;
431 atomic_set(&qdio->req_q_free, QDIO_MAX_BUFFERS_PER_Q);
432 atomic_set_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status);
433
434 if (adapter->scsi_host) {
435 adapter->scsi_host->sg_tablesize = qdio->max_sbale_per_req;
436 adapter->scsi_host->max_sectors = qdio->max_sbale_per_req * 8;
437 }
438
439 return 0;
440
441 failed_qdio:
442 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
443 failed_establish:
444 dev_err(&cdev->dev,
445 "Setting up the QDIO connection to the FCP adapter failed\n");
446 return -EIO;
447 }
448
zfcp_qdio_destroy(struct zfcp_qdio * qdio)449 void zfcp_qdio_destroy(struct zfcp_qdio *qdio)
450 {
451 int p;
452
453 if (!qdio)
454 return;
455
456 if (qdio->adapter->ccw_device)
457 qdio_free(qdio->adapter->ccw_device);
458
459 for (p = 0; p < QDIO_MAX_BUFFERS_PER_Q; p += QBUFF_PER_PAGE) {
460 free_page((unsigned long) qdio->req_q[p]);
461 free_page((unsigned long) qdio->res_q[p]);
462 }
463
464 kfree(qdio);
465 }
466
zfcp_qdio_setup(struct zfcp_adapter * adapter)467 int zfcp_qdio_setup(struct zfcp_adapter *adapter)
468 {
469 struct zfcp_qdio *qdio;
470
471 qdio = kzalloc(sizeof(struct zfcp_qdio), GFP_KERNEL);
472 if (!qdio)
473 return -ENOMEM;
474
475 qdio->adapter = adapter;
476
477 if (zfcp_qdio_allocate(qdio)) {
478 zfcp_qdio_destroy(qdio);
479 return -ENOMEM;
480 }
481
482 spin_lock_init(&qdio->req_q_lock);
483 spin_lock_init(&qdio->stat_lock);
484
485 adapter->qdio = qdio;
486 return 0;
487 }
488
489 /**
490 * zfcp_qdio_siosl - Trigger logging in FCP channel
491 * @adapter: The zfcp_adapter where to trigger logging
492 *
493 * Call the cio siosl function to trigger hardware logging. This
494 * wrapper function sets a flag to ensure hardware logging is only
495 * triggered once before going through qdio shutdown.
496 *
497 * The triggers are always run from qdio tasklet context, so no
498 * additional synchronization is necessary.
499 */
zfcp_qdio_siosl(struct zfcp_adapter * adapter)500 void zfcp_qdio_siosl(struct zfcp_adapter *adapter)
501 {
502 int rc;
503
504 if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_SIOSL_ISSUED)
505 return;
506
507 rc = ccw_device_siosl(adapter->ccw_device);
508 if (!rc)
509 atomic_set_mask(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED,
510 &adapter->status);
511 }
512