s390/cio/qdio_main.c

107  * @q: queue to manipulate
116 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
119 	int tmp_count = count, tmp_start = start, nr = q->nr;
122 	qperf_inc(q, eqbs);
124 	if (!q->is_input_q)
125 		nr += q->irq_ptr->nr_input_qs;
127 	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
137 		qperf_inc(q, eqbs_partial);
138 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",
143 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
146 		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
147 		DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
149 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
150 			   q->first_to_check, count, q->irq_ptr->int_parm);
157  * @q: queue to manipulate
166 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
171 	int nr = q->nr;
173 	qperf_inc(q, sqbs);
175 	if (!q->is_input_q)
176 		nr += q->irq_ptr->nr_input_qs;
178 	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
188 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
189 		qperf_inc(q, sqbs_partial);
192 		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
193 		DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
195 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
196 			   q->first_to_check, count, q->irq_ptr->int_parm);
205 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
212 	if (is_qebsm(q))
213 		return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
216 	__state = q->slsb.val[bufnr];
226 		if (q->slsb.val[bufnr] != __state)
235 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
238 	return get_buf_states(q, bufnr, state, 1, auto_ack);
242 static inline int set_buf_states(struct qdio_q *q, int bufnr,
247 	if (is_qebsm(q))
248 		return qdio_do_sqbs(q, state, bufnr, count);
254 		WRITE_ONCE(q->slsb.val[bufnr], state);
264 static inline int set_buf_state(struct qdio_q *q, int bufnr,
267 	return set_buf_states(q, bufnr, state, 1);
273 	struct qdio_q *q;
276 	for_each_input_queue(irq_ptr, q, i)
277 		set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
279 	for_each_output_queue(irq_ptr, q, i)
280 		set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
284 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
287 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
291 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
292 	qperf_inc(q, siga_sync);
294 	if (is_qebsm(q)) {
295 		schid = q->irq_ptr->sch_token;
301 		DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
305 static inline int qdio_sync_input_queue(struct qdio_q *q)
307 	return qdio_siga_sync(q, 0, q->mask);
310 static inline int qdio_sync_output_queue(struct qdio_q *q)
312 	return qdio_siga_sync(q, q->mask, 0);
315 static inline int qdio_siga_sync_q(struct qdio_q *q)
317 	if (q->is_input_q)
318 		return qdio_sync_input_queue(q);
320 		return qdio_sync_output_queue(q);
323 static int qdio_siga_output(struct qdio_q *q, unsigned int count,
326 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
331 	if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {
338 	if (is_qebsm(q)) {
339 		schid = q->irq_ptr->sch_token;
343 	cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);
357 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,
358 			      "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);
359 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);
364 static inline int qdio_siga_input(struct qdio_q *q)
366 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
370 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
371 	qperf_inc(q, siga_read);
373 	if (is_qebsm(q)) {
374 		schid = q->irq_ptr->sch_token;
378 	cc = do_siga_input(schid, q->mask, fc);
380 		DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
384 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
387 	if (qdio_need_siga_sync(q->irq_ptr))
388 		qdio_siga_sync_q(q);
389 	return get_buf_state(q, bufnr, state, 0);
392 static inline void qdio_stop_polling(struct qdio_q *q)
394 	if (!q->u.in.batch_count)
397 	qperf_inc(q, stop_polling);
400 	set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,
401 		       q->u.in.batch_count);
402 	q->u.in.batch_count = 0;
405 static inline void account_sbals(struct qdio_q *q, unsigned int count)
407 	q->q_stats.nr_sbal_total += count;
408 	q->q_stats.nr_sbals[ilog2(count)]++;
411 static void process_buffer_error(struct qdio_q *q, unsigned int start,
415 	if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&
416 	    q->sbal[start]->element[15].sflags == 0x10) {
417 		qperf_inc(q, target_full);
418 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);
422 	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
423 	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
426 		  q->sbal[start]->element[14].sflags,
427 		  q->sbal[start]->element[15].sflags);
430 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
435 		set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);
437 	if (!q->u.in.batch_count)
438 		q->u.in.batch_start = start;
439 	q->u.in.batch_count += count;
442 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start,
448 	q->timestamp = get_tod_clock_fast();
450 	count = atomic_read(&q->nr_buf_used);
454 	if (qdio_need_siga_sync(q->irq_ptr))
455 		qdio_sync_input_queue(q);
457 	count = get_buf_states(q, start, &state, count, 1);
463 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
466 		inbound_handle_work(q, start, count, is_qebsm(q));
467 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
468 			qperf_inc(q, inbound_queue_full);
469 		if (q->irq_ptr->perf_stat_enabled)
470 			account_sbals(q, count);
473 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
477 		process_buffer_error(q, start, count);
478 		inbound_handle_work(q, start, count, false);
479 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
480 			qperf_inc(q, inbound_queue_full);
481 		if (q->irq_ptr->perf_stat_enabled)
482 			account_sbals_error(q, count);
485 		if (q->irq_ptr->perf_stat_enabled)
486 			q->q_stats.nr_sbal_nop++;
487 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",
488 			      q->nr, start);
494 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
496 			      state, start, q->nr);
506 	struct qdio_q *q;
512 	q = irq->input_qs[nr];
513 	start = q->first_to_check;
516 	count = get_inbound_buffer_frontier(q, start, error);
521 	q->first_to_check = add_buf(start, count);
526 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)
530 	if (!atomic_read(&q->nr_buf_used))
533 	if (qdio_need_siga_sync(q->irq_ptr))
534 		qdio_sync_input_queue(q);
535 	get_buf_state(q, start, &state, 0);
544 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start,
550 	q->timestamp = get_tod_clock_fast();
552 	count = atomic_read(&q->nr_buf_used);
556 	if (qdio_need_siga_sync(q->irq_ptr))
557 		qdio_sync_output_queue(q);
559 	count = get_buf_states(q, start, &state, count, 0);
569 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,
570 			"out empty:%1d %02x", q->nr, count);
572 		atomic_sub(count, &q->nr_buf_used);
573 		if (q->irq_ptr->perf_stat_enabled)
574 			account_sbals(q, count);
577 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x",
578 			      q->nr, count);
581 		process_buffer_error(q, start, count);
582 		atomic_sub(count, &q->nr_buf_used);
583 		if (q->irq_ptr->perf_stat_enabled)
584 			account_sbals_error(q, count);
588 		if (q->irq_ptr->perf_stat_enabled)
589 			q->q_stats.nr_sbal_nop++;
590 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
591 			      q->nr);
598 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
600 			      state, start, q->nr);
610 	struct qdio_q *q;
616 	q = irq->output_qs[nr];
617 	start = q->first_to_check;
620 	count = get_outbound_buffer_frontier(q, start, error);
625 	q->first_to_check = add_buf(start, count);
630 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,
636 	if (!qdio_need_siga_out(q->irq_ptr))
639 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
641 	qperf_inc(q, siga_write);
643 	cc = qdio_siga_output(q, count, &busy_bit, aob);
653 			DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);
656 			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
662 		DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
667 		DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);
1087 	/* establish q */
1206  * @q: queue containing the buffers
1210 static int handle_inbound(struct qdio_q *q, int bufnr, int count)
1214 	qperf_inc(q, inbound_call);
1217 	overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),
1218 			     q->u.in.batch_count);
1220 		q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);
1221 		q->u.in.batch_count -= overlap;
1224 	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1225 	atomic_add(count, &q->nr_buf_used);
1227 	if (qdio_need_siga_in(q->irq_ptr))
1228 		return qdio_siga_input(q);
1264  * @q: queue containing the buffers
1269 static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count,
1275 	qperf_inc(q, outbound_call);
1277 	count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1278 	used = atomic_add_return(count, &q->nr_buf_used);
1281 		qperf_inc(q, outbound_queue_full);
1283 	if (queue_type(q) == QDIO_IQDIO_QFMT) {
1287 		rc = qdio_kick_outbound_q(q, count, phys_aob);
1288 	} else if (qdio_need_siga_sync(q->irq_ptr)) {
1289 		rc = qdio_sync_output_queue(q);
1291 		   get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 &&
1294 		qperf_inc(q, fast_requeue);
1296 		rc = qdio_kick_outbound_q(q, count, 0);
1343 	struct qdio_q *q;
1350 	for_each_input_queue(irq_ptr, q, i)
1351 		qdio_stop_polling(q);
1362 	for_each_input_queue(irq_ptr, q, i) {
1363 		if (!qdio_inbound_q_done(q, q->first_to_check))