24  *  - Redistributions of source code must retain the above copyright
26  *  - Redistributions in binary form must reproduce the above copyright
30  *  - Neither the name of Intel Corporation nor the names of its
50 /* additive distance between non-SOP and SOP space */
52 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
57  * pio_copy - copy data block to MMIO space
60  * @from: source, must be 8 byte aligned
61  * @count: number of DWORD (32-bit) quantities to copy from source
68  * o pbuf->start always starts on a block boundary
74 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in pio_copy()
76 	void __iomem *dend;			/* 8-byte data end */  in pio_copy()
82 	/* calculate where the QWORD data ends - in SOP=1 space */  in pio_copy()
106 		/* write 8-byte SOP chunk data */  in pio_copy()
113 		dest -= SOP_DISTANCE;  in pio_copy()
114 		dend -= SOP_DISTANCE;  in pio_copy()
121 		 * the buffer wraps, then pbuf->end == dend == dest  in pio_copy()
125 		if (pbuf->end <= dend) {  in pio_copy()
126 			while (dest < pbuf->end) {  in pio_copy()
132 			dest -= pbuf->sc->size;  in pio_copy()
133 			dend -= pbuf->sc->size;  in pio_copy()
136 		/* write 8-byte non-SOP, non-wrap chunk data */  in pio_copy()
155 	 * fill in rest of block, no need to check pbuf->end  in pio_copy()
164 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in pio_copy()
175  * "zero" shift - bit shift used to zero out upper bytes.  Input is
178 #define zshift(x) (8 * (8 - (x)))
181  * "merge" shift - bit shift used to merge with carry bytes.  Input is
187  * Jump copy - no-loop copy for < 8 bytes.
217  * of pbuf->carry.  Other bytes are left as-is.  Any previous
218  * value in pbuf->carry is lost.
222  * o from may _not_ be u64 aligned.
227 	pbuf->carry.val64 = 0;  in read_low_bytes()
228 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
229 	pbuf->carry_bytes = nbytes;  in read_low_bytes()
233  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
237  * o from may _not_ be u64 aligned
243 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
244 	pbuf->carry_bytes += nbytes;  in read_extra_bytes()
248  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
250  * pbuf->carry with the upper bytes zeroed..
254  * o src must be u64 aligned
264 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));  in merge_write8()
266 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);  in merge_write8()
280  * Returns 0 on nothing written, non-zero on quad word written.
284 	if (pbuf->carry_bytes) {  in carry_write8()
286 		writeq(pbuf->carry.val64, dest);  in carry_write8()
294  * Segmented PIO Copy - start
300  * @from: data source, QWORD aligned
306 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in seg_pio_copy_start()
308 	void __iomem *dend;			/* 8-byte data end */  in seg_pio_copy_start()
313 	/* calculate where the QWORD data ends - in SOP=1 space */  in seg_pio_copy_start()
337 		/* write 8-byte SOP chunk data */  in seg_pio_copy_start()
344 		dest -= SOP_DISTANCE;  in seg_pio_copy_start()
345 		dend -= SOP_DISTANCE;  in seg_pio_copy_start()
352 		 * the buffer wraps, then pbuf->end == dend == dest  in seg_pio_copy_start()
356 		if (pbuf->end <= dend) {  in seg_pio_copy_start()
357 			while (dest < pbuf->end) {  in seg_pio_copy_start()
363 			dest -= pbuf->sc->size;  in seg_pio_copy_start()
364 			dend -= pbuf->sc->size;  in seg_pio_copy_start()
367 		/* write 8-byte non-SOP, non-wrap chunk data */  in seg_pio_copy_start()
381 	pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);  in seg_pio_copy_start()
385  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
386  * bytes are non-zero.
391  * @from: data source, is QWORD aligned.
398 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_mix()
399 	void __iomem *dend;			/* 8-byte data end */  in mid_copy_mix()
403 	/* calculate 8-byte data end */  in mid_copy_mix()
406 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_mix()
419 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_mix()
426 		/* write 8-byte chunk data */  in mid_copy_mix()
434 		dest -= SOP_DISTANCE;  in mid_copy_mix()
438 	 * - at dend  in mid_copy_mix()
439 	 * - at the wrap  in mid_copy_mix()
450 	 * the buffer wraps, then pbuf->end == dend == dest  in mid_copy_mix()
453 	if (pbuf->end <= dend) {  in mid_copy_mix()
454 		while (dest < pbuf->end) {  in mid_copy_mix()
460 		dest -= pbuf->sc->size;  in mid_copy_mix()
461 		dend -= pbuf->sc->size;  in mid_copy_mix()
464 	/* write 8-byte non-SOP, non-wrap chunk data */  in mid_copy_mix()
471 	pbuf->qw_written += qw_to_write;  in mid_copy_mix()
473 	/* handle carry and left-over bytes */  in mid_copy_mix()
474 	if (pbuf->carry_bytes + bytes_left >= 8) {  in mid_copy_mix()
477 		/* there is enough to fill another qw - fill carry */  in mid_copy_mix()
478 		nread = 8 - pbuf->carry_bytes;  in mid_copy_mix()
482 		 * One more write - but need to make sure dest is correct.  in mid_copy_mix()
493 		if (dest >= pbuf->end)  in mid_copy_mix()
494 			dest -= pbuf->sc->size;  in mid_copy_mix()
496 		else if (pbuf->qw_written < PIO_BLOCK_QWS)  in mid_copy_mix()
500 		carry8_write8(pbuf->carry, dest);  in mid_copy_mix()
501 		pbuf->qw_written++;  in mid_copy_mix()
504 		bytes_left -= nread;  in mid_copy_mix()
505 		from += nread; /* from is now not aligned */  in mid_copy_mix()
514  * Mid copy helper, "straight case" - source pointer is 64-bit aligned
518  * @from: data source, is QWORD aligned
526 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_straight()
527 	void __iomem *dend;			/* 8-byte data end */  in mid_copy_straight()
529 	/* calculate 8-byte data end */  in mid_copy_straight()
532 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_straight()
545 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_straight()
552 		/* write 8-byte chunk data */  in mid_copy_straight()
560 		dest -= SOP_DISTANCE;  in mid_copy_straight()
564 	 * - at dend  in mid_copy_straight()
565 	 * - at the wrap  in mid_copy_straight()
576 	 * the buffer wraps, then pbuf->end == dend == dest  in mid_copy_straight()
579 	if (pbuf->end <= dend) {  in mid_copy_straight()
580 		while (dest < pbuf->end) {  in mid_copy_straight()
586 		dest -= pbuf->sc->size;  in mid_copy_straight()
587 		dend -= pbuf->sc->size;  in mid_copy_straight()
590 	/* write 8-byte non-SOP, non-wrap chunk data */  in mid_copy_straight()
600 	pbuf->qw_written += nbytes >> 3;  in mid_copy_straight()
604  * Segmented PIO Copy - middle
606  * Must handle any aligned tail and any aligned source with any byte count.
616 	if (pbuf->carry_bytes + nbytes < 8) {  in seg_pio_copy_mid()
623 		/* misaligned source pointer - align it */  in seg_pio_copy_mid()
627 		to_align = 8 - from_align;  in seg_pio_copy_mid()
630 		 * In the advance-to-alignment logic below, we do not need  in seg_pio_copy_mid()
635 		if (pbuf->carry_bytes + to_align < 8) {  in seg_pio_copy_mid()
639 			nbytes -= to_align;  in seg_pio_copy_mid()
642 			unsigned long to_fill = 8 - pbuf->carry_bytes;  in seg_pio_copy_mid()
643 			/* bytes left over to be read */  in seg_pio_copy_mid()
644 			unsigned long extra = to_align - to_fill;  in seg_pio_copy_mid()
650 			nbytes -= to_fill;  in seg_pio_copy_mid()
651 			/* may not be enough valid bytes left to align */  in seg_pio_copy_mid()
656 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_mid()
667 			if (dest >= pbuf->end)  in seg_pio_copy_mid()
668 				dest -= pbuf->sc->size;  in seg_pio_copy_mid()
670 			else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_mid()
673 			carry8_write8(pbuf->carry, dest);  in seg_pio_copy_mid()
674 			pbuf->qw_written++;  in seg_pio_copy_mid()
677 			/* this will overwrite anything in pbuf->carry */  in seg_pio_copy_mid()
680 			nbytes -= extra;  in seg_pio_copy_mid()
682 			 * If no bytes are left, return early - we are done.  in seg_pio_copy_mid()
683 			 * NOTE: This short-circuit is *required* because  in seg_pio_copy_mid()
685 			 * is not aligned, as required when leaving this  in seg_pio_copy_mid()
692 		/* at this point, from is QW aligned */  in seg_pio_copy_mid()
695 	if (pbuf->carry_bytes)  in seg_pio_copy_mid()
702  * Segmented PIO Copy - end
704  * Write any remainder (in pbuf->carry) and finish writing the whole block.
710 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_end()
720 	if (dest >= pbuf->end)  in seg_pio_copy_end()
721 		dest -= pbuf->sc->size;  in seg_pio_copy_end()
723 	else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_end()
754 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in seg_pio_copy_end()