2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Quick'n'dirty IP checksum ...
8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki
11 * Copyright (C) 2014 Imagination Technologies Ltd.
13 #include <linux/errno.h>
15 #include <asm/asm-offsets.h>
16 #include <asm/regdef.h>
20 * As we are sharing code base with the mips32 tree (which use the o32 ABI
21 * register definitions). We need to redefine the register definitions from
22 * the n64 ABI register naming to the o32 ABI register naming.
54 #endif /* USE_DOUBLE */
56 #define UNIT(unit) ((unit)*NBYTES)
58 #define ADDC(sum,reg) \
63 #define ADDC32(sum,reg) \
68 #define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
69 LOAD _t0, (offset + UNIT(0))(src); \
70 LOAD _t1, (offset + UNIT(1))(src); \
71 LOAD _t2, (offset + UNIT(2))(src); \
72 LOAD _t3, (offset + UNIT(3))(src); \
79 #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
80 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
82 #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
83 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
84 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
89 * a1: length of the area to checksum
90 * a2: partial checksum
104 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
107 andi t7, src, 0x1 /* odd buffer? */
110 beqz t7, .Lword_align
114 LONG_SUBU a1, a1, 0x1
119 PTR_ADDU src, src, 0x1
123 beqz t8, .Ldword_align
127 LONG_SUBU a1, a1, 0x2
130 PTR_ADDU src, src, 0x2
133 bnez t8, .Ldo_end_words
137 beqz t8, .Lqword_align
141 LONG_SUBU a1, a1, 0x4
143 PTR_ADDU src, src, 0x4
147 beqz t8, .Loword_align
152 LONG_SUBU a1, a1, 0x8
157 LONG_SUBU a1, a1, 0x8
161 PTR_ADDU src, src, 0x8
165 beqz t8, .Lbegin_movement
174 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
176 LONG_SUBU a1, a1, 0x10
177 PTR_ADDU src, src, 0x10
185 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
186 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
187 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
188 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
189 LONG_SUBU t8, t8, 0x01
190 .set reorder /* DADDI_WAR */
191 PTR_ADDU src, src, 0x80
192 bnez t8, .Lmove_128bytes
200 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
201 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
202 PTR_ADDU src, src, 0x40
205 beqz t2, .Ldo_end_words
209 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
211 PTR_ADDU src, src, 0x20
214 beqz t8, .Lsmall_csumcpy
220 LONG_SUBU t8, t8, 0x1
222 .set reorder /* DADDI_WAR */
223 PTR_ADDU src, src, 0x4
227 /* unknown src alignment and < 8 bytes to go */
235 /* Still a full word to go */
239 dsll t1, t1, 32 /* clear lower 32bit */
247 /* Still a halfword to go */
273 /* odd buffer alignment? */
274 #ifdef CONFIG_CPU_MIPSR2
278 beqz t7, 1f /* odd buffer alignment? */
289 /* Add the passed partial csum. */
297 * checksum and copy routines based on memcpy.S
299 * csum_partial_copy_nocheck(src, dst, len, sum)
300 * __csum_partial_copy_kernel(src, dst, len, sum, errp)
302 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
303 * function in this file use the standard calling convention.
315 * The exception handler for loads requires that:
316 * 1- AT contain the address of the byte just past the end of the source
318 * 2- src_entry <= src < AT, and
319 * 3- (dst - src) == (dst_entry - src_entry),
320 * The _entry suffix denotes values when __copy_user was called.
322 * (1) is set up up by __csum_partial_copy_from_user and maintained by
323 * not writing AT in __csum_partial_copy
324 * (2) is met by incrementing src by the number of bytes copied
325 * (3) is met by not doing loads between a pair of increments of dst and src
327 * The exception handlers for stores stores -EFAULT to errptr and return.
328 * These handlers do not need to overwrite any data.
331 /* Instruction type */
336 * Wrapper to add an entry in the exception table
337 * in case the insn causes a memory exception.
339 * insn : Load/store instruction
340 * type : Instruction type
343 * handler : Exception handler
345 #define EXC(insn, type, reg, addr, handler) \
347 .section __ex_table,"a"; \
355 #define LOADK ld /* No exception */
356 #define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
357 #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
358 #define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
359 #define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
360 #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
361 #define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
362 #define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
363 #define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
375 #define LOADK lw /* No exception */
376 #define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
377 #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
378 #define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
379 #define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
380 #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
381 #define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
382 #define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
383 #define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
393 #endif /* USE_DOUBLE */
395 #ifdef CONFIG_CPU_LITTLE_ENDIAN
396 #define LDFIRST LOADR
398 #define STFIRST STORER
399 #define STREST STOREL
400 #define SHIFT_DISCARD SLLV
401 #define SHIFT_DISCARD_REVERT SRLV
403 #define LDFIRST LOADL
405 #define STFIRST STOREL
406 #define STREST STORER
407 #define SHIFT_DISCARD SRLV
408 #define SHIFT_DISCARD_REVERT SLLV
411 #define FIRST(unit) ((unit)*NBYTES)
412 #define REST(unit) (FIRST(unit)+NBYTES-1)
414 #define ADDRMASK (NBYTES-1)
416 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
422 LEAF(__csum_partial_copy_kernel)
423 FEXPORT(__csum_partial_copy_to_user)
424 FEXPORT(__csum_partial_copy_from_user)
425 PTR_ADDU AT, src, len /* See (1) above. */
431 FEXPORT(csum_partial_copy_nocheck)
435 * Note: dst & src may be unaligned, len may be 0
439 * The "issue break"s below are very approximate.
440 * Issue delays for dcache fills will perturb the schedule, as will
441 * load queue full replay traps, etc.
443 * If len < NBYTES use byte operations.
446 and t1, dst, ADDRMASK
447 bnez t2, .Lcopy_bytes_checklen
448 and t0, src, ADDRMASK
449 andi odd, dst, 0x1 /* odd buffer? */
450 bnez t1, .Ldst_unaligned
452 bnez t0, .Lsrc_unaligned_dst_aligned
454 * use delay slot for fall-through
455 * src and dst are aligned; need to compute rem
458 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
459 beqz t0, .Lcleanup_both_aligned # len < 8*NBYTES
461 SUB len, 8*NBYTES # subtract here for bgez loop
464 LOAD(t0, UNIT(0)(src), .Ll_exc)
465 LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
466 LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
467 LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
468 LOAD(t4, UNIT(4)(src), .Ll_exc_copy)
469 LOAD(t5, UNIT(5)(src), .Ll_exc_copy)
470 LOAD(t6, UNIT(6)(src), .Ll_exc_copy)
471 LOAD(t7, UNIT(7)(src), .Ll_exc_copy)
472 SUB len, len, 8*NBYTES
473 ADD src, src, 8*NBYTES
474 STORE(t0, UNIT(0)(dst), .Ls_exc)
476 STORE(t1, UNIT(1)(dst), .Ls_exc)
478 STORE(t2, UNIT(2)(dst), .Ls_exc)
480 STORE(t3, UNIT(3)(dst), .Ls_exc)
482 STORE(t4, UNIT(4)(dst), .Ls_exc)
484 STORE(t5, UNIT(5)(dst), .Ls_exc)
486 STORE(t6, UNIT(6)(dst), .Ls_exc)
488 STORE(t7, UNIT(7)(dst), .Ls_exc)
490 .set reorder /* DADDI_WAR */
491 ADD dst, dst, 8*NBYTES
494 ADD len, 8*NBYTES # revert len (see above)
497 * len == the number of bytes left to copy < 8*NBYTES
499 .Lcleanup_both_aligned:
502 sltu t0, len, 4*NBYTES
503 bnez t0, .Lless_than_4units
504 and rem, len, (NBYTES-1) # rem = len % NBYTES
508 LOAD(t0, UNIT(0)(src), .Ll_exc)
509 LOAD(t1, UNIT(1)(src), .Ll_exc_copy)
510 LOAD(t2, UNIT(2)(src), .Ll_exc_copy)
511 LOAD(t3, UNIT(3)(src), .Ll_exc_copy)
512 SUB len, len, 4*NBYTES
513 ADD src, src, 4*NBYTES
514 STORE(t0, UNIT(0)(dst), .Ls_exc)
516 STORE(t1, UNIT(1)(dst), .Ls_exc)
518 STORE(t2, UNIT(2)(dst), .Ls_exc)
520 STORE(t3, UNIT(3)(dst), .Ls_exc)
522 .set reorder /* DADDI_WAR */
523 ADD dst, dst, 4*NBYTES
530 beq rem, len, .Lcopy_bytes
533 LOAD(t0, 0(src), .Ll_exc)
536 STORE(t0, 0(dst), .Ls_exc)
538 .set reorder /* DADDI_WAR */
544 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
545 * A loop would do only a byte at a time with possible branch
546 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
547 * because can't assume read-access to dst. Instead, use
548 * STREST dst, which doesn't require read access to dst.
550 * This code should perform better than a simple loop on modern,
551 * wide-issue mips processors because the code has fewer branches and
552 * more instruction-level parallelism.
556 ADD t1, dst, len # t1 is just past last byte of dst
558 SLL rem, len, 3 # rem = number of bits to keep
559 LOAD(t0, 0(src), .Ll_exc)
560 SUB bits, bits, rem # bits = number of bits to discard
561 SHIFT_DISCARD t0, t0, bits
562 STREST(t0, -1(t1), .Ls_exc)
563 SHIFT_DISCARD_REVERT t0, t0, bits
571 * t0 = src & ADDRMASK
572 * t1 = dst & ADDRMASK; T1 > 0
575 * Copy enough bytes to align dst
576 * Set match = (src and dst have same alignment)
579 LDFIRST(t3, FIRST(0)(src), .Ll_exc)
581 LDREST(t3, REST(0)(src), .Ll_exc_copy)
582 SUB t2, t2, t1 # t2 = number of bytes copied
584 STFIRST(t3, FIRST(0)(dst), .Ls_exc)
585 SLL t4, t1, 3 # t4 = number of bits to discard
586 SHIFT_DISCARD t3, t3, t4
587 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
592 beqz match, .Lboth_aligned
595 .Lsrc_unaligned_dst_aligned:
596 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
597 beqz t0, .Lcleanup_src_unaligned
598 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
601 * Avoid consecutive LD*'s to the same register since some mips
602 * implementations can't issue them in the same cycle.
603 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
604 * are to the same unit (unless src is aligned, but it's not).
606 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
607 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)
608 SUB len, len, 4*NBYTES
609 LDREST(t0, REST(0)(src), .Ll_exc_copy)
610 LDREST(t1, REST(1)(src), .Ll_exc_copy)
611 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)
612 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)
613 LDREST(t2, REST(2)(src), .Ll_exc_copy)
614 LDREST(t3, REST(3)(src), .Ll_exc_copy)
615 ADD src, src, 4*NBYTES
616 #ifdef CONFIG_CPU_SB1
617 nop # improves slotting
619 STORE(t0, UNIT(0)(dst), .Ls_exc)
621 STORE(t1, UNIT(1)(dst), .Ls_exc)
623 STORE(t2, UNIT(2)(dst), .Ls_exc)
625 STORE(t3, UNIT(3)(dst), .Ls_exc)
627 .set reorder /* DADDI_WAR */
628 ADD dst, dst, 4*NBYTES
632 .Lcleanup_src_unaligned:
634 and rem, len, NBYTES-1 # rem = len % NBYTES
635 beq rem, len, .Lcopy_bytes
638 LDFIRST(t0, FIRST(0)(src), .Ll_exc)
639 LDREST(t0, REST(0)(src), .Ll_exc_copy)
642 STORE(t0, 0(dst), .Ls_exc)
644 .set reorder /* DADDI_WAR */
649 .Lcopy_bytes_checklen:
653 /* 0 < len < NBYTES */
654 #ifdef CONFIG_CPU_LITTLE_ENDIAN
655 #define SHIFT_START 0
658 #define SHIFT_START 8*(NBYTES-1)
661 move t2, zero # partial word
662 li t3, SHIFT_START # shift
663 /* use .Ll_exc_copy here to return correct sum on fault */
664 #define COPY_BYTE(N) \
665 LOADBU(t0, N(src), .Ll_exc_copy); \
667 STOREB(t0, N(dst), .Ls_exc); \
669 addu t3, SHIFT_INC; \
670 beqz len, .Lcopy_bytes_done; \
681 LOADBU(t0, NBYTES-2(src), .Ll_exc_copy)
683 STOREB(t0, NBYTES-2(dst), .Ls_exc)
698 #ifdef CONFIG_CPU_MIPSR2
702 beqz odd, 1f /* odd buffer alignment? */
719 * Copy bytes from src until faulting load address (or until a
722 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
723 * may be more than a byte beyond the last address.
724 * Hence, the lb below may get an exception.
726 * Assumes src < THREAD_BUADDR($28)
728 LOADK t0, TI_TASK($28)
730 LOADK t0, THREAD_BUADDR(t0)
732 LOADBU(t1, 0(src), .Ll_exc)
734 sb t1, 0(dst) # can't fault -- we're copy_from_user
738 .set reorder /* DADDI_WAR */
743 LOADK t0, TI_TASK($28)
745 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
747 SUB len, AT, t0 # len number of uncopied bytes
749 * Here's where we rely on src and dst being incremented in tandem,
751 * dst += (fault addr - src) to put dst at first byte to clear
753 ADD dst, t0 # compute start address in a1
756 * Clear len bytes starting at dst. Can't call __bzero because it
757 * might modify len. An inefficient loop for these rare times...
759 .set reorder /* DADDI_WAR */
767 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
780 li v0, -1 /* invalid checksum */
785 END(__csum_partial_copy_kernel)