2 * Memory copy functions for 32-bit PowerPC.
4 * Copyright (C) 1996-2005 Paul Mackerras.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
11 #include <asm/processor.h>
12 #include <asm/cache.h>
13 #include <asm/errno.h>
14 #include <asm/ppc_asm.h>
16 #define COPY_16_BYTES \
26 #define COPY_16_BYTES_WITHEX(n) \
44 #define COPY_16_BYTES_EXCODE(n) \
46 addi r5,r5,-(16 * n); \
49 addi r5,r5,-(16 * n); \
51 .section __ex_table,"a"; \
53 .long 8 ## n ## 0b,9 ## n ## 0b; \
54 .long 8 ## n ## 1b,9 ## n ## 0b; \
55 .long 8 ## n ## 2b,9 ## n ## 0b; \
56 .long 8 ## n ## 3b,9 ## n ## 0b; \
57 .long 8 ## n ## 4b,9 ## n ## 1b; \
58 .long 8 ## n ## 5b,9 ## n ## 1b; \
59 .long 8 ## n ## 6b,9 ## n ## 1b; \
60 .long 8 ## n ## 7b,9 ## n ## 1b; \
64 .stabs "arch/powerpc/lib/",N_SO,0,0,0f
65 .stabs "copy_32.S",N_SO,0,0,0f
68 CACHELINE_BYTES = L1_CACHE_BYTES
69 LG_CACHELINE_BYTES = L1_CACHE_SHIFT
70 CACHELINE_MASK = (L1_CACHE_BYTES-1)
106 beq 2f /* if less than 8 bytes to do */
107 andi. r0,r6,3 /* get dest word aligned */
138 rlwinm. r7,r5,32-3,3,31
143 _GLOBAL(backwards_memcpy)
144 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
174 rlwinm. r7,r5,32-3,3,31
179 _GLOBAL(__copy_tofrom_user)
183 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
186 cmplw 0,r5,r0 /* is this more than total to do? */
187 blt 63f /* if not much to do */
188 andi. r8,r0,3 /* get it word-aligned first */
191 70: lbz r9,4(r4) /* do some bytes */
200 72: lwzu r9,4(r4) /* do some words */
204 .section __ex_table,"a"
212 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
213 clrlwi r5,r5,32-LG_CACHELINE_BYTES
217 /* Here we decide how far ahead to prefetch the source */
223 #if MAX_COPY_PREFETCH > 1
224 /* Heuristically, for large transfers we prefetch
225 MAX_COPY_PREFETCH cachelines ahead. For small transfers
226 we prefetch 1 cacheline ahead. */
227 cmpwi r0,MAX_COPY_PREFETCH
229 li r7,MAX_COPY_PREFETCH
232 addi r3,r3,CACHELINE_BYTES
236 addi r3,r3,CACHELINE_BYTES
237 #endif /* MAX_COPY_PREFETCH > 1 */
245 .section __ex_table,"a"
249 /* the main body of the cacheline loop */
250 COPY_16_BYTES_WITHEX(0)
251 #if L1_CACHE_BYTES >= 32
252 COPY_16_BYTES_WITHEX(1)
253 #if L1_CACHE_BYTES >= 64
254 COPY_16_BYTES_WITHEX(2)
255 COPY_16_BYTES_WITHEX(3)
256 #if L1_CACHE_BYTES >= 128
257 COPY_16_BYTES_WITHEX(4)
258 COPY_16_BYTES_WITHEX(5)
259 COPY_16_BYTES_WITHEX(6)
260 COPY_16_BYTES_WITHEX(7)
288 /* read fault, initial single-byte copy */
291 /* write fault, initial single-byte copy */
296 /* read fault, initial word copy */
299 /* write fault, initial word copy */
305 * this stuff handles faults in the cacheline loop and branches to either
306 * 104f (if in read part) or 105f (if in write part), after updating r5
308 COPY_16_BYTES_EXCODE(0)
309 #if L1_CACHE_BYTES >= 32
310 COPY_16_BYTES_EXCODE(1)
311 #if L1_CACHE_BYTES >= 64
312 COPY_16_BYTES_EXCODE(2)
313 COPY_16_BYTES_EXCODE(3)
314 #if L1_CACHE_BYTES >= 128
315 COPY_16_BYTES_EXCODE(4)
316 COPY_16_BYTES_EXCODE(5)
317 COPY_16_BYTES_EXCODE(6)
318 COPY_16_BYTES_EXCODE(7)
323 /* read fault in cacheline loop */
326 /* fault on dcbz (effectively a write fault) */
327 /* or write fault in cacheline loop */
329 92: li r3,LG_CACHELINE_BYTES
333 /* read fault in final word loop */
336 /* write fault in final word loop */
341 /* read fault in final byte loop */
344 /* write fault in final byte loop */
349 * At this stage the number of bytes not copied is
350 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
355 beq 120f /* shouldn't happen */
358 /* for a read fault, first try to continue the copy one byte at a time */
365 /* then clear out the destination: r3 bytes starting at 4(r6) */
381 .section __ex_table,"a"