Mercurial > hg > truffle
comparison src/os_cpu/linux_x86/vm/copy_linux_x86.inline.hpp @ 0:a61af66fc99e jdk7-b24
Initial load
author | duke |
---|---|
date | Sat, 01 Dec 2007 00:00:00 +0000 |
parents | |
children | c18cbe5936b8 |
comparison
equal
deleted
inserted
replaced
-1:000000000000 | 0:a61af66fc99e |
---|---|
1 /* | |
2 * Copyright 2003-2004 Sun Microsystems, Inc. All Rights Reserved. | |
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
4 * | |
5 * This code is free software; you can redistribute it and/or modify it | |
6 * under the terms of the GNU General Public License version 2 only, as | |
7 * published by the Free Software Foundation. | |
8 * | |
9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 * version 2 for more details (a copy is included in the LICENSE file that | |
13 * accompanied this code). | |
14 * | |
15 * You should have received a copy of the GNU General Public License version | |
16 * 2 along with this work; if not, write to the Free Software Foundation, | |
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
18 * | |
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
20 * CA 95054 USA or visit www.sun.com if you need additional information or | |
21 * have any questions. | |
22 * | |
23 */ | |
24 | |
25 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) { | |
26 #ifdef AMD64 | |
27 (void)memmove(to, from, count * HeapWordSize); | |
28 #else | |
29 // Same as pd_aligned_conjoint_words, except includes a zero-count check. | |
30 intx temp; | |
31 __asm__ volatile(" testl %6,%6 ;" | |
32 " jz 7f ;" | |
33 " cmpl %4,%5 ;" | |
34 " leal -4(%4,%6,4),%3;" | |
35 " jbe 1f ;" | |
36 " cmpl %7,%5 ;" | |
37 " jbe 4f ;" | |
38 "1: cmpl $32,%6 ;" | |
39 " ja 3f ;" | |
40 " subl %4,%1 ;" | |
41 "2: movl (%4),%3 ;" | |
42 " movl %7,(%5,%4,1) ;" | |
43 " addl $4,%0 ;" | |
44 " subl $1,%2 ;" | |
45 " jnz 2b ;" | |
46 " jmp 7f ;" | |
47 "3: rep; smovl ;" | |
48 " jmp 7f ;" | |
49 "4: cmpl $32,%2 ;" | |
50 " movl %7,%0 ;" | |
51 " leal -4(%5,%6,4),%1;" | |
52 " ja 6f ;" | |
53 " subl %4,%1 ;" | |
54 "5: movl (%4),%3 ;" | |
55 " movl %7,(%5,%4,1) ;" | |
56 " subl $4,%0 ;" | |
57 " subl $1,%2 ;" | |
58 " jnz 5b ;" | |
59 " jmp 7f ;" | |
60 "6: std ;" | |
61 " rep; smovl ;" | |
62 " cld ;" | |
63 "7: nop " | |
64 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp) | |
65 : "0" (from), "1" (to), "2" (count), "3" (temp) | |
66 : "memory", "flags"); | |
67 #endif // AMD64 | |
68 } | |
69 | |
70 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) { | |
71 #ifdef AMD64 | |
72 switch (count) { | |
73 case 8: to[7] = from[7]; | |
74 case 7: to[6] = from[6]; | |
75 case 6: to[5] = from[5]; | |
76 case 5: to[4] = from[4]; | |
77 case 4: to[3] = from[3]; | |
78 case 3: to[2] = from[2]; | |
79 case 2: to[1] = from[1]; | |
80 case 1: to[0] = from[0]; | |
81 case 0: break; | |
82 default: | |
83 (void)memcpy(to, from, count * HeapWordSize); | |
84 break; | |
85 } | |
86 #else | |
87 // Same as pd_aligned_disjoint_words, except includes a zero-count check. | |
88 intx temp; | |
89 __asm__ volatile(" testl %6,%6 ;" | |
90 " jz 3f ;" | |
91 " cmpl $32,%6 ;" | |
92 " ja 2f ;" | |
93 " subl %4,%1 ;" | |
94 "1: movl (%4),%3 ;" | |
95 " movl %7,(%5,%4,1);" | |
96 " addl $4,%0 ;" | |
97 " subl $1,%2 ;" | |
98 " jnz 1b ;" | |
99 " jmp 3f ;" | |
100 "2: rep; smovl ;" | |
101 "3: nop " | |
102 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp) | |
103 : "0" (from), "1" (to), "2" (count), "3" (temp) | |
104 : "memory", "cc"); | |
105 #endif // AMD64 | |
106 } | |
107 | |
108 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) { | |
109 #ifdef AMD64 | |
110 switch (count) { | |
111 case 8: to[7] = from[7]; | |
112 case 7: to[6] = from[6]; | |
113 case 6: to[5] = from[5]; | |
114 case 5: to[4] = from[4]; | |
115 case 4: to[3] = from[3]; | |
116 case 3: to[2] = from[2]; | |
117 case 2: to[1] = from[1]; | |
118 case 1: to[0] = from[0]; | |
119 case 0: break; | |
120 default: | |
121 while (count-- > 0) { | |
122 *to++ = *from++; | |
123 } | |
124 break; | |
125 } | |
126 #else | |
127 // pd_disjoint_words is word-atomic in this implementation. | |
128 pd_disjoint_words(from, to, count); | |
129 #endif // AMD64 | |
130 } | |
131 | |
132 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) { | |
133 #ifdef AMD64 | |
134 (void)memmove(to, from, count * HeapWordSize); | |
135 #else | |
136 // Same as pd_conjoint_words, except no zero-count check. | |
137 intx temp; | |
138 __asm__ volatile(" cmpl %4,%5 ;" | |
139 " leal -4(%4,%6,4),%3;" | |
140 " jbe 1f ;" | |
141 " cmpl %7,%5 ;" | |
142 " jbe 4f ;" | |
143 "1: cmpl $32,%6 ;" | |
144 " ja 3f ;" | |
145 " subl %4,%1 ;" | |
146 "2: movl (%4),%3 ;" | |
147 " movl %7,(%5,%4,1) ;" | |
148 " addl $4,%0 ;" | |
149 " subl $1,%2 ;" | |
150 " jnz 2b ;" | |
151 " jmp 7f ;" | |
152 "3: rep; smovl ;" | |
153 " jmp 7f ;" | |
154 "4: cmpl $32,%2 ;" | |
155 " movl %7,%0 ;" | |
156 " leal -4(%5,%6,4),%1;" | |
157 " ja 6f ;" | |
158 " subl %4,%1 ;" | |
159 "5: movl (%4),%3 ;" | |
160 " movl %7,(%5,%4,1) ;" | |
161 " subl $4,%0 ;" | |
162 " subl $1,%2 ;" | |
163 " jnz 5b ;" | |
164 " jmp 7f ;" | |
165 "6: std ;" | |
166 " rep; smovl ;" | |
167 " cld ;" | |
168 "7: nop " | |
169 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp) | |
170 : "0" (from), "1" (to), "2" (count), "3" (temp) | |
171 : "memory", "flags"); | |
172 #endif // AMD64 | |
173 } | |
174 | |
175 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) { | |
176 #ifdef AMD64 | |
177 pd_disjoint_words(from, to, count); | |
178 #else | |
179 // Same as pd_disjoint_words, except no zero-count check. | |
180 intx temp; | |
181 __asm__ volatile(" cmpl $32,%6 ;" | |
182 " ja 2f ;" | |
183 " subl %4,%1 ;" | |
184 "1: movl (%4),%3 ;" | |
185 " movl %7,(%5,%4,1);" | |
186 " addl $4,%0 ;" | |
187 " subl $1,%2 ;" | |
188 " jnz 1b ;" | |
189 " jmp 3f ;" | |
190 "2: rep; smovl ;" | |
191 "3: nop " | |
192 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp) | |
193 : "0" (from), "1" (to), "2" (count), "3" (temp) | |
194 : "memory", "cc"); | |
195 #endif // AMD64 | |
196 } | |
197 | |
198 static void pd_conjoint_bytes(void* from, void* to, size_t count) { | |
199 #ifdef AMD64 | |
200 (void)memmove(to, from, count); | |
201 #else | |
202 intx temp; | |
203 __asm__ volatile(" testl %6,%6 ;" | |
204 " jz 13f ;" | |
205 " cmpl %4,%5 ;" | |
206 " leal -1(%4,%6),%3 ;" | |
207 " jbe 1f ;" | |
208 " cmpl %7,%5 ;" | |
209 " jbe 8f ;" | |
210 "1: cmpl $3,%6 ;" | |
211 " jbe 6f ;" | |
212 " movl %6,%3 ;" | |
213 " movl $4,%2 ;" | |
214 " subl %4,%2 ;" | |
215 " andl $3,%2 ;" | |
216 " jz 2f ;" | |
217 " subl %6,%3 ;" | |
218 " rep; smovb ;" | |
219 "2: movl %7,%2 ;" | |
220 " shrl $2,%2 ;" | |
221 " jz 5f ;" | |
222 " cmpl $32,%2 ;" | |
223 " ja 4f ;" | |
224 " subl %4,%1 ;" | |
225 "3: movl (%4),%%edx ;" | |
226 " movl %%edx,(%5,%4,1);" | |
227 " addl $4,%0 ;" | |
228 " subl $1,%2 ;" | |
229 " jnz 3b ;" | |
230 " addl %4,%1 ;" | |
231 " jmp 5f ;" | |
232 "4: rep; smovl ;" | |
233 "5: movl %7,%2 ;" | |
234 " andl $3,%2 ;" | |
235 " jz 13f ;" | |
236 "6: xorl %7,%3 ;" | |
237 "7: movb (%4,%7,1),%%dl ;" | |
238 " movb %%dl,(%5,%7,1) ;" | |
239 " addl $1,%3 ;" | |
240 " subl $1,%2 ;" | |
241 " jnz 7b ;" | |
242 " jmp 13f ;" | |
243 "8: std ;" | |
244 " cmpl $12,%2 ;" | |
245 " ja 9f ;" | |
246 " movl %7,%0 ;" | |
247 " leal -1(%6,%5),%1 ;" | |
248 " jmp 11f ;" | |
249 "9: xchgl %3,%2 ;" | |
250 " movl %6,%0 ;" | |
251 " addl $1,%2 ;" | |
252 " leal -1(%7,%5),%1 ;" | |
253 " andl $3,%2 ;" | |
254 " jz 10f ;" | |
255 " subl %6,%3 ;" | |
256 " rep; smovb ;" | |
257 "10: movl %7,%2 ;" | |
258 " subl $3,%0 ;" | |
259 " shrl $2,%2 ;" | |
260 " subl $3,%1 ;" | |
261 " rep; smovl ;" | |
262 " andl $3,%3 ;" | |
263 " jz 12f ;" | |
264 " movl %7,%2 ;" | |
265 " addl $3,%0 ;" | |
266 " addl $3,%1 ;" | |
267 "11: rep; smovb ;" | |
268 "12: cld ;" | |
269 "13: nop ;" | |
270 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp) | |
271 : "0" (from), "1" (to), "2" (count), "3" (temp) | |
272 : "memory", "flags", "%edx"); | |
273 #endif // AMD64 | |
274 } | |
275 | |
276 static void pd_conjoint_bytes_atomic(void* from, void* to, size_t count) { | |
277 pd_conjoint_bytes(from, to, count); | |
278 } | |
279 | |
280 static void pd_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) { | |
281 _Copy_conjoint_jshorts_atomic(from, to, count); | |
282 } | |
283 | |
284 static void pd_conjoint_jints_atomic(jint* from, jint* to, size_t count) { | |
285 #ifdef AMD64 | |
286 _Copy_conjoint_jints_atomic(from, to, count); | |
287 #else | |
288 assert(HeapWordSize == BytesPerInt, "heapwords and jints must be the same size"); | |
289 // pd_conjoint_words is word-atomic in this implementation. | |
290 pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count); | |
291 #endif // AMD64 | |
292 } | |
293 | |
294 static void pd_conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) { | |
295 #ifdef AMD64 | |
296 _Copy_conjoint_jlongs_atomic(from, to, count); | |
297 #else | |
298 // Guarantee use of fild/fistp or xmm regs via some asm code, because compilers won't. | |
299 if (from > to) { | |
300 while (count-- > 0) { | |
301 __asm__ volatile("fildll (%0); fistpll (%1)" | |
302 : | |
303 : "r" (from), "r" (to) | |
304 : "memory" ); | |
305 ++from; | |
306 ++to; | |
307 } | |
308 } else { | |
309 while (count-- > 0) { | |
310 __asm__ volatile("fildll (%0,%2,8); fistpll (%1,%2,8)" | |
311 : | |
312 : "r" (from), "r" (to), "r" (count) | |
313 : "memory" ); | |
314 } | |
315 } | |
316 #endif // AMD64 | |
317 } | |
318 | |
319 static void pd_conjoint_oops_atomic(oop* from, oop* to, size_t count) { | |
320 #ifdef AMD64 | |
321 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size"); | |
322 _Copy_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count); | |
323 #else | |
324 assert(HeapWordSize == BytesPerOop, "heapwords and oops must be the same size"); | |
325 // pd_conjoint_words is word-atomic in this implementation. | |
326 pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count); | |
327 #endif // AMD64 | |
328 } | |
329 | |
330 static void pd_arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) { | |
331 _Copy_arrayof_conjoint_bytes(from, to, count); | |
332 } | |
333 | |
334 static void pd_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) { | |
335 _Copy_arrayof_conjoint_jshorts(from, to, count); | |
336 } | |
337 | |
338 static void pd_arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) { | |
339 #ifdef AMD64 | |
340 _Copy_arrayof_conjoint_jints(from, to, count); | |
341 #else | |
342 pd_conjoint_jints_atomic((jint*)from, (jint*)to, count); | |
343 #endif // AMD64 | |
344 } | |
345 | |
346 static void pd_arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) { | |
347 #ifdef AMD64 | |
348 _Copy_arrayof_conjoint_jlongs(from, to, count); | |
349 #else | |
350 pd_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count); | |
351 #endif // AMD64 | |
352 } | |
353 | |
354 static void pd_arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) { | |
355 #ifdef AMD64 | |
356 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size"); | |
357 _Copy_arrayof_conjoint_jlongs(from, to, count); | |
358 #else | |
359 pd_conjoint_oops_atomic((oop*)from, (oop*)to, count); | |
360 #endif // AMD64 | |
361 } |