0
|
1 /*
|
|
2 * Copyright 2003-2004 Sun Microsystems, Inc. All Rights Reserved.
|
|
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
4 *
|
|
5 * This code is free software; you can redistribute it and/or modify it
|
|
6 * under the terms of the GNU General Public License version 2 only, as
|
|
7 * published by the Free Software Foundation.
|
|
8 *
|
|
9 * This code is distributed in the hope that it will be useful, but WITHOUT
|
|
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
12 * version 2 for more details (a copy is included in the LICENSE file that
|
|
13 * accompanied this code).
|
|
14 *
|
|
15 * You should have received a copy of the GNU General Public License version
|
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
|
|
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
18 *
|
|
19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
|
|
20 * CA 95054 USA or visit www.sun.com if you need additional information or
|
|
21 * have any questions.
|
|
22 *
|
|
23 */
|
|
24
|
|
25 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
|
|
26 #ifdef AMD64
|
|
27 (void)memmove(to, from, count * HeapWordSize);
|
|
28 #else
|
|
29 // Same as pd_aligned_conjoint_words, except includes a zero-count check.
|
|
30 intx temp;
|
|
31 __asm__ volatile(" testl %6,%6 ;"
|
|
32 " jz 7f ;"
|
|
33 " cmpl %4,%5 ;"
|
|
34 " leal -4(%4,%6,4),%3;"
|
|
35 " jbe 1f ;"
|
|
36 " cmpl %7,%5 ;"
|
|
37 " jbe 4f ;"
|
|
38 "1: cmpl $32,%6 ;"
|
|
39 " ja 3f ;"
|
|
40 " subl %4,%1 ;"
|
|
41 "2: movl (%4),%3 ;"
|
|
42 " movl %7,(%5,%4,1) ;"
|
|
43 " addl $4,%0 ;"
|
|
44 " subl $1,%2 ;"
|
|
45 " jnz 2b ;"
|
|
46 " jmp 7f ;"
|
|
47 "3: rep; smovl ;"
|
|
48 " jmp 7f ;"
|
|
49 "4: cmpl $32,%2 ;"
|
|
50 " movl %7,%0 ;"
|
|
51 " leal -4(%5,%6,4),%1;"
|
|
52 " ja 6f ;"
|
|
53 " subl %4,%1 ;"
|
|
54 "5: movl (%4),%3 ;"
|
|
55 " movl %7,(%5,%4,1) ;"
|
|
56 " subl $4,%0 ;"
|
|
57 " subl $1,%2 ;"
|
|
58 " jnz 5b ;"
|
|
59 " jmp 7f ;"
|
|
60 "6: std ;"
|
|
61 " rep; smovl ;"
|
|
62 " cld ;"
|
|
63 "7: nop "
|
|
64 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
|
|
65 : "0" (from), "1" (to), "2" (count), "3" (temp)
|
|
66 : "memory", "flags");
|
|
67 #endif // AMD64
|
|
68 }
|
|
69
|
|
70 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
|
|
71 #ifdef AMD64
|
|
72 switch (count) {
|
|
73 case 8: to[7] = from[7];
|
|
74 case 7: to[6] = from[6];
|
|
75 case 6: to[5] = from[5];
|
|
76 case 5: to[4] = from[4];
|
|
77 case 4: to[3] = from[3];
|
|
78 case 3: to[2] = from[2];
|
|
79 case 2: to[1] = from[1];
|
|
80 case 1: to[0] = from[0];
|
|
81 case 0: break;
|
|
82 default:
|
|
83 (void)memcpy(to, from, count * HeapWordSize);
|
|
84 break;
|
|
85 }
|
|
86 #else
|
|
87 // Same as pd_aligned_disjoint_words, except includes a zero-count check.
|
|
88 intx temp;
|
|
89 __asm__ volatile(" testl %6,%6 ;"
|
|
90 " jz 3f ;"
|
|
91 " cmpl $32,%6 ;"
|
|
92 " ja 2f ;"
|
|
93 " subl %4,%1 ;"
|
|
94 "1: movl (%4),%3 ;"
|
|
95 " movl %7,(%5,%4,1);"
|
|
96 " addl $4,%0 ;"
|
|
97 " subl $1,%2 ;"
|
|
98 " jnz 1b ;"
|
|
99 " jmp 3f ;"
|
|
100 "2: rep; smovl ;"
|
|
101 "3: nop "
|
|
102 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
|
|
103 : "0" (from), "1" (to), "2" (count), "3" (temp)
|
|
104 : "memory", "cc");
|
|
105 #endif // AMD64
|
|
106 }
|
|
107
|
|
108 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
|
|
109 #ifdef AMD64
|
|
110 switch (count) {
|
|
111 case 8: to[7] = from[7];
|
|
112 case 7: to[6] = from[6];
|
|
113 case 6: to[5] = from[5];
|
|
114 case 5: to[4] = from[4];
|
|
115 case 4: to[3] = from[3];
|
|
116 case 3: to[2] = from[2];
|
|
117 case 2: to[1] = from[1];
|
|
118 case 1: to[0] = from[0];
|
|
119 case 0: break;
|
|
120 default:
|
|
121 while (count-- > 0) {
|
|
122 *to++ = *from++;
|
|
123 }
|
|
124 break;
|
|
125 }
|
|
126 #else
|
|
127 // pd_disjoint_words is word-atomic in this implementation.
|
|
128 pd_disjoint_words(from, to, count);
|
|
129 #endif // AMD64
|
|
130 }
|
|
131
|
|
132 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
|
|
133 #ifdef AMD64
|
|
134 (void)memmove(to, from, count * HeapWordSize);
|
|
135 #else
|
|
136 // Same as pd_conjoint_words, except no zero-count check.
|
|
137 intx temp;
|
|
138 __asm__ volatile(" cmpl %4,%5 ;"
|
|
139 " leal -4(%4,%6,4),%3;"
|
|
140 " jbe 1f ;"
|
|
141 " cmpl %7,%5 ;"
|
|
142 " jbe 4f ;"
|
|
143 "1: cmpl $32,%6 ;"
|
|
144 " ja 3f ;"
|
|
145 " subl %4,%1 ;"
|
|
146 "2: movl (%4),%3 ;"
|
|
147 " movl %7,(%5,%4,1) ;"
|
|
148 " addl $4,%0 ;"
|
|
149 " subl $1,%2 ;"
|
|
150 " jnz 2b ;"
|
|
151 " jmp 7f ;"
|
|
152 "3: rep; smovl ;"
|
|
153 " jmp 7f ;"
|
|
154 "4: cmpl $32,%2 ;"
|
|
155 " movl %7,%0 ;"
|
|
156 " leal -4(%5,%6,4),%1;"
|
|
157 " ja 6f ;"
|
|
158 " subl %4,%1 ;"
|
|
159 "5: movl (%4),%3 ;"
|
|
160 " movl %7,(%5,%4,1) ;"
|
|
161 " subl $4,%0 ;"
|
|
162 " subl $1,%2 ;"
|
|
163 " jnz 5b ;"
|
|
164 " jmp 7f ;"
|
|
165 "6: std ;"
|
|
166 " rep; smovl ;"
|
|
167 " cld ;"
|
|
168 "7: nop "
|
|
169 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
|
|
170 : "0" (from), "1" (to), "2" (count), "3" (temp)
|
|
171 : "memory", "flags");
|
|
172 #endif // AMD64
|
|
173 }
|
|
174
|
|
175 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
|
|
176 #ifdef AMD64
|
|
177 pd_disjoint_words(from, to, count);
|
|
178 #else
|
|
179 // Same as pd_disjoint_words, except no zero-count check.
|
|
180 intx temp;
|
|
181 __asm__ volatile(" cmpl $32,%6 ;"
|
|
182 " ja 2f ;"
|
|
183 " subl %4,%1 ;"
|
|
184 "1: movl (%4),%3 ;"
|
|
185 " movl %7,(%5,%4,1);"
|
|
186 " addl $4,%0 ;"
|
|
187 " subl $1,%2 ;"
|
|
188 " jnz 1b ;"
|
|
189 " jmp 3f ;"
|
|
190 "2: rep; smovl ;"
|
|
191 "3: nop "
|
|
192 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
|
|
193 : "0" (from), "1" (to), "2" (count), "3" (temp)
|
|
194 : "memory", "cc");
|
|
195 #endif // AMD64
|
|
196 }
|
|
197
|
|
198 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
|
|
199 #ifdef AMD64
|
|
200 (void)memmove(to, from, count);
|
|
201 #else
|
|
202 intx temp;
|
|
203 __asm__ volatile(" testl %6,%6 ;"
|
|
204 " jz 13f ;"
|
|
205 " cmpl %4,%5 ;"
|
|
206 " leal -1(%4,%6),%3 ;"
|
|
207 " jbe 1f ;"
|
|
208 " cmpl %7,%5 ;"
|
|
209 " jbe 8f ;"
|
|
210 "1: cmpl $3,%6 ;"
|
|
211 " jbe 6f ;"
|
|
212 " movl %6,%3 ;"
|
|
213 " movl $4,%2 ;"
|
|
214 " subl %4,%2 ;"
|
|
215 " andl $3,%2 ;"
|
|
216 " jz 2f ;"
|
|
217 " subl %6,%3 ;"
|
|
218 " rep; smovb ;"
|
|
219 "2: movl %7,%2 ;"
|
|
220 " shrl $2,%2 ;"
|
|
221 " jz 5f ;"
|
|
222 " cmpl $32,%2 ;"
|
|
223 " ja 4f ;"
|
|
224 " subl %4,%1 ;"
|
|
225 "3: movl (%4),%%edx ;"
|
|
226 " movl %%edx,(%5,%4,1);"
|
|
227 " addl $4,%0 ;"
|
|
228 " subl $1,%2 ;"
|
|
229 " jnz 3b ;"
|
|
230 " addl %4,%1 ;"
|
|
231 " jmp 5f ;"
|
|
232 "4: rep; smovl ;"
|
|
233 "5: movl %7,%2 ;"
|
|
234 " andl $3,%2 ;"
|
|
235 " jz 13f ;"
|
|
236 "6: xorl %7,%3 ;"
|
|
237 "7: movb (%4,%7,1),%%dl ;"
|
|
238 " movb %%dl,(%5,%7,1) ;"
|
|
239 " addl $1,%3 ;"
|
|
240 " subl $1,%2 ;"
|
|
241 " jnz 7b ;"
|
|
242 " jmp 13f ;"
|
|
243 "8: std ;"
|
|
244 " cmpl $12,%2 ;"
|
|
245 " ja 9f ;"
|
|
246 " movl %7,%0 ;"
|
|
247 " leal -1(%6,%5),%1 ;"
|
|
248 " jmp 11f ;"
|
|
249 "9: xchgl %3,%2 ;"
|
|
250 " movl %6,%0 ;"
|
|
251 " addl $1,%2 ;"
|
|
252 " leal -1(%7,%5),%1 ;"
|
|
253 " andl $3,%2 ;"
|
|
254 " jz 10f ;"
|
|
255 " subl %6,%3 ;"
|
|
256 " rep; smovb ;"
|
|
257 "10: movl %7,%2 ;"
|
|
258 " subl $3,%0 ;"
|
|
259 " shrl $2,%2 ;"
|
|
260 " subl $3,%1 ;"
|
|
261 " rep; smovl ;"
|
|
262 " andl $3,%3 ;"
|
|
263 " jz 12f ;"
|
|
264 " movl %7,%2 ;"
|
|
265 " addl $3,%0 ;"
|
|
266 " addl $3,%1 ;"
|
|
267 "11: rep; smovb ;"
|
|
268 "12: cld ;"
|
|
269 "13: nop ;"
|
|
270 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
|
|
271 : "0" (from), "1" (to), "2" (count), "3" (temp)
|
|
272 : "memory", "flags", "%edx");
|
|
273 #endif // AMD64
|
|
274 }
|
|
275
|
|
276 static void pd_conjoint_bytes_atomic(void* from, void* to, size_t count) {
|
|
277 pd_conjoint_bytes(from, to, count);
|
|
278 }
|
|
279
|
|
280 static void pd_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) {
|
|
281 _Copy_conjoint_jshorts_atomic(from, to, count);
|
|
282 }
|
|
283
|
|
284 static void pd_conjoint_jints_atomic(jint* from, jint* to, size_t count) {
|
|
285 #ifdef AMD64
|
|
286 _Copy_conjoint_jints_atomic(from, to, count);
|
|
287 #else
|
|
288 assert(HeapWordSize == BytesPerInt, "heapwords and jints must be the same size");
|
|
289 // pd_conjoint_words is word-atomic in this implementation.
|
|
290 pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count);
|
|
291 #endif // AMD64
|
|
292 }
|
|
293
|
|
294 static void pd_conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) {
|
|
295 #ifdef AMD64
|
|
296 _Copy_conjoint_jlongs_atomic(from, to, count);
|
|
297 #else
|
|
298 // Guarantee use of fild/fistp or xmm regs via some asm code, because compilers won't.
|
|
299 if (from > to) {
|
|
300 while (count-- > 0) {
|
|
301 __asm__ volatile("fildll (%0); fistpll (%1)"
|
|
302 :
|
|
303 : "r" (from), "r" (to)
|
|
304 : "memory" );
|
|
305 ++from;
|
|
306 ++to;
|
|
307 }
|
|
308 } else {
|
|
309 while (count-- > 0) {
|
|
310 __asm__ volatile("fildll (%0,%2,8); fistpll (%1,%2,8)"
|
|
311 :
|
|
312 : "r" (from), "r" (to), "r" (count)
|
|
313 : "memory" );
|
|
314 }
|
|
315 }
|
|
316 #endif // AMD64
|
|
317 }
|
|
318
|
|
319 static void pd_conjoint_oops_atomic(oop* from, oop* to, size_t count) {
|
|
320 #ifdef AMD64
|
|
321 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
|
|
322 _Copy_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
|
|
323 #else
|
|
324 assert(HeapWordSize == BytesPerOop, "heapwords and oops must be the same size");
|
|
325 // pd_conjoint_words is word-atomic in this implementation.
|
|
326 pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count);
|
|
327 #endif // AMD64
|
|
328 }
|
|
329
|
|
330 static void pd_arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) {
|
|
331 _Copy_arrayof_conjoint_bytes(from, to, count);
|
|
332 }
|
|
333
|
|
334 static void pd_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) {
|
|
335 _Copy_arrayof_conjoint_jshorts(from, to, count);
|
|
336 }
|
|
337
|
|
338 static void pd_arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) {
|
|
339 #ifdef AMD64
|
|
340 _Copy_arrayof_conjoint_jints(from, to, count);
|
|
341 #else
|
|
342 pd_conjoint_jints_atomic((jint*)from, (jint*)to, count);
|
|
343 #endif // AMD64
|
|
344 }
|
|
345
|
|
346 static void pd_arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) {
|
|
347 #ifdef AMD64
|
|
348 _Copy_arrayof_conjoint_jlongs(from, to, count);
|
|
349 #else
|
|
350 pd_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
|
|
351 #endif // AMD64
|
|
352 }
|
|
353
|
|
354 static void pd_arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) {
|
|
355 #ifdef AMD64
|
|
356 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
|
|
357 _Copy_arrayof_conjoint_jlongs(from, to, count);
|
|
358 #else
|
|
359 pd_conjoint_oops_atomic((oop*)from, (oop*)to, count);
|
|
360 #endif // AMD64
|
|
361 }
|