|
3960
|
1 #
|
|
17524
|
2 # Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
|
|
3960
|
3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
|
4 #
|
|
|
5 # This code is free software; you can redistribute it and/or modify it
|
|
|
6 # under the terms of the GNU General Public License version 2 only, as
|
|
|
7 # published by the Free Software Foundation.
|
|
|
8 #
|
|
|
9 # This code is distributed in the hope that it will be useful, but WITHOUT
|
|
|
10 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
11 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
12 # version 2 for more details (a copy is included in the LICENSE file that
|
|
|
13 # accompanied this code).
|
|
|
14 #
|
|
|
15 # You should have received a copy of the GNU General Public License version
|
|
|
16 # 2 along with this work; if not, write to the Free Software Foundation,
|
|
|
17 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
18 #
|
|
|
19 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
|
20 # or visit www.oracle.com if you need additional information or have any
|
|
|
21 # questions.
|
|
|
22 #
|
|
|
23
|
|
|
24 #ifdef __APPLE__
|
|
|
25 # Darwin uses _ prefixed global symbols
|
|
|
26 #define SYMBOL(s) _ ## s
|
|
|
27 #define ELF_TYPE(name, description)
|
|
|
28 #else
|
|
|
29 #define SYMBOL(s) s
|
|
|
30 #define ELF_TYPE(name, description) .type name,description
|
|
|
31 #endif
|
|
|
32
|
|
|
33 # NOTE WELL! The _Copy functions are called directly
|
|
|
34 # from server-compiler-generated code via CallLeafNoFP,
|
|
|
35 # which means that they *must* either not use floating
|
|
|
36 # point or use it in the same manner as does the server
|
|
|
37 # compiler.
|
|
|
38
|
|
|
39 .globl SYMBOL(_Copy_arrayof_conjoint_bytes)
|
|
|
40 .globl SYMBOL(_Copy_arrayof_conjoint_jshorts)
|
|
|
41 .globl SYMBOL(_Copy_conjoint_jshorts_atomic)
|
|
|
42 .globl SYMBOL(_Copy_arrayof_conjoint_jints)
|
|
|
43 .globl SYMBOL(_Copy_conjoint_jints_atomic)
|
|
|
44 .globl SYMBOL(_Copy_arrayof_conjoint_jlongs)
|
|
|
45 .globl SYMBOL(_Copy_conjoint_jlongs_atomic)
|
|
|
46
|
|
|
47 .text
|
|
|
48
|
|
|
49 .globl SYMBOL(SpinPause)
|
|
|
50 .p2align 4,,15
|
|
|
51 ELF_TYPE(SpinPause,@function)
|
|
|
52 SYMBOL(SpinPause):
|
|
|
53 rep
|
|
|
54 nop
|
|
|
55 movq $1, %rax
|
|
|
56 ret
|
|
|
57
|
|
|
58 # Support for void Copy::arrayof_conjoint_bytes(void* from,
|
|
|
59 # void* to,
|
|
|
60 # size_t count)
|
|
|
61 # rdi - from
|
|
|
62 # rsi - to
|
|
|
63 # rdx - count, treated as ssize_t
|
|
|
64 #
|
|
|
65 .p2align 4,,15
|
|
|
66 ELF_TYPE(_Copy_arrayof_conjoint_bytes,@function)
|
|
|
67 SYMBOL(_Copy_arrayof_conjoint_bytes):
|
|
|
68 movq %rdx,%r8 # byte count
|
|
|
69 shrq $3,%rdx # qword count
|
|
|
70 cmpq %rdi,%rsi
|
|
|
71 leaq -1(%rdi,%r8,1),%rax # from + bcount*1 - 1
|
|
|
72 jbe acb_CopyRight
|
|
|
73 cmpq %rax,%rsi
|
|
|
74 jbe acb_CopyLeft
|
|
|
75 acb_CopyRight:
|
|
|
76 leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
|
|
|
77 leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
|
|
|
78 negq %rdx
|
|
|
79 jmp 7f
|
|
|
80 .p2align 4,,15
|
|
|
81 1: movq 8(%rax,%rdx,8),%rsi
|
|
|
82 movq %rsi,8(%rcx,%rdx,8)
|
|
|
83 addq $1,%rdx
|
|
|
84 jnz 1b
|
|
|
85 2: testq $4,%r8 # check for trailing dword
|
|
|
86 jz 3f
|
|
|
87 movl 8(%rax),%esi # copy trailing dword
|
|
|
88 movl %esi,8(%rcx)
|
|
|
89 addq $4,%rax
|
|
|
90 addq $4,%rcx # original %rsi is trashed, so we
|
|
|
91 # can't use it as a base register
|
|
|
92 3: testq $2,%r8 # check for trailing word
|
|
|
93 jz 4f
|
|
|
94 movw 8(%rax),%si # copy trailing word
|
|
|
95 movw %si,8(%rcx)
|
|
|
96 addq $2,%rcx
|
|
|
97 4: testq $1,%r8 # check for trailing byte
|
|
|
98 jz 5f
|
|
|
99 movb -1(%rdi,%r8,1),%al # copy trailing byte
|
|
|
100 movb %al,8(%rcx)
|
|
|
101 5: ret
|
|
|
102 .p2align 4,,15
|
|
|
103 6: movq -24(%rax,%rdx,8),%rsi
|
|
|
104 movq %rsi,-24(%rcx,%rdx,8)
|
|
|
105 movq -16(%rax,%rdx,8),%rsi
|
|
|
106 movq %rsi,-16(%rcx,%rdx,8)
|
|
|
107 movq -8(%rax,%rdx,8),%rsi
|
|
|
108 movq %rsi,-8(%rcx,%rdx,8)
|
|
|
109 movq (%rax,%rdx,8),%rsi
|
|
|
110 movq %rsi,(%rcx,%rdx,8)
|
|
|
111 7: addq $4,%rdx
|
|
|
112 jle 6b
|
|
|
113 subq $4,%rdx
|
|
|
114 jl 1b
|
|
|
115 jmp 2b
|
|
|
116 acb_CopyLeft:
|
|
|
117 testq $1,%r8 # check for trailing byte
|
|
|
118 jz 1f
|
|
|
119 movb -1(%rdi,%r8,1),%cl # copy trailing byte
|
|
|
120 movb %cl,-1(%rsi,%r8,1)
|
|
|
121 subq $1,%r8 # adjust for possible trailing word
|
|
|
122 1: testq $2,%r8 # check for trailing word
|
|
|
123 jz 2f
|
|
|
124 movw -2(%rdi,%r8,1),%cx # copy trailing word
|
|
|
125 movw %cx,-2(%rsi,%r8,1)
|
|
|
126 2: testq $4,%r8 # check for trailing dword
|
|
|
127 jz 5f
|
|
|
128 movl (%rdi,%rdx,8),%ecx # copy trailing dword
|
|
|
129 movl %ecx,(%rsi,%rdx,8)
|
|
|
130 jmp 5f
|
|
|
131 .p2align 4,,15
|
|
|
132 3: movq -8(%rdi,%rdx,8),%rcx
|
|
|
133 movq %rcx,-8(%rsi,%rdx,8)
|
|
|
134 subq $1,%rdx
|
|
|
135 jnz 3b
|
|
|
136 ret
|
|
|
137 .p2align 4,,15
|
|
|
138 4: movq 24(%rdi,%rdx,8),%rcx
|
|
|
139 movq %rcx,24(%rsi,%rdx,8)
|
|
|
140 movq 16(%rdi,%rdx,8),%rcx
|
|
|
141 movq %rcx,16(%rsi,%rdx,8)
|
|
|
142 movq 8(%rdi,%rdx,8),%rcx
|
|
|
143 movq %rcx,8(%rsi,%rdx,8)
|
|
|
144 movq (%rdi,%rdx,8),%rcx
|
|
|
145 movq %rcx,(%rsi,%rdx,8)
|
|
|
146 5: subq $4,%rdx
|
|
|
147 jge 4b
|
|
|
148 addq $4,%rdx
|
|
|
149 jg 3b
|
|
|
150 ret
|
|
|
151
|
|
|
152 # Support for void Copy::arrayof_conjoint_jshorts(void* from,
|
|
|
153 # void* to,
|
|
|
154 # size_t count)
|
|
|
155 # Equivalent to
|
|
|
156 # conjoint_jshorts_atomic
|
|
|
157 #
|
|
|
158 # If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
|
|
|
159 # let the hardware handle it. The tow or four words within dwords
|
|
|
160 # or qwords that span cache line boundaries will still be loaded
|
|
|
161 # and stored atomically.
|
|
|
162 #
|
|
|
163 # rdi - from
|
|
|
164 # rsi - to
|
|
|
165 # rdx - count, treated as ssize_t
|
|
|
166 #
|
|
|
167 .p2align 4,,15
|
|
|
168 ELF_TYPE(_Copy_arrayof_conjoint_jshorts,@function)
|
|
|
169 ELF_TYPE(_Copy_conjoint_jshorts_atomic,@function)
|
|
|
170 SYMBOL(_Copy_arrayof_conjoint_jshorts):
|
|
|
171 SYMBOL(_Copy_conjoint_jshorts_atomic):
|
|
|
172 movq %rdx,%r8 # word count
|
|
|
173 shrq $2,%rdx # qword count
|
|
|
174 cmpq %rdi,%rsi
|
|
|
175 leaq -2(%rdi,%r8,2),%rax # from + wcount*2 - 2
|
|
|
176 jbe acs_CopyRight
|
|
|
177 cmpq %rax,%rsi
|
|
|
178 jbe acs_CopyLeft
|
|
|
179 acs_CopyRight:
|
|
|
180 leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
|
|
|
181 leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
|
|
|
182 negq %rdx
|
|
|
183 jmp 6f
|
|
|
184 1: movq 8(%rax,%rdx,8),%rsi
|
|
|
185 movq %rsi,8(%rcx,%rdx,8)
|
|
|
186 addq $1,%rdx
|
|
|
187 jnz 1b
|
|
|
188 2: testq $2,%r8 # check for trailing dword
|
|
|
189 jz 3f
|
|
|
190 movl 8(%rax),%esi # copy trailing dword
|
|
|
191 movl %esi,8(%rcx)
|
|
|
192 addq $4,%rcx # original %rsi is trashed, so we
|
|
|
193 # can't use it as a base register
|
|
|
194 3: testq $1,%r8 # check for trailing word
|
|
|
195 jz 4f
|
|
|
196 movw -2(%rdi,%r8,2),%si # copy trailing word
|
|
|
197 movw %si,8(%rcx)
|
|
|
198 4: ret
|
|
|
199 .p2align 4,,15
|
|
|
200 5: movq -24(%rax,%rdx,8),%rsi
|
|
|
201 movq %rsi,-24(%rcx,%rdx,8)
|
|
|
202 movq -16(%rax,%rdx,8),%rsi
|
|
|
203 movq %rsi,-16(%rcx,%rdx,8)
|
|
|
204 movq -8(%rax,%rdx,8),%rsi
|
|
|
205 movq %rsi,-8(%rcx,%rdx,8)
|
|
|
206 movq (%rax,%rdx,8),%rsi
|
|
|
207 movq %rsi,(%rcx,%rdx,8)
|
|
|
208 6: addq $4,%rdx
|
|
|
209 jle 5b
|
|
|
210 subq $4,%rdx
|
|
|
211 jl 1b
|
|
|
212 jmp 2b
|
|
|
213 acs_CopyLeft:
|
|
|
214 testq $1,%r8 # check for trailing word
|
|
|
215 jz 1f
|
|
|
216 movw -2(%rdi,%r8,2),%cx # copy trailing word
|
|
|
217 movw %cx,-2(%rsi,%r8,2)
|
|
|
218 1: testq $2,%r8 # check for trailing dword
|
|
|
219 jz 4f
|
|
|
220 movl (%rdi,%rdx,8),%ecx # copy trailing dword
|
|
|
221 movl %ecx,(%rsi,%rdx,8)
|
|
|
222 jmp 4f
|
|
|
223 2: movq -8(%rdi,%rdx,8),%rcx
|
|
|
224 movq %rcx,-8(%rsi,%rdx,8)
|
|
|
225 subq $1,%rdx
|
|
|
226 jnz 2b
|
|
|
227 ret
|
|
|
228 .p2align 4,,15
|
|
|
229 3: movq 24(%rdi,%rdx,8),%rcx
|
|
|
230 movq %rcx,24(%rsi,%rdx,8)
|
|
|
231 movq 16(%rdi,%rdx,8),%rcx
|
|
|
232 movq %rcx,16(%rsi,%rdx,8)
|
|
|
233 movq 8(%rdi,%rdx,8),%rcx
|
|
|
234 movq %rcx,8(%rsi,%rdx,8)
|
|
|
235 movq (%rdi,%rdx,8),%rcx
|
|
|
236 movq %rcx,(%rsi,%rdx,8)
|
|
|
237 4: subq $4,%rdx
|
|
|
238 jge 3b
|
|
|
239 addq $4,%rdx
|
|
|
240 jg 2b
|
|
|
241 ret
|
|
|
242
|
|
|
243 # Support for void Copy::arrayof_conjoint_jints(jint* from,
|
|
|
244 # jint* to,
|
|
|
245 # size_t count)
|
|
|
246 # Equivalent to
|
|
|
247 # conjoint_jints_atomic
|
|
|
248 #
|
|
|
249 # If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
|
|
|
250 # the hardware handle it. The two dwords within qwords that span
|
|
|
251 # cache line boundaries will still be loaded and stored atomically.
|
|
|
252 #
|
|
|
253 # rdi - from
|
|
|
254 # rsi - to
|
|
|
255 # rdx - count, treated as ssize_t
|
|
|
256 #
|
|
|
257 .p2align 4,,15
|
|
|
258 ELF_TYPE(_Copy_arrayof_conjoint_jints,@function)
|
|
|
259 ELF_TYPE(_Copy_conjoint_jints_atomic,@function)
|
|
|
260 SYMBOL(_Copy_arrayof_conjoint_jints):
|
|
|
261 SYMBOL(_Copy_conjoint_jints_atomic):
|
|
|
262 movq %rdx,%r8 # dword count
|
|
|
263 shrq %rdx # qword count
|
|
|
264 cmpq %rdi,%rsi
|
|
|
265 leaq -4(%rdi,%r8,4),%rax # from + dcount*4 - 4
|
|
|
266 jbe aci_CopyRight
|
|
|
267 cmpq %rax,%rsi
|
|
|
268 jbe aci_CopyLeft
|
|
|
269 aci_CopyRight:
|
|
|
270 leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
|
|
|
271 leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
|
|
|
272 negq %rdx
|
|
|
273 jmp 5f
|
|
|
274 .p2align 4,,15
|
|
|
275 1: movq 8(%rax,%rdx,8),%rsi
|
|
|
276 movq %rsi,8(%rcx,%rdx,8)
|
|
|
277 addq $1,%rdx
|
|
|
278 jnz 1b
|
|
|
279 2: testq $1,%r8 # check for trailing dword
|
|
|
280 jz 3f
|
|
|
281 movl 8(%rax),%esi # copy trailing dword
|
|
|
282 movl %esi,8(%rcx)
|
|
|
283 3: ret
|
|
|
284 .p2align 4,,15
|
|
|
285 4: movq -24(%rax,%rdx,8),%rsi
|
|
|
286 movq %rsi,-24(%rcx,%rdx,8)
|
|
|
287 movq -16(%rax,%rdx,8),%rsi
|
|
|
288 movq %rsi,-16(%rcx,%rdx,8)
|
|
|
289 movq -8(%rax,%rdx,8),%rsi
|
|
|
290 movq %rsi,-8(%rcx,%rdx,8)
|
|
|
291 movq (%rax,%rdx,8),%rsi
|
|
|
292 movq %rsi,(%rcx,%rdx,8)
|
|
|
293 5: addq $4,%rdx
|
|
|
294 jle 4b
|
|
|
295 subq $4,%rdx
|
|
|
296 jl 1b
|
|
|
297 jmp 2b
|
|
|
298 aci_CopyLeft:
|
|
|
299 testq $1,%r8 # check for trailing dword
|
|
|
300 jz 3f
|
|
|
301 movl -4(%rdi,%r8,4),%ecx # copy trailing dword
|
|
|
302 movl %ecx,-4(%rsi,%r8,4)
|
|
|
303 jmp 3f
|
|
|
304 1: movq -8(%rdi,%rdx,8),%rcx
|
|
|
305 movq %rcx,-8(%rsi,%rdx,8)
|
|
|
306 subq $1,%rdx
|
|
|
307 jnz 1b
|
|
|
308 ret
|
|
|
309 .p2align 4,,15
|
|
|
310 2: movq 24(%rdi,%rdx,8),%rcx
|
|
|
311 movq %rcx,24(%rsi,%rdx,8)
|
|
|
312 movq 16(%rdi,%rdx,8),%rcx
|
|
|
313 movq %rcx,16(%rsi,%rdx,8)
|
|
|
314 movq 8(%rdi,%rdx,8),%rcx
|
|
|
315 movq %rcx,8(%rsi,%rdx,8)
|
|
|
316 movq (%rdi,%rdx,8),%rcx
|
|
|
317 movq %rcx,(%rsi,%rdx,8)
|
|
|
318 3: subq $4,%rdx
|
|
|
319 jge 2b
|
|
|
320 addq $4,%rdx
|
|
|
321 jg 1b
|
|
|
322 ret
|
|
|
323
|
|
|
324 # Support for void Copy::arrayof_conjoint_jlongs(jlong* from,
|
|
|
325 # jlong* to,
|
|
|
326 # size_t count)
|
|
|
327 # Equivalent to
|
|
|
328 # conjoint_jlongs_atomic
|
|
|
329 # arrayof_conjoint_oops
|
|
|
330 # conjoint_oops_atomic
|
|
|
331 #
|
|
|
332 # rdi - from
|
|
|
333 # rsi - to
|
|
|
334 # rdx - count, treated as ssize_t
|
|
|
335 #
|
|
|
336 .p2align 4,,15
|
|
|
337 ELF_TYPE(_Copy_arrayof_conjoint_jlongs,@function)
|
|
|
338 ELF_TYPE(_Copy_conjoint_jlongs_atomic,@function)
|
|
|
339 SYMBOL(_Copy_arrayof_conjoint_jlongs):
|
|
|
340 SYMBOL(_Copy_conjoint_jlongs_atomic):
|
|
|
341 cmpq %rdi,%rsi
|
|
|
342 leaq -8(%rdi,%rdx,8),%rax # from + count*8 - 8
|
|
|
343 jbe acl_CopyRight
|
|
|
344 cmpq %rax,%rsi
|
|
|
345 jbe acl_CopyLeft
|
|
|
346 acl_CopyRight:
|
|
|
347 leaq -8(%rsi,%rdx,8),%rcx # to + count*8 - 8
|
|
|
348 negq %rdx
|
|
|
349 jmp 3f
|
|
|
350 1: movq 8(%rax,%rdx,8),%rsi
|
|
|
351 movq %rsi,8(%rcx,%rdx,8)
|
|
|
352 addq $1,%rdx
|
|
|
353 jnz 1b
|
|
|
354 ret
|
|
|
355 .p2align 4,,15
|
|
|
356 2: movq -24(%rax,%rdx,8),%rsi
|
|
|
357 movq %rsi,-24(%rcx,%rdx,8)
|
|
|
358 movq -16(%rax,%rdx,8),%rsi
|
|
|
359 movq %rsi,-16(%rcx,%rdx,8)
|
|
|
360 movq -8(%rax,%rdx,8),%rsi
|
|
|
361 movq %rsi,-8(%rcx,%rdx,8)
|
|
|
362 movq (%rax,%rdx,8),%rsi
|
|
|
363 movq %rsi,(%rcx,%rdx,8)
|
|
|
364 3: addq $4,%rdx
|
|
|
365 jle 2b
|
|
|
366 subq $4,%rdx
|
|
|
367 jl 1b
|
|
|
368 ret
|
|
|
369 4: movq -8(%rdi,%rdx,8),%rcx
|
|
|
370 movq %rcx,-8(%rsi,%rdx,8)
|
|
|
371 subq $1,%rdx
|
|
|
372 jnz 4b
|
|
|
373 ret
|
|
|
374 .p2align 4,,15
|
|
|
375 5: movq 24(%rdi,%rdx,8),%rcx
|
|
|
376 movq %rcx,24(%rsi,%rdx,8)
|
|
|
377 movq 16(%rdi,%rdx,8),%rcx
|
|
|
378 movq %rcx,16(%rsi,%rdx,8)
|
|
|
379 movq 8(%rdi,%rdx,8),%rcx
|
|
|
380 movq %rcx,8(%rsi,%rdx,8)
|
|
|
381 movq (%rdi,%rdx,8),%rcx
|
|
|
382 movq %rcx,(%rsi,%rdx,8)
|
|
|
383 acl_CopyLeft:
|
|
|
384 subq $4,%rdx
|
|
|
385 jge 5b
|
|
|
386 addq $4,%rdx
|
|
|
387 jg 4b
|
|
|
388 ret
|
