Bug Summary

File:jdk/src/hotspot/cpu/x86/assembler_x86.hpp
Warning:line 233, column 5
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name stubGenerator_x86_64.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -mthread-model posix -fno-delete-null-pointer-checks -mframe-pointer=all -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/libjvm/objs/precompiled -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D _GNU_SOURCE -D _REENTRANT -D LIBC=gnu -D LINUX -D VM_LITTLE_ENDIAN -D _LP64=1 -D ASSERT -D CHECK_UNHANDLED_OOPS -D TARGET_ARCH_x86 -D INCLUDE_SUFFIX_OS=_linux -D INCLUDE_SUFFIX_CPU=_x86 -D INCLUDE_SUFFIX_COMPILER=_gcc -D TARGET_COMPILER_gcc -D AMD64 -D HOTSPOT_LIB_ARCH="amd64" -D COMPILER1 -D COMPILER2 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc/adfiles -I /home/daniel/Projects/java/jdk/src/hotspot/share -I /home/daniel/Projects/java/jdk/src/hotspot/os/linux -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix -I /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86 -I /home/daniel/Projects/java/jdk/src/hotspot/os_cpu/linux_x86 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc -I /home/daniel/Projects/java/jdk/src/hotspot/share/precompiled -I /home/daniel/Projects/java/jdk/src/hotspot/share/include -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix/include -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/support/modules_include/java.base -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/support/modules_include/java.base/linux -I /home/daniel/Projects/java/jdk/src/java.base/share/native/libjimage -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc/adfiles -I /home/daniel/Projects/java/jdk/src/hotspot/share -I /home/daniel/Projects/java/jdk/src/hotspot/os/linux -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix -I /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86 -I /home/daniel/Projects/java/jdk/src/hotspot/os_cpu/linux_x86 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc -D _FORTIFY_SOURCE=2 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -Wno-format-zero-length -Wno-unused-parameter -Wno-unused -Wno-parentheses -Wno-comment -Wno-unknown-pragmas -Wno-address -Wno-delete-non-virtual-dtor -Wno-char-subscripts -Wno-array-bounds -Wno-int-in-bool-context -Wno-ignored-qualifiers -Wno-missing-field-initializers -Wno-implicit-fallthrough -Wno-empty-body -Wno-strict-overflow -Wno-sequence-point -Wno-maybe-uninitialized -Wno-misleading-indentation -Wno-cast-function-type -Wno-shift-negative-value -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /home/daniel/Projects/java/jdk/make/hotspot -ferror-limit 19 -fmessage-length 0 -fvisibility hidden -stack-protector 1 -fno-rtti -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -o /home/daniel/Projects/java/scan/2021-12-21-193737-8510-1 -x c++ /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

→
1/*
2 * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#include "precompiled.hpp"
26#include "asm/macroAssembler.hpp"
27#include "asm/macroAssembler.inline.hpp"
28#include "ci/ciUtilities.hpp"
29#include "compiler/oopMap.hpp"
30#include "gc/shared/barrierSet.hpp"
31#include "gc/shared/barrierSetAssembler.hpp"
32#include "gc/shared/barrierSetNMethod.hpp"
33#include "gc/shared/gc_globals.hpp"
34#include "interpreter/interpreter.hpp"
35#include "memory/universe.hpp"
36#include "nativeInst_x86.hpp"
37#include "oops/instanceOop.hpp"
38#include "oops/method.hpp"
39#include "oops/objArrayKlass.hpp"
40#include "oops/oop.inline.hpp"
41#include "prims/methodHandles.hpp"
42#include "runtime/arguments.hpp"
43#include "runtime/frame.inline.hpp"
44#include "runtime/handles.inline.hpp"
45#include "runtime/sharedRuntime.hpp"
46#include "runtime/stubCodeGenerator.hpp"
47#include "runtime/stubRoutines.hpp"
48#include "runtime/thread.inline.hpp"
49#ifdef COMPILER21
50#include "opto/runtime.hpp"
51#endif
52#if INCLUDE_JVMCI1
53#include "jvmci/jvmci_globals.hpp"
54#endif
55#if INCLUDE_ZGC1
56#include "gc/z/zThreadLocalData.hpp"
57#endif
58
59// Declaration and definition of StubGenerator (no .hpp file).
60// For a more detailed description of the stub routine structure
61// see the comment in stubRoutines.hpp
62
63#define __masm-> _masm->
64#define TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8) (UseCompressedOops ? Address::times_4 : Address::times_8)
65#define a__((Assembler*)_masm)-> ((Assembler*)_masm)->
66
67#ifdef PRODUCT
68#define BLOCK_COMMENT(str)masm-> block_comment(str) /* nothing */
69#else
70#define BLOCK_COMMENT(str)masm-> block_comment(str) __masm-> block_comment(str)
71#endif
72
73#define BIND(label)bind(label); masm-> block_comment("label" ":") bind(label); BLOCK_COMMENT(#label ":")masm-> block_comment(#label ":")
74const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
75
76// Stub Code definitions
77
78class StubGenerator: public StubCodeGenerator {
79 private:
80
81#ifdef PRODUCT
82#define inc_counter_np(counter)masm-> block_comment("inc_counter " "counter"); inc_counter_np_
(counter); ((void)0)
83#else
84 void inc_counter_np_(int& counter) {
85 // This can destroy rscratch1 if counter is far from the code cache
86 __masm-> incrementl(ExternalAddress((address)&counter));
87 }
88#define inc_counter_np(counter)masm-> block_comment("inc_counter " "counter"); inc_counter_np_
(counter); \
89 BLOCK_COMMENT("inc_counter " #counter)masm-> block_comment("inc_counter " #counter); \
90 inc_counter_np_(counter);
91#endif
92
93 // Call stubs are used to call Java from C
94 //
95 // Linux Arguments:
96 // c_rarg0: call wrapper address address
97 // c_rarg1: result address
98 // c_rarg2: result type BasicType
99 // c_rarg3: method Method*
100 // c_rarg4: (interpreter) entry point address
101 // c_rarg5: parameters intptr_t*
102 // 16(rbp): parameter size (in words) int
103 // 24(rbp): thread Thread*
104 //
105 // [ return_from_Java ] <--- rsp
106 // [ argument word n ]
107 // ...
108 // -12 [ argument word 1 ]
109 // -11 [ saved r15 ] <--- rsp_after_call
110 // -10 [ saved r14 ]
111 // -9 [ saved r13 ]
112 // -8 [ saved r12 ]
113 // -7 [ saved rbx ]
114 // -6 [ call wrapper ]
115 // -5 [ result ]
116 // -4 [ result type ]
117 // -3 [ method ]
118 // -2 [ entry point ]
119 // -1 [ parameters ]
120 // 0 [ saved rbp ] <--- rbp
121 // 1 [ return address ]
122 // 2 [ parameter size ]
123 // 3 [ thread ]
124 //
125 // Windows Arguments:
126 // c_rarg0: call wrapper address address
127 // c_rarg1: result address
128 // c_rarg2: result type BasicType
129 // c_rarg3: method Method*
130 // 48(rbp): (interpreter) entry point address
131 // 56(rbp): parameters intptr_t*
132 // 64(rbp): parameter size (in words) int
133 // 72(rbp): thread Thread*
134 //
135 // [ return_from_Java ] <--- rsp
136 // [ argument word n ]
137 // ...
138 // -60 [ argument word 1 ]
139 // -59 [ saved xmm31 ] <--- rsp after_call
140 // [ saved xmm16-xmm30 ] (EVEX enabled, else the space is blank)
141 // -27 [ saved xmm15 ]
142 // [ saved xmm7-xmm14 ]
143 // -9 [ saved xmm6 ] (each xmm register takes 2 slots)
144 // -7 [ saved r15 ]
145 // -6 [ saved r14 ]
146 // -5 [ saved r13 ]
147 // -4 [ saved r12 ]
148 // -3 [ saved rdi ]
149 // -2 [ saved rsi ]
150 // -1 [ saved rbx ]
151 // 0 [ saved rbp ] <--- rbp
152 // 1 [ return address ]
153 // 2 [ call wrapper ]
154 // 3 [ result ]
155 // 4 [ result type ]
156 // 5 [ method ]
157 // 6 [ entry point ]
158 // 7 [ parameters ]
159 // 8 [ parameter size ]
160 // 9 [ thread ]
161 //
162 // Windows reserves the callers stack space for arguments 1-4.
163 // We spill c_rarg0-c_rarg3 to this space.
164
165 // Call stub stack layout word offsets from rbp
166 enum call_stub_layout {
167#ifdef _WIN64
168 xmm_save_first = 6, // save from xmm6
169 xmm_save_last = 31, // to xmm31
170 xmm_save_base = -9,
171 rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
172 r15_off = -7,
173 r14_off = -6,
174 r13_off = -5,
175 r12_off = -4,
176 rdi_off = -3,
177 rsi_off = -2,
178 rbx_off = -1,
179 rbp_off = 0,
180 retaddr_off = 1,
181 call_wrapper_off = 2,
182 result_off = 3,
183 result_type_off = 4,
184 method_off = 5,
185 entry_point_off = 6,
186 parameters_off = 7,
187 parameter_size_off = 8,
188 thread_off = 9
189#else
190 rsp_after_call_off = -12,
191 mxcsr_off = rsp_after_call_off,
192 r15_off = -11,
193 r14_off = -10,
194 r13_off = -9,
195 r12_off = -8,
196 rbx_off = -7,
197 call_wrapper_off = -6,
198 result_off = -5,
199 result_type_off = -4,
200 method_off = -3,
201 entry_point_off = -2,
202 parameters_off = -1,
203 rbp_off = 0,
204 retaddr_off = 1,
205 parameter_size_off = 2,
206 thread_off = 3
207#endif
208 };
209
210#ifdef _WIN64
211 Address xmm_save(int reg) {
212 assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range")do { if (!(reg >= xmm_save_first && reg <= xmm_save_last
)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 212, "assert(" "reg >= xmm_save_first && reg <= xmm_save_last"
") failed", "XMM register number out of range"); ::breakpoint
(); } } while (0);
213 return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
214 }
215#endif
216
217 address generate_call_stub(address& return_address) {
218 assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&do { if (!((int)frame::entry_frame_after_call_words == -(int)
rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset
== (int)call_wrapper_off)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 220, "assert(" "(int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off"
") failed", "adjust this code"); ::breakpoint(); } } while (
0)
219 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,do { if (!((int)frame::entry_frame_after_call_words == -(int)
rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset
== (int)call_wrapper_off)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 220, "assert(" "(int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off"
") failed", "adjust this code"); ::breakpoint(); } } while (
0)
220 "adjust this code")do { if (!((int)frame::entry_frame_after_call_words == -(int)
rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset
== (int)call_wrapper_off)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 220, "assert(" "(int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off"
") failed", "adjust this code"); ::breakpoint(); } } while (
0);
221 StubCodeMark mark(this, "StubRoutines", "call_stub");
222 address start = __masm-> pc();
223
224 // same as in generate_catch_exception()!
225 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
226
227 const Address call_wrapper (rbp, call_wrapper_off * wordSize);
228 const Address result (rbp, result_off * wordSize);
229 const Address result_type (rbp, result_type_off * wordSize);
230 const Address method (rbp, method_off * wordSize);
231 const Address entry_point (rbp, entry_point_off * wordSize);
232 const Address parameters (rbp, parameters_off * wordSize);
233 const Address parameter_size(rbp, parameter_size_off * wordSize);
234
235 // same as in generate_catch_exception()!
236 const Address thread (rbp, thread_off * wordSize);
237
238 const Address r15_save(rbp, r15_off * wordSize);
239 const Address r14_save(rbp, r14_off * wordSize);
240 const Address r13_save(rbp, r13_off * wordSize);
241 const Address r12_save(rbp, r12_off * wordSize);
242 const Address rbx_save(rbp, rbx_off * wordSize);
243
244 // stub code
245 __masm-> enter();
246 __masm-> subptr(rsp, -rsp_after_call_off * wordSize);
247
248 // save register parameters
249#ifndef _WIN64
250 __masm-> movptr(parameters, c_rarg5); // parameters
251 __masm-> movptr(entry_point, c_rarg4); // entry_point
252#endif
253
254 __masm-> movptr(method, c_rarg3); // method
255 __masm-> movl(result_type, c_rarg2); // result type
256 __masm-> movptr(result, c_rarg1); // result
257 __masm-> movptr(call_wrapper, c_rarg0); // call wrapper
258
259 // save regs belonging to calling function
260 __masm-> movptr(rbx_save, rbx);
261 __masm-> movptr(r12_save, r12);
262 __masm-> movptr(r13_save, r13);
263 __masm-> movptr(r14_save, r14);
264 __masm-> movptr(r15_save, r15);
265
266#ifdef _WIN64
267 int last_reg = 15;
268 if (UseAVX > 2) {
269 last_reg = 31;
270 }
271 if (VM_Version::supports_evex()) {
272 for (int i = xmm_save_first; i <= last_reg; i++) {
273 __masm-> vextractf32x4(xmm_save(i), as_XMMRegister(i), 0);
274 }
275 } else {
276 for (int i = xmm_save_first; i <= last_reg; i++) {
277 __masm-> movdqu(xmm_save(i), as_XMMRegister(i));
278 }
279 }
280
281 const Address rdi_save(rbp, rdi_off * wordSize);
282 const Address rsi_save(rbp, rsi_off * wordSize);
283
284 __masm-> movptr(rsi_save, rsi);
285 __masm-> movptr(rdi_save, rdi);
286#else
287 const Address mxcsr_save(rbp, mxcsr_off * wordSize);
288 {
289 Label skip_ldmx;
290 __masm-> stmxcsr(mxcsr_save);
291 __masm-> movl(rax, mxcsr_save);
292 __masm-> andl(rax, MXCSR_MASK); // Only check control and mask bits
293 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
294 __masm-> cmp32(rax, mxcsr_std);
295 __masm-> jcc(Assembler::equal, skip_ldmx);
296 __masm-> ldmxcsr(mxcsr_std);
297 __masm-> bind(skip_ldmx);
298 }
299#endif
300
301 // Load up thread register
302 __masm-> movptr(r15_thread, thread);
303 __masm-> reinit_heapbase();
304
305#ifdef ASSERT1
306 // make sure we have no pending exceptions
307 {
308 Label L;
309 __masm-> cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD0L);
310 __masm-> jcc(Assembler::equal, L);
311 __masm-> stop("StubRoutines::call_stub: entered with pending exception");
312 __masm-> bind(L);
313 }
314#endif
315
316 // pass parameters if any
317 BLOCK_COMMENT("pass parameters if any")masm-> block_comment("pass parameters if any");
318 Label parameters_done;
319 __masm-> movl(c_rarg3, parameter_size);
320 __masm-> testl(c_rarg3, c_rarg3);
321 __masm-> jcc(Assembler::zero, parameters_done);
322
323 Label loop;
324 __masm-> movptr(c_rarg2, parameters); // parameter pointer
325 __masm-> movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1
326 __masm-> BIND(loop)bind(loop); masm-> block_comment("loop" ":");
327 __masm-> movptr(rax, Address(c_rarg2, 0));// get parameter
328 __masm-> addptr(c_rarg2, wordSize); // advance to next parameter
329 __masm-> decrementl(c_rarg1); // decrement counter
330 __masm-> push(rax); // pass parameter
331 __masm-> jcc(Assembler::notZero, loop);
332
333 // call Java function
334 __masm-> BIND(parameters_done)bind(parameters_done); masm-> block_comment("parameters_done"
":");
335 __masm-> movptr(rbx, method); // get Method*
336 __masm-> movptr(c_rarg1, entry_point); // get entry_point
337 __masm-> mov(r13, rsp); // set sender sp
338 BLOCK_COMMENT("call Java function")masm-> block_comment("call Java function");
339 __masm-> call(c_rarg1);
340
341 BLOCK_COMMENT("call_stub_return_address:")masm-> block_comment("call_stub_return_address:");
342 return_address = __masm-> pc();
343
344 // store result depending on type (everything that is not
345 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
346 __masm-> movptr(c_rarg0, result);
347 Label is_long, is_float, is_double, exit;
348 __masm-> movl(c_rarg1, result_type);
349 __masm-> cmpl(c_rarg1, T_OBJECT);
350 __masm-> jcc(Assembler::equal, is_long);
351 __masm-> cmpl(c_rarg1, T_LONG);
352 __masm-> jcc(Assembler::equal, is_long);
353 __masm-> cmpl(c_rarg1, T_FLOAT);
354 __masm-> jcc(Assembler::equal, is_float);
355 __masm-> cmpl(c_rarg1, T_DOUBLE);
356 __masm-> jcc(Assembler::equal, is_double);
357
358 // handle T_INT case
359 __masm-> movl(Address(c_rarg0, 0), rax);
360
361 __masm-> BIND(exit)bind(exit); masm-> block_comment("exit" ":");
362
363 // pop parameters
364 __masm-> lea(rsp, rsp_after_call);
365
366#ifdef ASSERT1
367 // verify that threads correspond
368 {
369 Label L1, L2, L3;
370 __masm-> cmpptr(r15_thread, thread);
371 __masm-> jcc(Assembler::equal, L1);
372 __masm-> stop("StubRoutines::call_stub: r15_thread is corrupted");
373 __masm-> bind(L1);
374 __masm-> get_thread(rbx);
375 __masm-> cmpptr(r15_thread, thread);
376 __masm-> jcc(Assembler::equal, L2);
377 __masm-> stop("StubRoutines::call_stub: r15_thread is modified by call");
378 __masm-> bind(L2);
379 __masm-> cmpptr(r15_thread, rbx);
380 __masm-> jcc(Assembler::equal, L3);
381 __masm-> stop("StubRoutines::call_stub: threads must correspond");
382 __masm-> bind(L3);
383 }
384#endif
385
386 // restore regs belonging to calling function
387#ifdef _WIN64
388 // emit the restores for xmm regs
389 if (VM_Version::supports_evex()) {
390 for (int i = xmm_save_first; i <= last_reg; i++) {
391 __masm-> vinsertf32x4(as_XMMRegister(i), as_XMMRegister(i), xmm_save(i), 0);
392 }
393 } else {
394 for (int i = xmm_save_first; i <= last_reg; i++) {
395 __masm-> movdqu(as_XMMRegister(i), xmm_save(i));
396 }
397 }
398#endif
399 __masm-> movptr(r15, r15_save);
400 __masm-> movptr(r14, r14_save);
401 __masm-> movptr(r13, r13_save);
402 __masm-> movptr(r12, r12_save);
403 __masm-> movptr(rbx, rbx_save);
404
405#ifdef _WIN64
406 __masm-> movptr(rdi, rdi_save);
407 __masm-> movptr(rsi, rsi_save);
408#else
409 __masm-> ldmxcsr(mxcsr_save);
410#endif
411
412 // restore rsp
413 __masm-> addptr(rsp, -rsp_after_call_off * wordSize);
414
415 // return
416 __masm-> vzeroupper();
417 __masm-> pop(rbp);
418 __masm-> ret(0);
419
420 // handle return types different from T_INT
421 __masm-> BIND(is_long)bind(is_long); masm-> block_comment("is_long" ":");
422 __masm-> movq(Address(c_rarg0, 0), rax);
423 __masm-> jmp(exit);
424
425 __masm-> BIND(is_float)bind(is_float); masm-> block_comment("is_float" ":");
426 __masm-> movflt(Address(c_rarg0, 0), xmm0);
427 __masm-> jmp(exit);
428
429 __masm-> BIND(is_double)bind(is_double); masm-> block_comment("is_double" ":");
430 __masm-> movdbl(Address(c_rarg0, 0), xmm0);
431 __masm-> jmp(exit);
432
433 return start;
434 }
435
436 // Return point for a Java call if there's an exception thrown in
437 // Java code. The exception is caught and transformed into a
438 // pending exception stored in JavaThread that can be tested from
439 // within the VM.
440 //
441 // Note: Usually the parameters are removed by the callee. In case
442 // of an exception crossing an activation frame boundary, that is
443 // not the case if the callee is compiled code => need to setup the
444 // rsp.
445 //
446 // rax: exception oop
447
448 address generate_catch_exception() {
449 StubCodeMark mark(this, "StubRoutines", "catch_exception");
450 address start = __masm-> pc();
451
452 // same as in generate_call_stub():
453 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
454 const Address thread (rbp, thread_off * wordSize);
455
456#ifdef ASSERT1
457 // verify that threads correspond
458 {
459 Label L1, L2, L3;
460 __masm-> cmpptr(r15_thread, thread);
461 __masm-> jcc(Assembler::equal, L1);
462 __masm-> stop("StubRoutines::catch_exception: r15_thread is corrupted");
463 __masm-> bind(L1);
464 __masm-> get_thread(rbx);
465 __masm-> cmpptr(r15_thread, thread);
466 __masm-> jcc(Assembler::equal, L2);
467 __masm-> stop("StubRoutines::catch_exception: r15_thread is modified by call");
468 __masm-> bind(L2);
469 __masm-> cmpptr(r15_thread, rbx);
470 __masm-> jcc(Assembler::equal, L3);
471 __masm-> stop("StubRoutines::catch_exception: threads must correspond");
472 __masm-> bind(L3);
473 }
474#endif
475
476 // set pending exception
477 __masm-> verify_oop(rax)_verify_oop_checked(rax, "broken oop " "rax", "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 477);
478
479 __masm-> movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
480 __masm-> lea(rscratch1, ExternalAddress((address)__FILE__"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"));
481 __masm-> movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
482 __masm-> movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__482);
483
484 // complete return to VM
485 assert(StubRoutines::_call_stub_return_address != NULL,do { if (!(StubRoutines::_call_stub_return_address != __null)
) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 486, "assert(" "StubRoutines::_call_stub_return_address != __null"
") failed", "_call_stub_return_address must have been generated before"
); ::breakpoint(); } } while (0)
486 "_call_stub_return_address must have been generated before")do { if (!(StubRoutines::_call_stub_return_address != __null)
) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 486, "assert(" "StubRoutines::_call_stub_return_address != __null"
") failed", "_call_stub_return_address must have been generated before"
); ::breakpoint(); } } while (0);
487 __masm-> jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
488
489 return start;
490 }
491
492 // Continuation point for runtime calls returning with a pending
493 // exception. The pending exception check happened in the runtime
494 // or native call stub. The pending exception in Thread is
495 // converted into a Java-level exception.
496 //
497 // Contract with Java-level exception handlers:
498 // rax: exception
499 // rdx: throwing pc
500 //
501 // NOTE: At entry of this stub, exception-pc must be on stack !!
502
503 address generate_forward_exception() {
504 StubCodeMark mark(this, "StubRoutines", "forward exception");
505 address start = __masm-> pc();
506
507 // Upon entry, the sp points to the return address returning into
508 // Java (interpreted or compiled) code; i.e., the return address
509 // becomes the throwing pc.
510 //
511 // Arguments pushed before the runtime call are still on the stack
512 // but the exception handler will reset the stack pointer ->
513 // ignore them. A potential result in registers can be ignored as
514 // well.
515
516#ifdef ASSERT1
517 // make sure this code is only executed if there is a pending exception
518 {
519 Label L;
520 __masm-> cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL__null);
521 __masm-> jcc(Assembler::notEqual, L);
522 __masm-> stop("StubRoutines::forward exception: no pending exception (1)");
523 __masm-> bind(L);
524 }
525#endif
526
527 // compute exception handler into rbx
528 __masm-> movptr(c_rarg0, Address(rsp, 0));
529 BLOCK_COMMENT("call exception_handler_for_return_address")masm-> block_comment("call exception_handler_for_return_address"
);
530 __masm-> call_VM_leaf(CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime::exception_handler_for_return_address
)))
531 SharedRuntime::exception_handler_for_return_address)((address)((address_word)(SharedRuntime::exception_handler_for_return_address
))),
532 r15_thread, c_rarg0);
533 __masm-> mov(rbx, rax);
534
535 // setup rax & rdx, remove return address & clear pending exception
536 __masm-> pop(rdx);
537 __masm-> movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
538 __masm-> movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD0L);
539
540#ifdef ASSERT1
541 // make sure exception is set
542 {
543 Label L;
544 __masm-> testptr(rax, rax);
545 __masm-> jcc(Assembler::notEqual, L);
546 __masm-> stop("StubRoutines::forward exception: no pending exception (2)");
547 __masm-> bind(L);
548 }
549#endif
550
551 // continue at exception handler (return address removed)
552 // rax: exception
553 // rbx: exception handler
554 // rdx: throwing pc
555 __masm-> verify_oop(rax)_verify_oop_checked(rax, "broken oop " "rax", "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 555);
556 __masm-> jmp(rbx);
557
558 return start;
559 }
560
561 // Support for intptr_t OrderAccess::fence()
562 //
563 // Arguments :
564 //
565 // Result:
566 address generate_orderaccess_fence() {
567 StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
568 address start = __masm-> pc();
569 __masm-> membar(Assembler::StoreLoad);
570 __masm-> ret(0);
571
572 return start;
573 }
574
575
576 // Support for intptr_t get_previous_sp()
577 //
578 // This routine is used to find the previous stack pointer for the
579 // caller.
580 address generate_get_previous_sp() {
581 StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
582 address start = __masm-> pc();
583
584 __masm-> movptr(rax, rsp);
585 __masm-> addptr(rax, 8); // return address is at the top of the stack.
586 __masm-> ret(0);
587
588 return start;
589 }
590
591 //----------------------------------------------------------------------------------------------------
592 // Support for void verify_mxcsr()
593 //
594 // This routine is used with -Xcheck:jni to verify that native
595 // JNI code does not return to Java code without restoring the
596 // MXCSR register to our expected state.
597
598 address generate_verify_mxcsr() {
599 StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
600 address start = __masm-> pc();
601
602 const Address mxcsr_save(rsp, 0);
603
604 if (CheckJNICalls) {
605 Label ok_ret;
606 ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
607 __masm-> push(rax);
608 __masm-> subptr(rsp, wordSize); // allocate a temp location
609 __masm-> stmxcsr(mxcsr_save);
610 __masm-> movl(rax, mxcsr_save);
611 __masm-> andl(rax, MXCSR_MASK); // Only check control and mask bits
612 __masm-> cmp32(rax, mxcsr_std);
613 __masm-> jcc(Assembler::equal, ok_ret);
614
615 __masm-> warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
616
617 __masm-> ldmxcsr(mxcsr_std);
618
619 __masm-> bind(ok_ret);
620 __masm-> addptr(rsp, wordSize);
621 __masm-> pop(rax);
622 }
623
624 __masm-> ret(0);
625
626 return start;
627 }
628
629 address generate_f2i_fixup() {
630 StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
631 Address inout(rsp, 5 * wordSize); // return address + 4 saves
632
633 address start = __masm-> pc();
634
635 Label L;
636
637 __masm-> push(rax);
638 __masm-> push(c_rarg3);
639 __masm-> push(c_rarg2);
640 __masm-> push(c_rarg1);
641
642 __masm-> movl(rax, 0x7f800000);
643 __masm-> xorl(c_rarg3, c_rarg3);
644 __masm-> movl(c_rarg2, inout);
645 __masm-> movl(c_rarg1, c_rarg2);
646 __masm-> andl(c_rarg1, 0x7fffffff);
647 __masm-> cmpl(rax, c_rarg1); // NaN? -> 0
648 __masm-> jcc(Assembler::negative, L);
649 __masm-> testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
650 __masm-> movl(c_rarg3, 0x80000000);
651 __masm-> movl(rax, 0x7fffffff);
652 __masm-> cmovl(Assembler::positive, c_rarg3, rax);
653
654 __masm-> bind(L);
655 __masm-> movptr(inout, c_rarg3);
656
657 __masm-> pop(c_rarg1);
658 __masm-> pop(c_rarg2);
659 __masm-> pop(c_rarg3);
660 __masm-> pop(rax);
661
662 __masm-> ret(0);
663
664 return start;
665 }
666
667 address generate_f2l_fixup() {
668 StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
669 Address inout(rsp, 5 * wordSize); // return address + 4 saves
670 address start = __masm-> pc();
671
672 Label L;
673
674 __masm-> push(rax);
675 __masm-> push(c_rarg3);
676 __masm-> push(c_rarg2);
677 __masm-> push(c_rarg1);
678
679 __masm-> movl(rax, 0x7f800000);
680 __masm-> xorl(c_rarg3, c_rarg3);
681 __masm-> movl(c_rarg2, inout);
682 __masm-> movl(c_rarg1, c_rarg2);
683 __masm-> andl(c_rarg1, 0x7fffffff);
684 __masm-> cmpl(rax, c_rarg1); // NaN? -> 0
685 __masm-> jcc(Assembler::negative, L);
686 __masm-> testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
687 __masm-> mov64(c_rarg3, 0x8000000000000000);
688 __masm-> mov64(rax, 0x7fffffffffffffff);
689 __masm-> cmov(Assembler::positive, c_rarg3, rax);
690
691 __masm-> bind(L);
692 __masm-> movptr(inout, c_rarg3);
693
694 __masm-> pop(c_rarg1);
695 __masm-> pop(c_rarg2);
696 __masm-> pop(c_rarg3);
697 __masm-> pop(rax);
698
699 __masm-> ret(0);
700
701 return start;
702 }
703
704 address generate_d2i_fixup() {
705 StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
706 Address inout(rsp, 6 * wordSize); // return address + 5 saves
707
708 address start = __masm-> pc();
709
710 Label L;
711
712 __masm-> push(rax);
713 __masm-> push(c_rarg3);
714 __masm-> push(c_rarg2);
715 __masm-> push(c_rarg1);
716 __masm-> push(c_rarg0);
717
718 __masm-> movl(rax, 0x7ff00000);
719 __masm-> movq(c_rarg2, inout);
720 __masm-> movl(c_rarg3, c_rarg2);
721 __masm-> mov(c_rarg1, c_rarg2);
722 __masm-> mov(c_rarg0, c_rarg2);
723 __masm-> negl(c_rarg3);
724 __masm-> shrptr(c_rarg1, 0x20);
725 __masm-> orl(c_rarg3, c_rarg2);
726 __masm-> andl(c_rarg1, 0x7fffffff);
727 __masm-> xorl(c_rarg2, c_rarg2);
728 __masm-> shrl(c_rarg3, 0x1f);
729 __masm-> orl(c_rarg1, c_rarg3);
730 __masm-> cmpl(rax, c_rarg1);
731 __masm-> jcc(Assembler::negative, L); // NaN -> 0
732 __masm-> testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
733 __masm-> movl(c_rarg2, 0x80000000);
734 __masm-> movl(rax, 0x7fffffff);
735 __masm-> cmov(Assembler::positive, c_rarg2, rax);
736
737 __masm-> bind(L);
738 __masm-> movptr(inout, c_rarg2);
739
740 __masm-> pop(c_rarg0);
741 __masm-> pop(c_rarg1);
742 __masm-> pop(c_rarg2);
743 __masm-> pop(c_rarg3);
744 __masm-> pop(rax);
745
746 __masm-> ret(0);
747
748 return start;
749 }
750
751 address generate_d2l_fixup() {
752 StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
753 Address inout(rsp, 6 * wordSize); // return address + 5 saves
754
755 address start = __masm-> pc();
756
757 Label L;
758
759 __masm-> push(rax);
760 __masm-> push(c_rarg3);
761 __masm-> push(c_rarg2);
762 __masm-> push(c_rarg1);
763 __masm-> push(c_rarg0);
764
765 __masm-> movl(rax, 0x7ff00000);
766 __masm-> movq(c_rarg2, inout);
767 __masm-> movl(c_rarg3, c_rarg2);
768 __masm-> mov(c_rarg1, c_rarg2);
769 __masm-> mov(c_rarg0, c_rarg2);
770 __masm-> negl(c_rarg3);
771 __masm-> shrptr(c_rarg1, 0x20);
772 __masm-> orl(c_rarg3, c_rarg2);
773 __masm-> andl(c_rarg1, 0x7fffffff);
774 __masm-> xorl(c_rarg2, c_rarg2);
775 __masm-> shrl(c_rarg3, 0x1f);
776 __masm-> orl(c_rarg1, c_rarg3);
777 __masm-> cmpl(rax, c_rarg1);
778 __masm-> jcc(Assembler::negative, L); // NaN -> 0
779 __masm-> testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
780 __masm-> mov64(c_rarg2, 0x8000000000000000);
781 __masm-> mov64(rax, 0x7fffffffffffffff);
782 __masm-> cmovq(Assembler::positive, c_rarg2, rax);
783
784 __masm-> bind(L);
785 __masm-> movq(inout, c_rarg2);
786
787 __masm-> pop(c_rarg0);
788 __masm-> pop(c_rarg1);
789 __masm-> pop(c_rarg2);
790 __masm-> pop(c_rarg3);
791 __masm-> pop(rax);
792
793 __masm-> ret(0);
794
795 return start;
796 }
797
798 address generate_iota_indices(const char *stub_name) {
799 __masm-> align(CodeEntryAlignment);
800 StubCodeMark mark(this, "StubRoutines", stub_name);
801 address start = __masm-> pc();
802 __masm-> emit_data64(0x0706050403020100, relocInfo::none);
803 __masm-> emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
804 __masm-> emit_data64(0x1716151413121110, relocInfo::none);
805 __masm-> emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none);
806 __masm-> emit_data64(0x2726252423222120, relocInfo::none);
807 __masm-> emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none);
808 __masm-> emit_data64(0x3736353433323130, relocInfo::none);
809 __masm-> emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none);
810 return start;
811 }
812
813 address generate_vector_byte_shuffle_mask(const char *stub_name) {
814 __masm-> align(CodeEntryAlignment);
815 StubCodeMark mark(this, "StubRoutines", stub_name);
816 address start = __masm-> pc();
817 __masm-> emit_data64(0x7070707070707070, relocInfo::none);
818 __masm-> emit_data64(0x7070707070707070, relocInfo::none);
819 __masm-> emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none);
820 __masm-> emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none);
821 return start;
822 }
823
824 address generate_fp_mask(const char *stub_name, int64_t mask) {
825 __masm-> align(CodeEntryAlignment);
826 StubCodeMark mark(this, "StubRoutines", stub_name);
827 address start = __masm-> pc();
828
829 __masm-> emit_data64( mask, relocInfo::none );
830 __masm-> emit_data64( mask, relocInfo::none );
831
832 return start;
833 }
834
835 address generate_vector_mask(const char *stub_name, int64_t mask) {
836 __masm-> align(CodeEntryAlignment);
837 StubCodeMark mark(this, "StubRoutines", stub_name);
838 address start = __masm-> pc();
839
840 __masm-> emit_data64(mask, relocInfo::none);
841 __masm-> emit_data64(mask, relocInfo::none);
842 __masm-> emit_data64(mask, relocInfo::none);
843 __masm-> emit_data64(mask, relocInfo::none);
844 __masm-> emit_data64(mask, relocInfo::none);
845 __masm-> emit_data64(mask, relocInfo::none);
846 __masm-> emit_data64(mask, relocInfo::none);
847 __masm-> emit_data64(mask, relocInfo::none);
848
849 return start;
850 }
851
852 address generate_vector_byte_perm_mask(const char *stub_name) {
853 __masm-> align(CodeEntryAlignment);
854 StubCodeMark mark(this, "StubRoutines", stub_name);
855 address start = __masm-> pc();
856
857 __masm-> emit_data64(0x0000000000000001, relocInfo::none);
858 __masm-> emit_data64(0x0000000000000003, relocInfo::none);
859 __masm-> emit_data64(0x0000000000000005, relocInfo::none);
860 __masm-> emit_data64(0x0000000000000007, relocInfo::none);
861 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
862 __masm-> emit_data64(0x0000000000000002, relocInfo::none);
863 __masm-> emit_data64(0x0000000000000004, relocInfo::none);
864 __masm-> emit_data64(0x0000000000000006, relocInfo::none);
865
866 return start;
867 }
868
869 address generate_vector_fp_mask(const char *stub_name, int64_t mask) {
870 __masm-> align(CodeEntryAlignment);
871 StubCodeMark mark(this, "StubRoutines", stub_name);
872 address start = __masm-> pc();
873
874 __masm-> emit_data64(mask, relocInfo::none);
875 __masm-> emit_data64(mask, relocInfo::none);
876 __masm-> emit_data64(mask, relocInfo::none);
877 __masm-> emit_data64(mask, relocInfo::none);
878 __masm-> emit_data64(mask, relocInfo::none);
879 __masm-> emit_data64(mask, relocInfo::none);
880 __masm-> emit_data64(mask, relocInfo::none);
881 __masm-> emit_data64(mask, relocInfo::none);
882
883 return start;
884 }
885
886 address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len,
887 int32_t val0, int32_t val1, int32_t val2, int32_t val3,
888 int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0,
889 int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0,
890 int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) {
891 __masm-> align(CodeEntryAlignment);
892 StubCodeMark mark(this, "StubRoutines", stub_name);
893 address start = __masm-> pc();
894
895 assert(len != Assembler::AVX_NoVec, "vector len must be specified")do { if (!(len != Assembler::AVX_NoVec)) { (*g_assert_poison)
= 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 895, "assert(" "len != Assembler::AVX_NoVec" ") failed", "vector len must be specified"
); ::breakpoint(); } } while (0);
896 __masm-> emit_data(val0, relocInfo::none, 0);
897 __masm-> emit_data(val1, relocInfo::none, 0);
898 __masm-> emit_data(val2, relocInfo::none, 0);
899 __masm-> emit_data(val3, relocInfo::none, 0);
900 if (len >= Assembler::AVX_256bit) {
901 __masm-> emit_data(val4, relocInfo::none, 0);
902 __masm-> emit_data(val5, relocInfo::none, 0);
903 __masm-> emit_data(val6, relocInfo::none, 0);
904 __masm-> emit_data(val7, relocInfo::none, 0);
905 if (len >= Assembler::AVX_512bit) {
906 __masm-> emit_data(val8, relocInfo::none, 0);
907 __masm-> emit_data(val9, relocInfo::none, 0);
908 __masm-> emit_data(val10, relocInfo::none, 0);
909 __masm-> emit_data(val11, relocInfo::none, 0);
910 __masm-> emit_data(val12, relocInfo::none, 0);
911 __masm-> emit_data(val13, relocInfo::none, 0);
912 __masm-> emit_data(val14, relocInfo::none, 0);
913 __masm-> emit_data(val15, relocInfo::none, 0);
914 }
915 }
916
917 return start;
918 }
919
920 // Non-destructive plausibility checks for oops
921 //
922 // Arguments:
923 // all args on stack!
924 //
925 // Stack after saving c_rarg3:
926 // [tos + 0]: saved c_rarg3
927 // [tos + 1]: saved c_rarg2
928 // [tos + 2]: saved r12 (several TemplateTable methods use it)
929 // [tos + 3]: saved flags
930 // [tos + 4]: return address
931 // * [tos + 5]: error message (char*)
932 // * [tos + 6]: object to verify (oop)
933 // * [tos + 7]: saved rax - saved by caller and bashed
934 // * [tos + 8]: saved r10 (rscratch1) - saved by caller
935 // * = popped on exit
936 address generate_verify_oop() {
937 StubCodeMark mark(this, "StubRoutines", "verify_oop");
938 address start = __masm-> pc();
939
940 Label exit, error;
941
942 __masm-> pushf();
943 __masm-> incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
944
945 __masm-> push(r12);
946
947 // save c_rarg2 and c_rarg3
948 __masm-> push(c_rarg2);
949 __masm-> push(c_rarg3);
950
951 enum {
952 // After previous pushes.
953 oop_to_verify = 6 * wordSize,
954 saved_rax = 7 * wordSize,
955 saved_r10 = 8 * wordSize,
956
957 // Before the call to MacroAssembler::debug(), see below.
958 return_addr = 16 * wordSize,
959 error_msg = 17 * wordSize
960 };
961
962 // get object
963 __masm-> movptr(rax, Address(rsp, oop_to_verify));
964
965 // make sure object is 'reasonable'
966 __masm-> testptr(rax, rax);
967 __masm-> jcc(Assembler::zero, exit); // if obj is NULL it is OK
968
969#if INCLUDE_ZGC1
970 if (UseZGC) {
971 // Check if metadata bits indicate a bad oop
972 __masm-> testptr(rax, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
973 __masm-> jcc(Assembler::notZero, error);
974 }
975#endif
976
977 // Check if the oop is in the right area of memory
978 __masm-> movptr(c_rarg2, rax);
979 __masm-> movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
980 __masm-> andptr(c_rarg2, c_rarg3);
981 __masm-> movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
982 __masm-> cmpptr(c_rarg2, c_rarg3);
983 __masm-> jcc(Assembler::notZero, error);
984
985 // make sure klass is 'reasonable', which is not zero.
986 __masm-> load_klass(rax, rax, rscratch1); // get klass
987 __masm-> testptr(rax, rax);
988 __masm-> jcc(Assembler::zero, error); // if klass is NULL it is broken
989
990 // return if everything seems ok
991 __masm-> bind(exit);
992 __masm-> movptr(rax, Address(rsp, saved_rax)); // get saved rax back
993 __masm-> movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
994 __masm-> pop(c_rarg3); // restore c_rarg3
995 __masm-> pop(c_rarg2); // restore c_rarg2
996 __masm-> pop(r12); // restore r12
997 __masm-> popf(); // restore flags
998 __masm-> ret(4 * wordSize); // pop caller saved stuff
999
1000 // handle errors
1001 __masm-> bind(error);
1002 __masm-> movptr(rax, Address(rsp, saved_rax)); // get saved rax back
1003 __masm-> movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1004 __masm-> pop(c_rarg3); // get saved c_rarg3 back
1005 __masm-> pop(c_rarg2); // get saved c_rarg2 back
1006 __masm-> pop(r12); // get saved r12 back
1007 __masm-> popf(); // get saved flags off stack --
1008 // will be ignored
1009
1010 __masm-> pusha(); // push registers
1011 // (rip is already
1012 // already pushed)
1013 // debug(char* msg, int64_t pc, int64_t regs[])
1014 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1015 // pushed all the registers, so now the stack looks like:
1016 // [tos + 0] 16 saved registers
1017 // [tos + 16] return address
1018 // * [tos + 17] error message (char*)
1019 // * [tos + 18] object to verify (oop)
1020 // * [tos + 19] saved rax - saved by caller and bashed
1021 // * [tos + 20] saved r10 (rscratch1) - saved by caller
1022 // * = popped on exit
1023
1024 __masm-> movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message
1025 __masm-> movptr(c_rarg1, Address(rsp, return_addr)); // pass return address
1026 __masm-> movq(c_rarg2, rsp); // pass address of regs on stack
1027 __masm-> mov(r12, rsp); // remember rsp
1028 __masm-> subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1029 __masm-> andptr(rsp, -16); // align stack as required by ABI
1030 BLOCK_COMMENT("call MacroAssembler::debug")masm-> block_comment("call MacroAssembler::debug");
1031 __masm-> call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)((address)((address_word)(MacroAssembler::debug64)))));
1032 __masm-> hlt();
1033 return start;
1034 }
1035
1036 //
1037 // Verify that a register contains clean 32-bits positive value
1038 // (high 32-bits are 0) so it could be used in 64-bits shifts.
1039 //
1040 // Input:
1041 // Rint - 32-bits value
1042 // Rtmp - scratch
1043 //
1044 void assert_clean_int(Register Rint, Register Rtmp) {
1045#ifdef ASSERT1
1046 Label L;
1047 assert_different_registers(Rtmp, Rint);
1048 __masm-> movslq(Rtmp, Rint);
1049 __masm-> cmpq(Rtmp, Rint);
1050 __masm-> jcc(Assembler::equal, L);
1051 __masm-> stop("high 32-bits of int value are not 0");
1052 __masm-> bind(L);
1053#endif
1054 }
1055
1056 // Generate overlap test for array copy stubs
1057 //
1058 // Input:
1059 // c_rarg0 - from
1060 // c_rarg1 - to
1061 // c_rarg2 - element count
1062 //
1063 // Output:
1064 // rax - &from[element count - 1]
1065 //
1066 void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1067 assert(no_overlap_target != NULL, "must be generated")do { if (!(no_overlap_target != __null)) { (*g_assert_poison)
= 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1067, "assert(" "no_overlap_target != __null" ") failed", "must be generated"
); ::breakpoint(); } } while (0);
1068 array_overlap_test(no_overlap_target, NULL__null, sf);
1069 }
1070 void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1071 array_overlap_test(NULL__null, &L_no_overlap, sf);
1072 }
1073 void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1074 const Register from = c_rarg0;
1075 const Register to = c_rarg1;
1076 const Register count = c_rarg2;
1077 const Register end_from = rax;
1078
1079 __masm-> cmpptr(to, from);
1080 __masm-> lea(end_from, Address(from, count, sf, 0));
1081 if (NOLp == NULL__null) {
1082 ExternalAddress no_overlap(no_overlap_target);
1083 __masm-> jump_cc(Assembler::belowEqual, no_overlap);
1084 __masm-> cmpptr(to, end_from);
1085 __masm-> jump_cc(Assembler::aboveEqual, no_overlap);
1086 } else {
1087 __masm-> jcc(Assembler::belowEqual, (*NOLp));
1088 __masm-> cmpptr(to, end_from);
1089 __masm-> jcc(Assembler::aboveEqual, (*NOLp));
1090 }
1091 }
1092
1093 // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1094 //
1095 // Outputs:
1096 // rdi - rcx
1097 // rsi - rdx
1098 // rdx - r8
1099 // rcx - r9
1100 //
1101 // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1102 // are non-volatile. r9 and r10 should not be used by the caller.
1103 //
1104 DEBUG_ONLY(bool regs_in_thread;)bool regs_in_thread;
1105
1106 void setup_arg_regs(int nargs = 3) {
1107 const Register saved_rdi = r9;
1108 const Register saved_rsi = r10;
1109 assert(nargs == 3 || nargs == 4, "else fix")do { if (!(nargs == 3 || nargs == 4)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1109, "assert(" "nargs == 3 || nargs == 4" ") failed", "else fix"
); ::breakpoint(); } } while (0);
1110#ifdef _WIN64
1111 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,do { if (!(c_rarg0 == rcx && c_rarg1 == rdx &&
c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1112, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0)
1112 "unexpected argument registers")do { if (!(c_rarg0 == rcx && c_rarg1 == rdx &&
c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1112, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0);
1113 if (nargs >= 4)
1114 __masm-> mov(rax, r9); // r9 is also saved_rdi
1115 __masm-> movptr(saved_rdi, rdi);
1116 __masm-> movptr(saved_rsi, rsi);
1117 __masm-> mov(rdi, rcx); // c_rarg0
1118 __masm-> mov(rsi, rdx); // c_rarg1
1119 __masm-> mov(rdx, r8); // c_rarg2
1120 if (nargs >= 4)
1121 __masm-> mov(rcx, rax); // c_rarg3 (via rax)
1122#else
1123 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,do { if (!(c_rarg0 == rdi && c_rarg1 == rsi &&
c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1124, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0)
1124 "unexpected argument registers")do { if (!(c_rarg0 == rdi && c_rarg1 == rsi &&
c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1124, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0);
1125#endif
1126 DEBUG_ONLY(regs_in_thread = false;)regs_in_thread = false;
1127 }
1128
1129 void restore_arg_regs() {
1130 assert(!regs_in_thread, "wrong call to restore_arg_regs")do { if (!(!regs_in_thread)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1130, "assert(" "!regs_in_thread" ") failed", "wrong call to restore_arg_regs"
); ::breakpoint(); } } while (0);
1131 const Register saved_rdi = r9;
1132 const Register saved_rsi = r10;
1133#ifdef _WIN64
1134 __masm-> movptr(rdi, saved_rdi);
1135 __masm-> movptr(rsi, saved_rsi);
1136#endif
1137 }
1138
1139 // This is used in places where r10 is a scratch register, and can
1140 // be adapted if r9 is needed also.
1141 void setup_arg_regs_using_thread() {
1142 const Register saved_r15 = r9;
1143#ifdef _WIN64
1144 __masm-> mov(saved_r15, r15); // r15 is callee saved and needs to be restored
1145 __masm-> get_thread(r15_thread);
1146 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,do { if (!(c_rarg0 == rcx && c_rarg1 == rdx &&
c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1147, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0)
1147 "unexpected argument registers")do { if (!(c_rarg0 == rcx && c_rarg1 == rdx &&
c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1147, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0);
1148 __masm-> movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())), rdi);
1149 __masm-> movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())), rsi);
1150
1151 __masm-> mov(rdi, rcx); // c_rarg0
1152 __masm-> mov(rsi, rdx); // c_rarg1
1153 __masm-> mov(rdx, r8); // c_rarg2
1154#else
1155 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,do { if (!(c_rarg0 == rdi && c_rarg1 == rsi &&
c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1156, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0)
1156 "unexpected argument registers")do { if (!(c_rarg0 == rdi && c_rarg1 == rsi &&
c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1156, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx"
") failed", "unexpected argument registers"); ::breakpoint()
; } } while (0);
1157#endif
1158 DEBUG_ONLY(regs_in_thread = true;)regs_in_thread = true;
1159 }
1160
1161 void restore_arg_regs_using_thread() {
1162 assert(regs_in_thread, "wrong call to restore_arg_regs")do { if (!(regs_in_thread)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1162, "assert(" "regs_in_thread" ") failed", "wrong call to restore_arg_regs"
); ::breakpoint(); } } while (0);
1163 const Register saved_r15 = r9;
1164#ifdef _WIN64
1165 __masm-> get_thread(r15_thread);
1166 __masm-> movptr(rsi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())));
1167 __masm-> movptr(rdi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())));
1168 __masm-> mov(r15, saved_r15); // r15 is callee saved and needs to be restored
1169#endif
1170 }
1171
1172 // Copy big chunks forward
1173 //
1174 // Inputs:
1175 // end_from - source arrays end address
1176 // end_to - destination array end address
1177 // qword_count - 64-bits element count, negative
1178 // to - scratch
1179 // L_copy_bytes - entry label
1180 // L_copy_8_bytes - exit label
1181 //
1182 void copy_bytes_forward(Register end_from, Register end_to,
1183 Register qword_count, Register to,
1184 Label& L_copy_bytes, Label& L_copy_8_bytes) {
1185 DEBUG_ONLY(__ stop("enter at entry label, not here"))masm-> stop("enter at entry label, not here");
1186 Label L_loop;
1187 __masm-> align(OptoLoopAlignment);
1188 if (UseUnalignedLoadStores) {
1189 Label L_end;
1190 __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":");
1191 if (UseAVX >= 2) {
1192 __masm-> vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1193 __masm-> vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1194 __masm-> vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1195 __masm-> vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1196 } else {
1197 __masm-> movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1198 __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1199 __masm-> movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1200 __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1201 __masm-> movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1202 __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1203 __masm-> movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1204 __masm-> movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1205 }
1206
1207 __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":"
);
1208 __masm-> addptr(qword_count, 8);
1209 __masm-> jcc(Assembler::lessEqual, L_loop);
1210 __masm-> subptr(qword_count, 4); // sub(8) and add(4)
1211 __masm-> jccb(Assembler::greater, L_end)jccb_0(Assembler::greater, L_end, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1211);
1212 // Copy trailing 32 bytes
1213 if (UseAVX >= 2) {
1214 __masm-> vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1215 __masm-> vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1216 } else {
1217 __masm-> movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1218 __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1219 __masm-> movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1220 __masm-> movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1221 }
1222 __masm-> addptr(qword_count, 4);
1223 __masm-> BIND(L_end)bind(L_end); masm-> block_comment("L_end" ":");
1224 if (UseAVX >= 2) {
1225 // clean upper bits of YMM registers
1226 __masm-> vpxor(xmm0, xmm0);
1227 __masm-> vpxor(xmm1, xmm1);
1228 }
1229 } else {
1230 // Copy 32-bytes per iteration
1231 __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":");
1232 __masm-> movq(to, Address(end_from, qword_count, Address::times_8, -24));
1233 __masm-> movq(Address(end_to, qword_count, Address::times_8, -24), to);
1234 __masm-> movq(to, Address(end_from, qword_count, Address::times_8, -16));
1235 __masm-> movq(Address(end_to, qword_count, Address::times_8, -16), to);
1236 __masm-> movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1237 __masm-> movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1238 __masm-> movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1239 __masm-> movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1240
1241 __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":"
);
1242 __masm-> addptr(qword_count, 4);
1243 __masm-> jcc(Assembler::lessEqual, L_loop);
1244 }
1245 __masm-> subptr(qword_count, 4);
1246 __masm-> jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1247 }
1248
1249 // Copy big chunks backward
1250 //
1251 // Inputs:
1252 // from - source arrays address
1253 // dest - destination array address
1254 // qword_count - 64-bits element count
1255 // to - scratch
1256 // L_copy_bytes - entry label
1257 // L_copy_8_bytes - exit label
1258 //
1259 void copy_bytes_backward(Register from, Register dest,
1260 Register qword_count, Register to,
1261 Label& L_copy_bytes, Label& L_copy_8_bytes) {
1262 DEBUG_ONLY(__ stop("enter at entry label, not here"))masm-> stop("enter at entry label, not here");
1263 Label L_loop;
1264 __masm-> align(OptoLoopAlignment);
1265 if (UseUnalignedLoadStores) {
1266 Label L_end;
1267 __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":");
1268 if (UseAVX >= 2) {
1269 __masm-> vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1270 __masm-> vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1271 __masm-> vmovdqu(xmm1, Address(from, qword_count, Address::times_8, 0));
1272 __masm-> vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm1);
1273 } else {
1274 __masm-> movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1275 __masm-> movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1276 __masm-> movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1277 __masm-> movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1278 __masm-> movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1279 __masm-> movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1280 __masm-> movdqu(xmm3, Address(from, qword_count, Address::times_8, 0));
1281 __masm-> movdqu(Address(dest, qword_count, Address::times_8, 0), xmm3);
1282 }
1283
1284 __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":"
);
1285 __masm-> subptr(qword_count, 8);
1286 __masm-> jcc(Assembler::greaterEqual, L_loop);
1287
1288 __masm-> addptr(qword_count, 4); // add(8) and sub(4)
1289 __masm-> jccb(Assembler::less, L_end)jccb_0(Assembler::less, L_end, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1289);
1290 // Copy trailing 32 bytes
1291 if (UseAVX >= 2) {
1292 __masm-> vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1293 __masm-> vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1294 } else {
1295 __masm-> movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1296 __masm-> movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1297 __masm-> movdqu(xmm1, Address(from, qword_count, Address::times_8, 0));
1298 __masm-> movdqu(Address(dest, qword_count, Address::times_8, 0), xmm1);
1299 }
1300 __masm-> subptr(qword_count, 4);
1301 __masm-> BIND(L_end)bind(L_end); masm-> block_comment("L_end" ":");
1302 if (UseAVX >= 2) {
1303 // clean upper bits of YMM registers
1304 __masm-> vpxor(xmm0, xmm0);
1305 __masm-> vpxor(xmm1, xmm1);
1306 }
1307 } else {
1308 // Copy 32-bytes per iteration
1309 __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":");
1310 __masm-> movq(to, Address(from, qword_count, Address::times_8, 24));
1311 __masm-> movq(Address(dest, qword_count, Address::times_8, 24), to);
1312 __masm-> movq(to, Address(from, qword_count, Address::times_8, 16));
1313 __masm-> movq(Address(dest, qword_count, Address::times_8, 16), to);
1314 __masm-> movq(to, Address(from, qword_count, Address::times_8, 8));
1315 __masm-> movq(Address(dest, qword_count, Address::times_8, 8), to);
1316 __masm-> movq(to, Address(from, qword_count, Address::times_8, 0));
1317 __masm-> movq(Address(dest, qword_count, Address::times_8, 0), to);
1318
1319 __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":"
);
1320 __masm-> subptr(qword_count, 4);
1321 __masm-> jcc(Assembler::greaterEqual, L_loop);
1322 }
1323 __masm-> addptr(qword_count, 4);
1324 __masm-> jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1325 }
1326
1327#ifndef PRODUCT
1328 int& get_profile_ctr(int shift) {
1329 if ( 0 == shift)
1330 return SharedRuntime::_jbyte_array_copy_ctr;
1331 else if(1 == shift)
1332 return SharedRuntime::_jshort_array_copy_ctr;
1333 else if(2 == shift)
1334 return SharedRuntime::_jint_array_copy_ctr;
1335 else
1336 return SharedRuntime::_jlong_array_copy_ctr;
1337 }
1338#endif
1339
1340 void setup_argument_regs(BasicType type) {
1341 if (type == T_BYTE || type == T_SHORT) {
1342 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1343 // r9 and r10 may be used to save non-volatile registers
1344 } else {
1345 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
1346 // r9 is used to save r15_thread
1347 }
1348 }
1349
1350 void restore_argument_regs(BasicType type) {
1351 if (type == T_BYTE || type == T_SHORT) {
1352 restore_arg_regs();
1353 } else {
1354 restore_arg_regs_using_thread();
1355 }
1356 }
1357
1358#if COMPILER2_OR_JVMCI1
1359 // Note: Following rules apply to AVX3 optimized arraycopy stubs:-
1360 // - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs)
1361 // for both special cases (various small block sizes) and aligned copy loop. This is the
1362 // default configuration.
1363 // - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs)
1364 // for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it.
1365 // - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a
1366 // better performance for disjoint copies. For conjoint/backward copy vector based
1367 // copy performs better.
1368 // - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over
1369 // 64 byte vector registers (ZMMs).
1370
1371 // Inputs:
1372 // c_rarg0 - source array address
1373 // c_rarg1 - destination array address
1374 // c_rarg2 - element count, treated as ssize_t, can be zero
1375 //
1376 //
1377 // Side Effects:
1378 // disjoint_copy_avx3_masked is set to the no-overlap entry point
1379 // used by generate_conjoint_[byte/int/short/long]_copy().
1380 //
1381
1382 address generate_disjoint_copy_avx3_masked(address* entry, const char *name, int shift,
1383 bool aligned, bool is_oop, bool dest_uninitialized) {
1384 __masm-> align(CodeEntryAlignment);
1385 StubCodeMark mark(this, "StubRoutines", name);
1386 address start = __masm-> pc();
1387 int avx3threshold = VM_Version::avx3_threshold();
1388 bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
1389 Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
1390 Label L_repmovs, L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
1391 const Register from = rdi; // source array address
1392 const Register to = rsi; // destination array address
1393 const Register count = rdx; // elements count
1394 const Register temp1 = r8;
1395 const Register temp2 = r11;
1396 const Register temp3 = rax;
1397 const Register temp4 = rcx;
1398 // End pointers are inclusive, and if count is not zero they point
1399 // to the last unit copied: end_to[0] := end_from[0]
1400
1401 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
1402 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
1403
1404 if (entry != NULL__null) {
1405 *entry = __masm-> pc();
1406 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1407 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
1408 }
1409
1410 BasicType type_vec[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
1411 BasicType type = is_oop ? T_OBJECT : type_vec[shift];
1412
1413 setup_argument_regs(type);
1414
1415 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1416 if (dest_uninitialized) {
1417 decorators |= IS_DEST_UNINITIALIZED;
1418 }
1419 if (aligned) {
1420 decorators |= ARRAYCOPY_ALIGNED;
1421 }
1422 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1423 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1424
1425 {
1426 // Type(shift) byte(0), short(1), int(2), long(3)
1427 int loop_size[] = { 192, 96, 48, 24};
1428 int threshold[] = { 4096, 2048, 1024, 512};
1429
1430 // UnsafeCopyMemory page error: continue after ucm
1431 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
1432 // 'from', 'to' and 'count' are now valid
1433
1434 // temp1 holds remaining count and temp4 holds running count used to compute
1435 // next address offset for start of to/from addresses (temp4 * scale).
1436 __masm-> mov64(temp4, 0);
1437 __masm-> movq(temp1, count);
1438
1439 // Zero length check.
1440 __masm-> BIND(L_tail)bind(L_tail); masm-> block_comment("L_tail" ":");
1441 __masm-> cmpq(temp1, 0);
1442 __masm-> jcc(Assembler::lessEqual, L_exit);
1443
1444 // Special cases using 32 byte [masked] vector copy operations.
1445 __masm-> arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift,
1446 temp4, temp3, use64byteVector, L_entry, L_exit);
1447
1448 // PRE-MAIN-POST loop for aligned copy.
1449 __masm-> BIND(L_entry)bind(L_entry); masm-> block_comment("L_entry" ":");
1450
1451 if (avx3threshold != 0) {
1452 __masm-> cmpq(count, threshold[shift]);
1453 if (MaxVectorSize == 64) {
1454 // Copy using 64 byte vectors.
1455 __masm-> jcc(Assembler::greaterEqual, L_pre_main_post_64);
1456 } else {
1457 assert(MaxVectorSize < 64, "vector size should be < 64 bytes")do { if (!(MaxVectorSize < 64)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1457, "assert(" "MaxVectorSize < 64" ") failed", "vector size should be < 64 bytes"
); ::breakpoint(); } } while (0);
1458 // REP MOVS offer a faster copy path.
1459 __masm-> jcc(Assembler::greaterEqual, L_repmovs);
1460 }
1461 }
1462
1463 if ((MaxVectorSize < 64) || (avx3threshold != 0)) {
1464 // Partial copy to make dst address 32 byte aligned.
1465 __masm-> movq(temp2, to);
1466 __masm-> andq(temp2, 31);
1467 __masm-> jcc(Assembler::equal, L_main_pre_loop);
1468
1469 __masm-> negptr(temp2);
1470 __masm-> addq(temp2, 32);
1471 if (shift) {
1472 __masm-> shrq(temp2, shift);
1473 }
1474 __masm-> movq(temp3, temp2);
1475 __masm-> copy32_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift);
1476 __masm-> movq(temp4, temp2);
1477 __masm-> movq(temp1, count);
1478 __masm-> subq(temp1, temp2);
1479
1480 __masm-> cmpq(temp1, loop_size[shift]);
1481 __masm-> jcc(Assembler::less, L_tail);
1482
1483 __masm-> BIND(L_main_pre_loop)bind(L_main_pre_loop); masm-> block_comment("L_main_pre_loop"
":");
1484 __masm-> subq(temp1, loop_size[shift]);
1485
1486 // Main loop with aligned copy block size of 192 bytes at 32 byte granularity.
1487 __masm-> align32();
1488 __masm-> BIND(L_main_loop)bind(L_main_loop); masm-> block_comment("L_main_loop" ":");
1489 __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 0);
1490 __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 64);
1491 __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 128);
1492 __masm-> addptr(temp4, loop_size[shift]);
1493 __masm-> subq(temp1, loop_size[shift]);
1494 __masm-> jcc(Assembler::greater, L_main_loop);
1495
1496 __masm-> addq(temp1, loop_size[shift]);
1497
1498 // Tail loop.
1499 __masm-> jmp(L_tail);
1500
1501 __masm-> BIND(L_repmovs)bind(L_repmovs); masm-> block_comment("L_repmovs" ":");
1502 __masm-> movq(temp2, temp1);
1503 // Swap to(RSI) and from(RDI) addresses to comply with REP MOVs semantics.
1504 __masm-> movq(temp3, to);
1505 __masm-> movq(to, from);
1506 __masm-> movq(from, temp3);
1507 // Save to/from for restoration post rep_mov.
1508 __masm-> movq(temp1, to);
1509 __masm-> movq(temp3, from);
1510 if(shift < 3) {
1511 __masm-> shrq(temp2, 3-shift); // quad word count
1512 }
1513 __masm-> movq(temp4 , temp2); // move quad ward count into temp4(RCX).
1514 __masm-> rep_mov();
1515 __masm-> shlq(temp2, 3); // convert quad words into byte count.
1516 if(shift) {
1517 __masm-> shrq(temp2, shift); // type specific count.
1518 }
1519 // Restore original addresses in to/from.
1520 __masm-> movq(to, temp3);
1521 __masm-> movq(from, temp1);
1522 __masm-> movq(temp4, temp2);
1523 __masm-> movq(temp1, count);
1524 __masm-> subq(temp1, temp2); // tailing part (less than a quad ward size).
1525 __masm-> jmp(L_tail);
1526 }
1527
1528 if (MaxVectorSize > 32) {
1529 __masm-> BIND(L_pre_main_post_64)bind(L_pre_main_post_64); masm-> block_comment("L_pre_main_post_64"
":");
1530 // Partial copy to make dst address 64 byte aligned.
1531 __masm-> movq(temp2, to);
1532 __masm-> andq(temp2, 63);
1533 __masm-> jcc(Assembler::equal, L_main_pre_loop_64bytes);
1534
1535 __masm-> negptr(temp2);
1536 __masm-> addq(temp2, 64);
1537 if (shift) {
1538 __masm-> shrq(temp2, shift);
1539 }
1540 __masm-> movq(temp3, temp2);
1541 __masm-> copy64_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift, 0 , true);
1542 __masm-> movq(temp4, temp2);
1543 __masm-> movq(temp1, count);
1544 __masm-> subq(temp1, temp2);
1545
1546 __masm-> cmpq(temp1, loop_size[shift]);
1547 __masm-> jcc(Assembler::less, L_tail64);
1548
1549 __masm-> BIND(L_main_pre_loop_64bytes)bind(L_main_pre_loop_64bytes); masm-> block_comment("L_main_pre_loop_64bytes"
":");
1550 __masm-> subq(temp1, loop_size[shift]);
1551
1552 // Main loop with aligned copy block size of 192 bytes at
1553 // 64 byte copy granularity.
1554 __masm-> align32();
1555 __masm-> BIND(L_main_loop_64bytes)bind(L_main_loop_64bytes); masm-> block_comment("L_main_loop_64bytes"
":");
1556 __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 0 , true);
1557 __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 64, true);
1558 __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 128, true);
1559 __masm-> addptr(temp4, loop_size[shift]);
1560 __masm-> subq(temp1, loop_size[shift]);
1561 __masm-> jcc(Assembler::greater, L_main_loop_64bytes);
1562
1563 __masm-> addq(temp1, loop_size[shift]);
1564 // Zero length check.
1565 __masm-> jcc(Assembler::lessEqual, L_exit);
1566
1567 __masm-> BIND(L_tail64)bind(L_tail64); masm-> block_comment("L_tail64" ":");
1568
1569 // Tail handling using 64 byte [masked] vector copy operations.
1570 use64byteVector = true;
1571 __masm-> arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift,
1572 temp4, temp3, use64byteVector, L_entry, L_exit);
1573 }
1574 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
1575 }
1576
1577 address ucme_exit_pc = __masm-> pc();
1578 // When called from generic_arraycopy r11 contains specific values
1579 // used during arraycopy epilogue, re-initializing r11.
1580 if (is_oop) {
1581 __masm-> movq(r11, shift == 3 ? count : to);
1582 }
1583 bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1584 restore_argument_regs(type);
1585 inc_counter_np(get_profile_ctr(shift))masm-> block_comment("inc_counter " "get_profile_ctr(shift)"
); inc_counter_np_(get_profile_ctr(shift));; // Update counter after rscratch1 is free
1586 __masm-> xorptr(rax, rax); // return 0
1587 __masm-> vzeroupper();
1588 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
1589 __masm-> ret(0);
1590 return start;
1591 }
1592
1593 // Inputs:
1594 // c_rarg0 - source array address
1595 // c_rarg1 - destination array address
1596 // c_rarg2 - element count, treated as ssize_t, can be zero
1597 //
1598 //
1599 address generate_conjoint_copy_avx3_masked(address* entry, const char *name, int shift,
1600 address nooverlap_target, bool aligned, bool is_oop,
1601 bool dest_uninitialized) {
1602 __masm-> align(CodeEntryAlignment);
1603 StubCodeMark mark(this, "StubRoutines", name);
1604 address start = __masm-> pc();
1605
1606 int avx3threshold = VM_Version::avx3_threshold();
1607 bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0);
1608
1609 Label L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64;
1610 Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry;
1611 const Register from = rdi; // source array address
1612 const Register to = rsi; // destination array address
1613 const Register count = rdx; // elements count
1614 const Register temp1 = r8;
1615 const Register temp2 = rcx;
1616 const Register temp3 = r11;
1617 const Register temp4 = rax;
1618 // End pointers are inclusive, and if count is not zero they point
1619 // to the last unit copied: end_to[0] := end_from[0]
1620
1621 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
1622 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
1623
1624 if (entry != NULL__null) {
1625 *entry = __masm-> pc();
1626 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1627 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
1628 }
1629
1630 array_overlap_test(nooverlap_target, (Address::ScaleFactor)(shift));
1631
1632 BasicType type_vec[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
1633 BasicType type = is_oop ? T_OBJECT : type_vec[shift];
1634
1635 setup_argument_regs(type);
1636
1637 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1638 if (dest_uninitialized) {
1639 decorators |= IS_DEST_UNINITIALIZED;
1640 }
1641 if (aligned) {
1642 decorators |= ARRAYCOPY_ALIGNED;
1643 }
1644 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1645 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1646 {
1647 // Type(shift) byte(0), short(1), int(2), long(3)
1648 int loop_size[] = { 192, 96, 48, 24};
1649 int threshold[] = { 4096, 2048, 1024, 512};
1650
1651 // UnsafeCopyMemory page error: continue after ucm
1652 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
1653 // 'from', 'to' and 'count' are now valid
1654
1655 // temp1 holds remaining count.
1656 __masm-> movq(temp1, count);
1657
1658 // Zero length check.
1659 __masm-> BIND(L_tail)bind(L_tail); masm-> block_comment("L_tail" ":");
1660 __masm-> cmpq(temp1, 0);
1661 __masm-> jcc(Assembler::lessEqual, L_exit);
1662
1663 __masm-> mov64(temp2, 0);
1664 __masm-> movq(temp3, temp1);
1665 // Special cases using 32 byte [masked] vector copy operations.
1666 __masm-> arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift,
1667 temp4, use64byteVector, L_entry, L_exit);
1668
1669 // PRE-MAIN-POST loop for aligned copy.
1670 __masm-> BIND(L_entry)bind(L_entry); masm-> block_comment("L_entry" ":");
1671
1672 if ((MaxVectorSize > 32) && (avx3threshold != 0)) {
1673 __masm-> cmpq(temp1, threshold[shift]);
1674 __masm-> jcc(Assembler::greaterEqual, L_pre_main_post_64);
1675 }
1676
1677 if ((MaxVectorSize < 64) || (avx3threshold != 0)) {
1678 // Partial copy to make dst address 32 byte aligned.
1679 __masm-> leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0));
1680 __masm-> andq(temp2, 31);
1681 __masm-> jcc(Assembler::equal, L_main_pre_loop);
1682
1683 if (shift) {
1684 __masm-> shrq(temp2, shift);
1685 }
1686 __masm-> subq(temp1, temp2);
1687 __masm-> copy32_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift);
1688
1689 __masm-> cmpq(temp1, loop_size[shift]);
1690 __masm-> jcc(Assembler::less, L_tail);
1691
1692 __masm-> BIND(L_main_pre_loop)bind(L_main_pre_loop); masm-> block_comment("L_main_pre_loop"
":");
1693
1694 // Main loop with aligned copy block size of 192 bytes at 32 byte granularity.
1695 __masm-> align32();
1696 __masm-> BIND(L_main_loop)bind(L_main_loop); masm-> block_comment("L_main_loop" ":");
1697 __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -64);
1698 __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -128);
1699 __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -192);
1700 __masm-> subptr(temp1, loop_size[shift]);
1701 __masm-> cmpq(temp1, loop_size[shift]);
1702 __masm-> jcc(Assembler::greater, L_main_loop);
1703
1704 // Tail loop.
1705 __masm-> jmp(L_tail);
1706 }
1707
1708 if (MaxVectorSize > 32) {
1709 __masm-> BIND(L_pre_main_post_64)bind(L_pre_main_post_64); masm-> block_comment("L_pre_main_post_64"
":");
1710 // Partial copy to make dst address 64 byte aligned.
1711 __masm-> leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0));
1712 __masm-> andq(temp2, 63);
1713 __masm-> jcc(Assembler::equal, L_main_pre_loop_64bytes);
1714
1715 if (shift) {
1716 __masm-> shrq(temp2, shift);
1717 }
1718 __masm-> subq(temp1, temp2);
1719 __masm-> copy64_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift, 0 , true);
1720
1721 __masm-> cmpq(temp1, loop_size[shift]);
1722 __masm-> jcc(Assembler::less, L_tail64);
1723
1724 __masm-> BIND(L_main_pre_loop_64bytes)bind(L_main_pre_loop_64bytes); masm-> block_comment("L_main_pre_loop_64bytes"
":");
1725
1726 // Main loop with aligned copy block size of 192 bytes at
1727 // 64 byte copy granularity.
1728 __masm-> align32();
1729 __masm-> BIND(L_main_loop_64bytes)bind(L_main_loop_64bytes); masm-> block_comment("L_main_loop_64bytes"
":");
1730 __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -64 , true);
1731 __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -128, true);
1732 __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -192, true);
1733 __masm-> subq(temp1, loop_size[shift]);
1734 __masm-> cmpq(temp1, loop_size[shift]);
1735 __masm-> jcc(Assembler::greater, L_main_loop_64bytes);
1736
1737 // Zero length check.
1738 __masm-> cmpq(temp1, 0);
1739 __masm-> jcc(Assembler::lessEqual, L_exit);
1740
1741 __masm-> BIND(L_tail64)bind(L_tail64); masm-> block_comment("L_tail64" ":");
1742
1743 // Tail handling using 64 byte [masked] vector copy operations.
1744 use64byteVector = true;
1745 __masm-> mov64(temp2, 0);
1746 __masm-> movq(temp3, temp1);
1747 __masm-> arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift,
1748 temp4, use64byteVector, L_entry, L_exit);
1749 }
1750 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
1751 }
1752 address ucme_exit_pc = __masm-> pc();
1753 // When called from generic_arraycopy r11 contains specific values
1754 // used during arraycopy epilogue, re-initializing r11.
1755 if(is_oop) {
1756 __masm-> movq(r11, count);
1757 }
1758 bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1759 restore_argument_regs(type);
1760 inc_counter_np(get_profile_ctr(shift))masm-> block_comment("inc_counter " "get_profile_ctr(shift)"
); inc_counter_np_(get_profile_ctr(shift));; // Update counter after rscratch1 is free
1761 __masm-> xorptr(rax, rax); // return 0
1762 __masm-> vzeroupper();
1763 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
1764 __masm-> ret(0);
1765 return start;
1766 }
1767#endif // COMPILER2_OR_JVMCI
1768
1769
1770 // Arguments:
1771 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1772 // ignored
1773 // name - stub name string
1774 //
1775 // Inputs:
1776 // c_rarg0 - source array address
1777 // c_rarg1 - destination array address
1778 // c_rarg2 - element count, treated as ssize_t, can be zero
1779 //
1780 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1781 // we let the hardware handle it. The one to eight bytes within words,
1782 // dwords or qwords that span cache line boundaries will still be loaded
1783 // and stored atomically.
1784 //
1785 // Side Effects:
1786 // disjoint_byte_copy_entry is set to the no-overlap entry point
1787 // used by generate_conjoint_byte_copy().
1788 //
1789 address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1790#if COMPILER2_OR_JVMCI1
1791 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
1792 return generate_disjoint_copy_avx3_masked(entry, "jbyte_disjoint_arraycopy_avx3", 0,
1793 aligned, false, false);
1794 }
1795#endif
1796 __masm-> align(CodeEntryAlignment);
1797 StubCodeMark mark(this, "StubRoutines", name);
1798 address start = __masm-> pc();
1799
1800 Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1801 Label L_copy_byte, L_exit;
1802 const Register from = rdi; // source array address
1803 const Register to = rsi; // destination array address
1804 const Register count = rdx; // elements count
1805 const Register byte_count = rcx;
1806 const Register qword_count = count;
1807 const Register end_from = from; // source array end address
1808 const Register end_to = to; // destination array end address
1809 // End pointers are inclusive, and if count is not zero they point
1810 // to the last unit copied: end_to[0] := end_from[0]
1811
1812 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
1813 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
1814
1815 if (entry != NULL__null) {
1816 *entry = __masm-> pc();
1817 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1818 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
1819 }
1820
1821 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1822 // r9 and r10 may be used to save non-volatile registers
1823
1824 {
1825 // UnsafeCopyMemory page error: continue after ucm
1826 UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1827 // 'from', 'to' and 'count' are now valid
1828 __masm-> movptr(byte_count, count);
1829 __masm-> shrptr(count, 3); // count => qword_count
1830
1831 // Copy from low to high addresses. Use 'to' as scratch.
1832 __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8));
1833 __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8));
1834 __masm-> negptr(qword_count); // make the count negative
1835 __masm-> jmp(L_copy_bytes);
1836
1837 // Copy trailing qwords
1838 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
1839 __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1840 __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1841 __masm-> increment(qword_count);
1842 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
1843
1844 // Check for and copy trailing dword
1845 __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes"
":");
1846 __masm-> testl(byte_count, 4);
1847 __masm-> jccb(Assembler::zero, L_copy_2_bytes)jccb_0(Assembler::zero, L_copy_2_bytes, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1847);
1848 __masm-> movl(rax, Address(end_from, 8));
1849 __masm-> movl(Address(end_to, 8), rax);
1850
1851 __masm-> addptr(end_from, 4);
1852 __masm-> addptr(end_to, 4);
1853
1854 // Check for and copy trailing word
1855 __masm-> BIND(L_copy_2_bytes)bind(L_copy_2_bytes); masm-> block_comment("L_copy_2_bytes"
":");
1856 __masm-> testl(byte_count, 2);
1857 __masm-> jccb(Assembler::zero, L_copy_byte)jccb_0(Assembler::zero, L_copy_byte, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1857);
1858 __masm-> movw(rax, Address(end_from, 8));
1859 __masm-> movw(Address(end_to, 8), rax);
1860
1861 __masm-> addptr(end_from, 2);
1862 __masm-> addptr(end_to, 2);
1863
1864 // Check for and copy trailing byte
1865 __masm-> BIND(L_copy_byte)bind(L_copy_byte); masm-> block_comment("L_copy_byte" ":");
1866 __masm-> testl(byte_count, 1);
1867 __masm-> jccb(Assembler::zero, L_exit)jccb_0(Assembler::zero, L_exit, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 1867);
1868 __masm-> movb(rax, Address(end_from, 8));
1869 __masm-> movb(Address(end_to, 8), rax);
1870 }
1871 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
1872 address ucme_exit_pc = __masm-> pc();
1873 restore_arg_regs();
1874 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jbyte_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jbyte_array_copy_ctr);; // Update counter after rscratch1 is free
1875 __masm-> xorptr(rax, rax); // return 0
1876 __masm-> vzeroupper();
1877 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
1878 __masm-> ret(0);
1879
1880 {
1881 UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
1882 // Copy in multi-bytes chunks
1883 copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1884 __masm-> jmp(L_copy_4_bytes);
1885 }
1886 return start;
1887 }
1888
1889 // Arguments:
1890 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1891 // ignored
1892 // name - stub name string
1893 //
1894 // Inputs:
1895 // c_rarg0 - source array address
1896 // c_rarg1 - destination array address
1897 // c_rarg2 - element count, treated as ssize_t, can be zero
1898 //
1899 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1900 // we let the hardware handle it. The one to eight bytes within words,
1901 // dwords or qwords that span cache line boundaries will still be loaded
1902 // and stored atomically.
1903 //
1904 address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1905 address* entry, const char *name) {
1906#if COMPILER2_OR_JVMCI1
1907 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
1908 return generate_conjoint_copy_avx3_masked(entry, "jbyte_conjoint_arraycopy_avx3", 0,
1909 nooverlap_target, aligned, false, false);
1910 }
1911#endif
1912 __masm-> align(CodeEntryAlignment);
1913 StubCodeMark mark(this, "StubRoutines", name);
1914 address start = __masm-> pc();
1915
1916 Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1917 const Register from = rdi; // source array address
1918 const Register to = rsi; // destination array address
1919 const Register count = rdx; // elements count
1920 const Register byte_count = rcx;
1921 const Register qword_count = count;
1922
1923 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
1924 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
1925
1926 if (entry != NULL__null) {
1927 *entry = __masm-> pc();
1928 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1929 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
1930 }
1931
1932 array_overlap_test(nooverlap_target, Address::times_1);
1933 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1934 // r9 and r10 may be used to save non-volatile registers
1935
1936 {
1937 // UnsafeCopyMemory page error: continue after ucm
1938 UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1939 // 'from', 'to' and 'count' are now valid
1940 __masm-> movptr(byte_count, count);
1941 __masm-> shrptr(count, 3); // count => qword_count
1942
1943 // Copy from high to low addresses.
1944
1945 // Check for and copy trailing byte
1946 __masm-> testl(byte_count, 1);
1947 __masm-> jcc(Assembler::zero, L_copy_2_bytes);
1948 __masm-> movb(rax, Address(from, byte_count, Address::times_1, -1));
1949 __masm-> movb(Address(to, byte_count, Address::times_1, -1), rax);
1950 __masm-> decrement(byte_count); // Adjust for possible trailing word
1951
1952 // Check for and copy trailing word
1953 __masm-> BIND(L_copy_2_bytes)bind(L_copy_2_bytes); masm-> block_comment("L_copy_2_bytes"
":");
1954 __masm-> testl(byte_count, 2);
1955 __masm-> jcc(Assembler::zero, L_copy_4_bytes);
1956 __masm-> movw(rax, Address(from, byte_count, Address::times_1, -2));
1957 __masm-> movw(Address(to, byte_count, Address::times_1, -2), rax);
1958
1959 // Check for and copy trailing dword
1960 __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes"
":");
1961 __masm-> testl(byte_count, 4);
1962 __masm-> jcc(Assembler::zero, L_copy_bytes);
1963 __masm-> movl(rax, Address(from, qword_count, Address::times_8));
1964 __masm-> movl(Address(to, qword_count, Address::times_8), rax);
1965 __masm-> jmp(L_copy_bytes);
1966
1967 // Copy trailing qwords
1968 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
1969 __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8));
1970 __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax);
1971 __masm-> decrement(qword_count);
1972 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
1973 }
1974 restore_arg_regs();
1975 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jbyte_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jbyte_array_copy_ctr);; // Update counter after rscratch1 is free
1976 __masm-> xorptr(rax, rax); // return 0
1977 __masm-> vzeroupper();
1978 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
1979 __masm-> ret(0);
1980
1981 {
1982 // UnsafeCopyMemory page error: continue after ucm
1983 UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1984 // Copy in multi-bytes chunks
1985 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1986 }
1987 restore_arg_regs();
1988 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jbyte_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jbyte_array_copy_ctr);; // Update counter after rscratch1 is free
1989 __masm-> xorptr(rax, rax); // return 0
1990 __masm-> vzeroupper();
1991 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
1992 __masm-> ret(0);
1993
1994 return start;
1995 }
1996
1997 // Arguments:
1998 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1999 // ignored
2000 // name - stub name string
2001 //
2002 // Inputs:
2003 // c_rarg0 - source array address
2004 // c_rarg1 - destination array address
2005 // c_rarg2 - element count, treated as ssize_t, can be zero
2006 //
2007 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
2008 // let the hardware handle it. The two or four words within dwords
2009 // or qwords that span cache line boundaries will still be loaded
2010 // and stored atomically.
2011 //
2012 // Side Effects:
2013 // disjoint_short_copy_entry is set to the no-overlap entry point
2014 // used by generate_conjoint_short_copy().
2015 //
2016 address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
2017#if COMPILER2_OR_JVMCI1
2018 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
2019 return generate_disjoint_copy_avx3_masked(entry, "jshort_disjoint_arraycopy_avx3", 1,
2020 aligned, false, false);
2021 }
2022#endif
2023
2024 __masm-> align(CodeEntryAlignment);
2025 StubCodeMark mark(this, "StubRoutines", name);
2026 address start = __masm-> pc();
2027
2028 Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
2029 const Register from = rdi; // source array address
2030 const Register to = rsi; // destination array address
2031 const Register count = rdx; // elements count
2032 const Register word_count = rcx;
2033 const Register qword_count = count;
2034 const Register end_from = from; // source array end address
2035 const Register end_to = to; // destination array end address
2036 // End pointers are inclusive, and if count is not zero they point
2037 // to the last unit copied: end_to[0] := end_from[0]
2038
2039 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2040 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
2041
2042 if (entry != NULL__null) {
2043 *entry = __masm-> pc();
2044 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2045 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2046 }
2047
2048 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2049 // r9 and r10 may be used to save non-volatile registers
2050
2051 {
2052 // UnsafeCopyMemory page error: continue after ucm
2053 UnsafeCopyMemoryMark ucmm(this, !aligned, true);
2054 // 'from', 'to' and 'count' are now valid
2055 __masm-> movptr(word_count, count);
2056 __masm-> shrptr(count, 2); // count => qword_count
2057
2058 // Copy from low to high addresses. Use 'to' as scratch.
2059 __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8));
2060 __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8));
2061 __masm-> negptr(qword_count);
2062 __masm-> jmp(L_copy_bytes);
2063
2064 // Copy trailing qwords
2065 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
2066 __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2067 __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2068 __masm-> increment(qword_count);
2069 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
2070
2071 // Original 'dest' is trashed, so we can't use it as a
2072 // base register for a possible trailing word copy
2073
2074 // Check for and copy trailing dword
2075 __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes"
":");
2076 __masm-> testl(word_count, 2);
2077 __masm-> jccb(Assembler::zero, L_copy_2_bytes)jccb_0(Assembler::zero, L_copy_2_bytes, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2077);
2078 __masm-> movl(rax, Address(end_from, 8));
2079 __masm-> movl(Address(end_to, 8), rax);
2080
2081 __masm-> addptr(end_from, 4);
2082 __masm-> addptr(end_to, 4);
2083
2084 // Check for and copy trailing word
2085 __masm-> BIND(L_copy_2_bytes)bind(L_copy_2_bytes); masm-> block_comment("L_copy_2_bytes"
":");
2086 __masm-> testl(word_count, 1);
2087 __masm-> jccb(Assembler::zero, L_exit)jccb_0(Assembler::zero, L_exit, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2087);
2088 __masm-> movw(rax, Address(end_from, 8));
2089 __masm-> movw(Address(end_to, 8), rax);
2090 }
2091 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
2092 address ucme_exit_pc = __masm-> pc();
2093 restore_arg_regs();
2094 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jshort_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jshort_array_copy_ctr);; // Update counter after rscratch1 is free
2095 __masm-> xorptr(rax, rax); // return 0
2096 __masm-> vzeroupper();
2097 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2098 __masm-> ret(0);
2099
2100 {
2101 UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
2102 // Copy in multi-bytes chunks
2103 copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2104 __masm-> jmp(L_copy_4_bytes);
2105 }
2106
2107 return start;
2108 }
2109
2110 address generate_fill(BasicType t, bool aligned, const char *name) {
2111 __masm-> align(CodeEntryAlignment);
2112 StubCodeMark mark(this, "StubRoutines", name);
2113 address start = __masm-> pc();
2114
2115 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2116
2117 const Register to = c_rarg0; // destination array address
2118 const Register value = c_rarg1; // value
2119 const Register count = c_rarg2; // elements count
2120 __masm-> mov(r11, count);
2121
2122 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2123
2124 __masm-> generate_fill(t, aligned, to, value, r11, rax, xmm0);
2125
2126 __masm-> vzeroupper();
2127 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2128 __masm-> ret(0);
2129 return start;
2130 }
2131
2132 // Arguments:
2133 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
2134 // ignored
2135 // name - stub name string
2136 //
2137 // Inputs:
2138 // c_rarg0 - source array address
2139 // c_rarg1 - destination array address
2140 // c_rarg2 - element count, treated as ssize_t, can be zero
2141 //
2142 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
2143 // let the hardware handle it. The two or four words within dwords
2144 // or qwords that span cache line boundaries will still be loaded
2145 // and stored atomically.
2146 //
2147 address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
2148 address *entry, const char *name) {
2149#if COMPILER2_OR_JVMCI1
2150 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
2151 return generate_conjoint_copy_avx3_masked(entry, "jshort_conjoint_arraycopy_avx3", 1,
2152 nooverlap_target, aligned, false, false);
2153 }
2154#endif
2155 __masm-> align(CodeEntryAlignment);
2156 StubCodeMark mark(this, "StubRoutines", name);
2157 address start = __masm-> pc();
2158
2159 Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
2160 const Register from = rdi; // source array address
2161 const Register to = rsi; // destination array address
2162 const Register count = rdx; // elements count
2163 const Register word_count = rcx;
2164 const Register qword_count = count;
2165
2166 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2167 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
2168
2169 if (entry != NULL__null) {
2170 *entry = __masm-> pc();
2171 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2172 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2173 }
2174
2175 array_overlap_test(nooverlap_target, Address::times_2);
2176 setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2177 // r9 and r10 may be used to save non-volatile registers
2178
2179 {
2180 // UnsafeCopyMemory page error: continue after ucm
2181 UnsafeCopyMemoryMark ucmm(this, !aligned, true);
2182 // 'from', 'to' and 'count' are now valid
2183 __masm-> movptr(word_count, count);
2184 __masm-> shrptr(count, 2); // count => qword_count
2185
2186 // Copy from high to low addresses. Use 'to' as scratch.
2187
2188 // Check for and copy trailing word
2189 __masm-> testl(word_count, 1);
2190 __masm-> jccb(Assembler::zero, L_copy_4_bytes)jccb_0(Assembler::zero, L_copy_4_bytes, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2190);
2191 __masm-> movw(rax, Address(from, word_count, Address::times_2, -2));
2192 __masm-> movw(Address(to, word_count, Address::times_2, -2), rax);
2193
2194 // Check for and copy trailing dword
2195 __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes"
":");
2196 __masm-> testl(word_count, 2);
2197 __masm-> jcc(Assembler::zero, L_copy_bytes);
2198 __masm-> movl(rax, Address(from, qword_count, Address::times_8));
2199 __masm-> movl(Address(to, qword_count, Address::times_8), rax);
2200 __masm-> jmp(L_copy_bytes);
2201
2202 // Copy trailing qwords
2203 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
2204 __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8));
2205 __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax);
2206 __masm-> decrement(qword_count);
2207 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
2208 }
2209 restore_arg_regs();
2210 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jshort_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jshort_array_copy_ctr);; // Update counter after rscratch1 is free
2211 __masm-> xorptr(rax, rax); // return 0
2212 __masm-> vzeroupper();
2213 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2214 __masm-> ret(0);
2215
2216 {
2217 // UnsafeCopyMemory page error: continue after ucm
2218 UnsafeCopyMemoryMark ucmm(this, !aligned, true);
2219 // Copy in multi-bytes chunks
2220 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2221 }
2222 restore_arg_regs();
2223 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jshort_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jshort_array_copy_ctr);; // Update counter after rscratch1 is free
2224 __masm-> xorptr(rax, rax); // return 0
2225 __masm-> vzeroupper();
2226 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2227 __masm-> ret(0);
2228
2229 return start;
2230 }
2231
2232 // Arguments:
2233 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
2234 // ignored
2235 // is_oop - true => oop array, so generate store check code
2236 // name - stub name string
2237 //
2238 // Inputs:
2239 // c_rarg0 - source array address
2240 // c_rarg1 - destination array address
2241 // c_rarg2 - element count, treated as ssize_t, can be zero
2242 //
2243 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
2244 // the hardware handle it. The two dwords within qwords that span
2245 // cache line boundaries will still be loaded and stored atomicly.
2246 //
2247 // Side Effects:
2248 // disjoint_int_copy_entry is set to the no-overlap entry point
2249 // used by generate_conjoint_int_oop_copy().
2250 //
2251 address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
2252 const char *name, bool dest_uninitialized = false) {
2253#if COMPILER2_OR_JVMCI1
2254 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
2255 return generate_disjoint_copy_avx3_masked(entry, "jint_disjoint_arraycopy_avx3", 2,
2256 aligned, is_oop, dest_uninitialized);
2257 }
2258#endif
2259
2260 __masm-> align(CodeEntryAlignment);
2261 StubCodeMark mark(this, "StubRoutines", name);
2262 address start = __masm-> pc();
2263
2264 Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
2265 const Register from = rdi; // source array address
2266 const Register to = rsi; // destination array address
2267 const Register count = rdx; // elements count
2268 const Register dword_count = rcx;
2269 const Register qword_count = count;
2270 const Register end_from = from; // source array end address
2271 const Register end_to = to; // destination array end address
2272 // End pointers are inclusive, and if count is not zero they point
2273 // to the last unit copied: end_to[0] := end_from[0]
2274
2275 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2276 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
2277
2278 if (entry != NULL__null) {
2279 *entry = __masm-> pc();
2280 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2281 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2282 }
2283
2284 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2285 // r9 is used to save r15_thread
2286
2287 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
2288 if (dest_uninitialized) {
2289 decorators |= IS_DEST_UNINITIALIZED;
2290 }
2291 if (aligned) {
2292 decorators |= ARRAYCOPY_ALIGNED;
2293 }
2294
2295 BasicType type = is_oop ? T_OBJECT : T_INT;
2296 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2297 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2298
2299 {
2300 // UnsafeCopyMemory page error: continue after ucm
2301 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2302 // 'from', 'to' and 'count' are now valid
2303 __masm-> movptr(dword_count, count);
2304 __masm-> shrptr(count, 1); // count => qword_count
2305
2306 // Copy from low to high addresses. Use 'to' as scratch.
2307 __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8));
2308 __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8));
2309 __masm-> negptr(qword_count);
2310 __masm-> jmp(L_copy_bytes);
2311
2312 // Copy trailing qwords
2313 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
2314 __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2315 __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2316 __masm-> increment(qword_count);
2317 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
2318
2319 // Check for and copy trailing dword
2320 __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes"
":");
2321 __masm-> testl(dword_count, 1); // Only byte test since the value is 0 or 1
2322 __masm-> jccb(Assembler::zero, L_exit)jccb_0(Assembler::zero, L_exit, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2322);
2323 __masm-> movl(rax, Address(end_from, 8));
2324 __masm-> movl(Address(end_to, 8), rax);
2325 }
2326 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
2327 address ucme_exit_pc = __masm-> pc();
2328 bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
2329 restore_arg_regs_using_thread();
2330 inc_counter_np(SharedRuntime::_jint_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jint_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jint_array_copy_ctr);; // Update counter after rscratch1 is free
2331 __masm-> vzeroupper();
2332 __masm-> xorptr(rax, rax); // return 0
2333 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2334 __masm-> ret(0);
2335
2336 {
2337 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, false, ucme_exit_pc);
2338 // Copy in multi-bytes chunks
2339 copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2340 __masm-> jmp(L_copy_4_bytes);
2341 }
2342
2343 return start;
2344 }
2345
2346 // Arguments:
2347 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
2348 // ignored
2349 // is_oop - true => oop array, so generate store check code
2350 // name - stub name string
2351 //
2352 // Inputs:
2353 // c_rarg0 - source array address
2354 // c_rarg1 - destination array address
2355 // c_rarg2 - element count, treated as ssize_t, can be zero
2356 //
2357 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
2358 // the hardware handle it. The two dwords within qwords that span
2359 // cache line boundaries will still be loaded and stored atomicly.
2360 //
2361 address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
2362 address *entry, const char *name,
2363 bool dest_uninitialized = false) {
2364#if COMPILER2_OR_JVMCI1
2365 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
2366 return generate_conjoint_copy_avx3_masked(entry, "jint_conjoint_arraycopy_avx3", 2,
2367 nooverlap_target, aligned, is_oop, dest_uninitialized);
2368 }
2369#endif
2370 __masm-> align(CodeEntryAlignment);
2371 StubCodeMark mark(this, "StubRoutines", name);
2372 address start = __masm-> pc();
2373
2374 Label L_copy_bytes, L_copy_8_bytes, L_exit;
2375 const Register from = rdi; // source array address
2376 const Register to = rsi; // destination array address
2377 const Register count = rdx; // elements count
2378 const Register dword_count = rcx;
2379 const Register qword_count = count;
2380
2381 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2382 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
2383
2384 if (entry != NULL__null) {
2385 *entry = __masm-> pc();
2386 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2387 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2388 }
2389
2390 array_overlap_test(nooverlap_target, Address::times_4);
2391 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2392 // r9 is used to save r15_thread
2393
2394 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2395 if (dest_uninitialized) {
2396 decorators |= IS_DEST_UNINITIALIZED;
2397 }
2398 if (aligned) {
2399 decorators |= ARRAYCOPY_ALIGNED;
2400 }
2401
2402 BasicType type = is_oop ? T_OBJECT : T_INT;
2403 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2404 // no registers are destroyed by this call
2405 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2406
2407 assert_clean_int(count, rax); // Make sure 'count' is clean int.
2408 {
2409 // UnsafeCopyMemory page error: continue after ucm
2410 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2411 // 'from', 'to' and 'count' are now valid
2412 __masm-> movptr(dword_count, count);
2413 __masm-> shrptr(count, 1); // count => qword_count
2414
2415 // Copy from high to low addresses. Use 'to' as scratch.
2416
2417 // Check for and copy trailing dword
2418 __masm-> testl(dword_count, 1);
2419 __masm-> jcc(Assembler::zero, L_copy_bytes);
2420 __masm-> movl(rax, Address(from, dword_count, Address::times_4, -4));
2421 __masm-> movl(Address(to, dword_count, Address::times_4, -4), rax);
2422 __masm-> jmp(L_copy_bytes);
2423
2424 // Copy trailing qwords
2425 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
2426 __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8));
2427 __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax);
2428 __masm-> decrement(qword_count);
2429 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
2430 }
2431 if (is_oop) {
2432 __masm-> jmp(L_exit);
2433 }
2434 restore_arg_regs_using_thread();
2435 inc_counter_np(SharedRuntime::_jint_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jint_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jint_array_copy_ctr);; // Update counter after rscratch1 is free
2436 __masm-> xorptr(rax, rax); // return 0
2437 __masm-> vzeroupper();
2438 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2439 __masm-> ret(0);
2440
2441 {
2442 // UnsafeCopyMemory page error: continue after ucm
2443 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2444 // Copy in multi-bytes chunks
2445 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2446 }
2447
2448 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
2449 bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
2450 restore_arg_regs_using_thread();
2451 inc_counter_np(SharedRuntime::_jint_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jint_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jint_array_copy_ctr);; // Update counter after rscratch1 is free
2452 __masm-> xorptr(rax, rax); // return 0
2453 __masm-> vzeroupper();
2454 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2455 __masm-> ret(0);
2456
2457 return start;
2458 }
2459
2460 // Arguments:
2461 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2462 // ignored
2463 // is_oop - true => oop array, so generate store check code
2464 // name - stub name string
2465 //
2466 // Inputs:
2467 // c_rarg0 - source array address
2468 // c_rarg1 - destination array address
2469 // c_rarg2 - element count, treated as ssize_t, can be zero
2470 //
2471 // Side Effects:
2472 // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
2473 // no-overlap entry point used by generate_conjoint_long_oop_copy().
2474 //
2475 address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
2476 const char *name, bool dest_uninitialized = false) {
2477#if COMPILER2_OR_JVMCI1
2478 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
2479 return generate_disjoint_copy_avx3_masked(entry, "jlong_disjoint_arraycopy_avx3", 3,
2480 aligned, is_oop, dest_uninitialized);
2481 }
2482#endif
2483 __masm-> align(CodeEntryAlignment);
2484 StubCodeMark mark(this, "StubRoutines", name);
2485 address start = __masm-> pc();
2486
2487 Label L_copy_bytes, L_copy_8_bytes, L_exit;
2488 const Register from = rdi; // source array address
2489 const Register to = rsi; // destination array address
2490 const Register qword_count = rdx; // elements count
2491 const Register end_from = from; // source array end address
2492 const Register end_to = rcx; // destination array end address
2493 const Register saved_count = r11;
2494 // End pointers are inclusive, and if count is not zero they point
2495 // to the last unit copied: end_to[0] := end_from[0]
2496
2497 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2498 // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2499 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
2500
2501 if (entry != NULL__null) {
2502 *entry = __masm-> pc();
2503 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2504 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2505 }
2506
2507 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2508 // r9 is used to save r15_thread
2509 // 'from', 'to' and 'qword_count' are now valid
2510
2511 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
2512 if (dest_uninitialized) {
2513 decorators |= IS_DEST_UNINITIALIZED;
2514 }
2515 if (aligned) {
2516 decorators |= ARRAYCOPY_ALIGNED;
2517 }
2518
2519 BasicType type = is_oop ? T_OBJECT : T_LONG;
2520 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2521 bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2522 {
2523 // UnsafeCopyMemory page error: continue after ucm
2524 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2525
2526 // Copy from low to high addresses. Use 'to' as scratch.
2527 __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8));
2528 __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8));
2529 __masm-> negptr(qword_count);
2530 __masm-> jmp(L_copy_bytes);
2531
2532 // Copy trailing qwords
2533 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
2534 __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2535 __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2536 __masm-> increment(qword_count);
2537 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
2538 }
2539 if (is_oop) {
2540 __masm-> jmp(L_exit);
2541 } else {
2542 restore_arg_regs_using_thread();
2543 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free
2544 __masm-> xorptr(rax, rax); // return 0
2545 __masm-> vzeroupper();
2546 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2547 __masm-> ret(0);
2548 }
2549
2550 {
2551 // UnsafeCopyMemory page error: continue after ucm
2552 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2553 // Copy in multi-bytes chunks
2554 copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2555 }
2556
2557 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
2558 bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2559 restore_arg_regs_using_thread();
2560 if (is_oop) {
2561 inc_counter_np(SharedRuntime::_oop_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_oop_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_oop_array_copy_ctr);; // Update counter after rscratch1 is free
2562 } else {
2563 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free
2564 }
2565 __masm-> vzeroupper();
2566 __masm-> xorptr(rax, rax); // return 0
2567 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2568 __masm-> ret(0);
2569
2570 return start;
2571 }
2572
2573 // Arguments:
2574 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2575 // ignored
2576 // is_oop - true => oop array, so generate store check code
2577 // name - stub name string
2578 //
2579 // Inputs:
2580 // c_rarg0 - source array address
2581 // c_rarg1 - destination array address
2582 // c_rarg2 - element count, treated as ssize_t, can be zero
2583 //
2584 address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
2585 address nooverlap_target, address *entry,
2586 const char *name, bool dest_uninitialized = false) {
2587#if COMPILER2_OR_JVMCI1
2588 if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) {
2589 return generate_conjoint_copy_avx3_masked(entry, "jlong_conjoint_arraycopy_avx3", 3,
2590 nooverlap_target, aligned, is_oop, dest_uninitialized);
2591 }
2592#endif
2593 __masm-> align(CodeEntryAlignment);
2594 StubCodeMark mark(this, "StubRoutines", name);
2595 address start = __masm-> pc();
2596
2597 Label L_copy_bytes, L_copy_8_bytes, L_exit;
2598 const Register from = rdi; // source array address
2599 const Register to = rsi; // destination array address
2600 const Register qword_count = rdx; // elements count
2601 const Register saved_count = rcx;
2602
2603 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2604 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
2605
2606 if (entry != NULL__null) {
2607 *entry = __masm-> pc();
2608 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2609 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2610 }
2611
2612 array_overlap_test(nooverlap_target, Address::times_8);
2613 setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2614 // r9 is used to save r15_thread
2615 // 'from', 'to' and 'qword_count' are now valid
2616
2617 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2618 if (dest_uninitialized) {
2619 decorators |= IS_DEST_UNINITIALIZED;
2620 }
2621 if (aligned) {
2622 decorators |= ARRAYCOPY_ALIGNED;
2623 }
2624
2625 BasicType type = is_oop ? T_OBJECT : T_LONG;
2626 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2627 bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2628 {
2629 // UnsafeCopyMemory page error: continue after ucm
2630 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2631
2632 __masm-> jmp(L_copy_bytes);
2633
2634 // Copy trailing qwords
2635 __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes"
":");
2636 __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8));
2637 __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax);
2638 __masm-> decrement(qword_count);
2639 __masm-> jcc(Assembler::notZero, L_copy_8_bytes);
2640 }
2641 if (is_oop) {
2642 __masm-> jmp(L_exit);
2643 } else {
2644 restore_arg_regs_using_thread();
2645 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free
2646 __masm-> xorptr(rax, rax); // return 0
2647 __masm-> vzeroupper();
2648 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2649 __masm-> ret(0);
2650 }
2651 {
2652 // UnsafeCopyMemory page error: continue after ucm
2653 UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2654
2655 // Copy in multi-bytes chunks
2656 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2657 }
2658 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
2659 bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2660 restore_arg_regs_using_thread();
2661 if (is_oop) {
2662 inc_counter_np(SharedRuntime::_oop_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_oop_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_oop_array_copy_ctr);; // Update counter after rscratch1 is free
2663 } else {
2664 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free
2665 }
2666 __masm-> vzeroupper();
2667 __masm-> xorptr(rax, rax); // return 0
2668 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2669 __masm-> ret(0);
2670
2671 return start;
2672 }
2673
2674
2675 // Helper for generating a dynamic type check.
2676 // Smashes no registers.
2677 void generate_type_check(Register sub_klass,
2678 Register super_check_offset,
2679 Register super_klass,
2680 Label& L_success) {
2681 assert_different_registers(sub_klass, super_check_offset, super_klass);
2682
2683 BLOCK_COMMENT("type_check:")masm-> block_comment("type_check:");
2684
2685 Label L_miss;
2686
2687 __masm-> check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, NULL__null,
2688 super_check_offset);
2689 __masm-> check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL__null);
2690
2691 // Fall through on failure!
2692 __masm-> BIND(L_miss)bind(L_miss); masm-> block_comment("L_miss" ":");
2693 }
2694
2695 //
2696 // Generate checkcasting array copy stub
2697 //
2698 // Input:
2699 // c_rarg0 - source array address
2700 // c_rarg1 - destination array address
2701 // c_rarg2 - element count, treated as ssize_t, can be zero
2702 // c_rarg3 - size_t ckoff (super_check_offset)
2703 // not Win64
2704 // c_rarg4 - oop ckval (super_klass)
2705 // Win64
2706 // rsp+40 - oop ckval (super_klass)
2707 //
2708 // Output:
2709 // rax == 0 - success
2710 // rax == -1^K - failure, where K is partial transfer count
2711 //
2712 address generate_checkcast_copy(const char *name, address *entry,
2713 bool dest_uninitialized = false) {
2714
2715 Label L_load_element, L_store_element, L_do_card_marks, L_done;
2716
2717 // Input registers (after setup_arg_regs)
2718 const Register from = rdi; // source array address
2719 const Register to = rsi; // destination array address
2720 const Register length = rdx; // elements count
2721 const Register ckoff = rcx; // super_check_offset
2722 const Register ckval = r8; // super_klass
2723
2724 // Registers used as temps (r13, r14 are save-on-entry)
2725 const Register end_from = from; // source array end address
2726 const Register end_to = r13; // destination array end address
2727 const Register count = rdx; // -(count_remaining)
2728 const Register r14_length = r14; // saved copy of length
2729 // End pointers are inclusive, and if length is not zero they point
2730 // to the last unit copied: end_to[0] := end_from[0]
2731
2732 const Register rax_oop = rax; // actual oop copied
2733 const Register r11_klass = r11; // oop._klass
2734
2735 //---------------------------------------------------------------
2736 // Assembler stub will be used for this call to arraycopy
2737 // if the two arrays are subtypes of Object[] but the
2738 // destination array type is not equal to or a supertype
2739 // of the source type. Each element must be separately
2740 // checked.
2741
2742 __masm-> align(CodeEntryAlignment);
2743 StubCodeMark mark(this, "StubRoutines", name);
2744 address start = __masm-> pc();
2745
2746 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2747
2748#ifdef ASSERT1
2749 // caller guarantees that the arrays really are different
2750 // otherwise, we would have to make conjoint checks
2751 { Label L;
2752 array_overlap_test(L, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8));
2753 __masm-> stop("checkcast_copy within a single array");
2754 __masm-> bind(L);
2755 }
2756#endif //ASSERT
2757
2758 setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2759 // ckoff => rcx, ckval => r8
2760 // r9 and r10 may be used to save non-volatile registers
2761#ifdef _WIN64
2762 // last argument (#4) is on stack on Win64
2763 __masm-> movptr(ckval, Address(rsp, 6 * wordSize));
2764#endif
2765
2766 // Caller of this entry point must set up the argument registers.
2767 if (entry != NULL__null) {
2768 *entry = __masm-> pc();
2769 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
2770 }
2771
2772 // allocate spill slots for r13, r14
2773 enum {
2774 saved_r13_offset,
2775 saved_r14_offset,
2776 saved_r10_offset,
2777 saved_rbp_offset
2778 };
2779 __masm-> subptr(rsp, saved_rbp_offset * wordSize);
2780 __masm-> movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2781 __masm-> movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2782 __masm-> movptr(Address(rsp, saved_r10_offset * wordSize), r10);
2783
2784#ifdef ASSERT1
2785 Label L2;
2786 __masm-> get_thread(r14);
2787 __masm-> cmpptr(r15_thread, r14);
2788 __masm-> jcc(Assembler::equal, L2);
2789 __masm-> stop("StubRoutines::call_stub: r15_thread is modified by call");
2790 __masm-> bind(L2);
2791#endif // ASSERT
2792
2793 // check that int operands are properly extended to size_t
2794 assert_clean_int(length, rax);
2795 assert_clean_int(ckoff, rax);
2796
2797#ifdef ASSERT1
2798 BLOCK_COMMENT("assert consistent ckoff/ckval")masm-> block_comment("assert consistent ckoff/ckval");
2799 // The ckoff and ckval must be mutually consistent,
2800 // even though caller generates both.
2801 { Label L;
2802 int sco_offset = in_bytes(Klass::super_check_offset_offset());
2803 __masm-> cmpl(ckoff, Address(ckval, sco_offset));
2804 __masm-> jcc(Assembler::equal, L);
2805 __masm-> stop("super_check_offset inconsistent");
2806 __masm-> bind(L);
2807 }
2808#endif //ASSERT
2809
2810 // Loop-invariant addresses. They are exclusive end pointers.
2811 Address end_from_addr(from, length, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0);
2812 Address end_to_addr(to, length, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0);
2813 // Loop-variant addresses. They assume post-incremented count < 0.
2814 Address from_element_addr(end_from, count, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0);
2815 Address to_element_addr(end_to, count, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0);
2816
2817 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
2818 if (dest_uninitialized) {
2819 decorators |= IS_DEST_UNINITIALIZED;
2820 }
2821
2822 BasicType type = T_OBJECT;
2823 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2824 bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2825
2826 // Copy from low to high addresses, indexed from the end of each array.
2827 __masm-> lea(end_from, end_from_addr);
2828 __masm-> lea(end_to, end_to_addr);
2829 __masm-> movptr(r14_length, length); // save a copy of the length
2830 assert(length == count, "")do { if (!(length == count)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2830, "assert(" "length == count" ") failed", ""); ::breakpoint
(); } } while (0); // else fix next line:
2831 __masm-> negptr(count); // negate and test the length
2832 __masm-> jcc(Assembler::notZero, L_load_element);
2833
2834 // Empty array: Nothing to do.
2835 __masm-> xorptr(rax, rax); // return 0 on (trivial) success
2836 __masm-> jmp(L_done);
2837
2838 // ======== begin loop ========
2839 // (Loop is rotated; its entry is L_load_element.)
2840 // Loop control:
2841 // for (count = -count; count != 0; count++)
2842 // Base pointers src, dst are biased by 8*(count-1),to last element.
2843 __masm-> align(OptoLoopAlignment);
2844
2845 __masm-> BIND(L_store_element)bind(L_store_element); masm-> block_comment("L_store_element"
":");
2846 __masm-> store_heap_oop(to_element_addr, rax_oop, noreg, noreg, AS_RAW); // store the oop
2847 __masm-> increment(count); // increment the count toward zero
2848 __masm-> jcc(Assembler::zero, L_do_card_marks);
2849
2850 // ======== loop entry is here ========
2851 __masm-> BIND(L_load_element)bind(L_load_element); masm-> block_comment("L_load_element"
":");
2852 __masm-> load_heap_oop(rax_oop, from_element_addr, noreg, noreg, AS_RAW); // load the oop
2853 __masm-> testptr(rax_oop, rax_oop);
2854 __masm-> jcc(Assembler::zero, L_store_element);
2855
2856 __masm-> load_klass(r11_klass, rax_oop, rscratch1);// query the object klass
2857 generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2858 // ======== end loop ========
2859
2860 // It was a real error; we must depend on the caller to finish the job.
2861 // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2862 // Emit GC store barriers for the oops we have copied (r14 + rdx),
2863 // and report their number to the caller.
2864 assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2865 Label L_post_barrier;
2866 __masm-> addptr(r14_length, count); // K = (original - remaining) oops
2867 __masm-> movptr(rax, r14_length); // save the value
2868 __masm-> notptr(rax); // report (-1^K) to caller (does not affect flags)
2869 __masm-> jccb(Assembler::notZero, L_post_barrier)jccb_0(Assembler::notZero, L_post_barrier, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2869);
2870 __masm-> jmp(L_done); // K == 0, nothing was copied, skip post barrier
2871
2872 // Come here on success only.
2873 __masm-> BIND(L_do_card_marks)bind(L_do_card_marks); masm-> block_comment("L_do_card_marks"
":");
2874 __masm-> xorptr(rax, rax); // return 0 on success
2875
2876 __masm-> BIND(L_post_barrier)bind(L_post_barrier); masm-> block_comment("L_post_barrier"
":");
2877 bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length);
2878
2879 // Common exit point (success or failure).
2880 __masm-> BIND(L_done)bind(L_done); masm-> block_comment("L_done" ":");
2881 __masm-> movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2882 __masm-> movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2883 __masm-> movptr(r10, Address(rsp, saved_r10_offset * wordSize));
2884 restore_arg_regs();
2885 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_checkcast_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_checkcast_array_copy_ctr);; // Update counter after rscratch1 is free
2886 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
2887 __masm-> ret(0);
2888
2889 return start;
2890 }
2891
2892 //
2893 // Generate 'unsafe' array copy stub
2894 // Though just as safe as the other stubs, it takes an unscaled
2895 // size_t argument instead of an element count.
2896 //
2897 // Input:
2898 // c_rarg0 - source array address
2899 // c_rarg1 - destination array address
2900 // c_rarg2 - byte count, treated as ssize_t, can be zero
2901 //
2902 // Examines the alignment of the operands and dispatches
2903 // to a long, int, short, or byte copy loop.
2904 //
2905 address generate_unsafe_copy(const char *name,
2906 address byte_copy_entry, address short_copy_entry,
2907 address int_copy_entry, address long_copy_entry) {
2908
2909 Label L_long_aligned, L_int_aligned, L_short_aligned;
2910
2911 // Input registers (before setup_arg_regs)
2912 const Register from = c_rarg0; // source array address
2913 const Register to = c_rarg1; // destination array address
2914 const Register size = c_rarg2; // byte count (size_t)
2915
2916 // Register used as a temp
2917 const Register bits = rax; // test copy of low bits
2918
2919 __masm-> align(CodeEntryAlignment);
2920 StubCodeMark mark(this, "StubRoutines", name);
2921 address start = __masm-> pc();
2922
2923 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
2924
2925 // bump this on entry, not on exit:
2926 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_unsafe_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_unsafe_array_copy_ctr);;
2927
2928 __masm-> mov(bits, from);
2929 __masm-> orptr(bits, to);
2930 __masm-> orptr(bits, size);
2931
2932 __masm-> testb(bits, BytesPerLong-1);
2933 __masm-> jccb(Assembler::zero, L_long_aligned)jccb_0(Assembler::zero, L_long_aligned, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2933);
2934
2935 __masm-> testb(bits, BytesPerInt-1);
2936 __masm-> jccb(Assembler::zero, L_int_aligned)jccb_0(Assembler::zero, L_int_aligned, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 2936);
2937
2938 __masm-> testb(bits, BytesPerShort-1);
2939 __masm-> jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2940
2941 __masm-> BIND(L_short_aligned)bind(L_short_aligned); masm-> block_comment("L_short_aligned"
":");
2942 __masm-> shrptr(size, LogBytesPerShort); // size => short_count
2943 __masm-> jump(RuntimeAddress(short_copy_entry));
2944
2945 __masm-> BIND(L_int_aligned)bind(L_int_aligned); masm-> block_comment("L_int_aligned" ":"
);
2946 __masm-> shrptr(size, LogBytesPerInt); // size => int_count
2947 __masm-> jump(RuntimeAddress(int_copy_entry));
2948
2949 __masm-> BIND(L_long_aligned)bind(L_long_aligned); masm-> block_comment("L_long_aligned"
":");
2950 __masm-> shrptr(size, LogBytesPerLong); // size => qword_count
2951 __masm-> jump(RuntimeAddress(long_copy_entry));
2952
2953 return start;
2954 }
2955
2956 // Perform range checks on the proposed arraycopy.
2957 // Kills temp, but nothing else.
2958 // Also, clean the sign bits of src_pos and dst_pos.
2959 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
2960 Register src_pos, // source position (c_rarg1)
2961 Register dst, // destination array oo (c_rarg2)
2962 Register dst_pos, // destination position (c_rarg3)
2963 Register length,
2964 Register temp,
2965 Label& L_failed) {
2966 BLOCK_COMMENT("arraycopy_range_checks:")masm-> block_comment("arraycopy_range_checks:");
2967
2968 // if (src_pos + length > arrayOop(src)->length()) FAIL;
2969 __masm-> movl(temp, length);
2970 __masm-> addl(temp, src_pos); // src_pos + length
2971 __masm-> cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2972 __masm-> jcc(Assembler::above, L_failed);
2973
2974 // if (dst_pos + length > arrayOop(dst)->length()) FAIL;
2975 __masm-> movl(temp, length);
2976 __masm-> addl(temp, dst_pos); // dst_pos + length
2977 __masm-> cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2978 __masm-> jcc(Assembler::above, L_failed);
2979
2980 // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2981 // Move with sign extension can be used since they are positive.
2982 __masm-> movslq(src_pos, src_pos);
2983 __masm-> movslq(dst_pos, dst_pos);
2984
2985 BLOCK_COMMENT("arraycopy_range_checks done")masm-> block_comment("arraycopy_range_checks done");
2986 }
2987
2988 //
2989 // Generate generic array copy stubs
2990 //
2991 // Input:
2992 // c_rarg0 - src oop
2993 // c_rarg1 - src_pos (32-bits)
2994 // c_rarg2 - dst oop
2995 // c_rarg3 - dst_pos (32-bits)
2996 // not Win64
2997 // c_rarg4 - element count (32-bits)
2998 // Win64
2999 // rsp+40 - element count (32-bits)
3000 //
3001 // Output:
3002 // rax == 0 - success
3003 // rax == -1^K - failure, where K is partial transfer count
3004 //
3005 address generate_generic_copy(const char *name,
3006 address byte_copy_entry, address short_copy_entry,
3007 address int_copy_entry, address oop_copy_entry,
3008 address long_copy_entry, address checkcast_copy_entry) {
3009
3010 Label L_failed, L_failed_0, L_objArray;
3011 Label L_copy_shorts, L_copy_ints, L_copy_longs;
3012
3013 // Input registers
3014 const Register src = c_rarg0; // source array oop
3015 const Register src_pos = c_rarg1; // source position
3016 const Register dst = c_rarg2; // destination array oop
3017 const Register dst_pos = c_rarg3; // destination position
3018#ifndef _WIN64
3019 const Register length = c_rarg4;
3020 const Register rklass_tmp = r9; // load_klass
3021#else
3022 const Address length(rsp, 7 * wordSize); // elements count is on stack on Win64
3023 const Register rklass_tmp = rdi; // load_klass
3024#endif
3025
3026 { int modulus = CodeEntryAlignment;
3027 int target = modulus - 5; // 5 = sizeof jmp(L_failed)
3028 int advance = target - (__masm-> offset() % modulus);
3029 if (advance < 0) advance += modulus;
3030 if (advance > 0) __masm-> nop(advance);
3031 }
3032 StubCodeMark mark(this, "StubRoutines", name);
3033
3034 // Short-hop target to L_failed. Makes for denser prologue code.
3035 __masm-> BIND(L_failed_0)bind(L_failed_0); masm-> block_comment("L_failed_0" ":");
3036 __masm-> jmp(L_failed);
3037 assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed")do { if (!(masm-> offset() % CodeEntryAlignment == 0)) { (
*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3037, "assert(" "masm-> offset() % CodeEntryAlignment == 0"
") failed", "no further alignment needed"); ::breakpoint(); }
} while (0);
3038
3039 __masm-> align(CodeEntryAlignment);
3040 address start = __masm-> pc();
3041
3042 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
3043
3044#ifdef _WIN64
3045 __masm-> push(rklass_tmp); // rdi is callee-save on Windows
3046#endif
3047
3048 // bump this on entry, not on exit:
3049 inc_counter_np(SharedRuntime::_generic_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_generic_array_copy_ctr"
); inc_counter_np_(SharedRuntime::_generic_array_copy_ctr);;
3050
3051 //-----------------------------------------------------------------------
3052 // Assembler stub will be used for this call to arraycopy
3053 // if the following conditions are met:
3054 //
3055 // (1) src and dst must not be null.
3056 // (2) src_pos must not be negative.
3057 // (3) dst_pos must not be negative.
3058 // (4) length must not be negative.
3059 // (5) src klass and dst klass should be the same and not NULL.
3060 // (6) src and dst should be arrays.
3061 // (7) src_pos + length must not exceed length of src.
3062 // (8) dst_pos + length must not exceed length of dst.
3063 //
3064
3065 // if (src == NULL) return -1;
3066 __masm-> testptr(src, src); // src oop
3067 size_t j1off = __masm-> offset();
3068 __masm-> jccb(Assembler::zero, L_failed_0)jccb_0(Assembler::zero, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3068);
3069
3070 // if (src_pos < 0) return -1;
3071 __masm-> testl(src_pos, src_pos); // src_pos (32-bits)
3072 __masm-> jccb(Assembler::negative, L_failed_0)jccb_0(Assembler::negative, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3072);
3073
3074 // if (dst == NULL) return -1;
3075 __masm-> testptr(dst, dst); // dst oop
3076 __masm-> jccb(Assembler::zero, L_failed_0)jccb_0(Assembler::zero, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3076);
3077
3078 // if (dst_pos < 0) return -1;
3079 __masm-> testl(dst_pos, dst_pos); // dst_pos (32-bits)
3080 size_t j4off = __masm-> offset();
3081 __masm-> jccb(Assembler::negative, L_failed_0)jccb_0(Assembler::negative, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3081);
3082
3083 // The first four tests are very dense code,
3084 // but not quite dense enough to put four
3085 // jumps in a 16-byte instruction fetch buffer.
3086 // That's good, because some branch predicters
3087 // do not like jumps so close together.
3088 // Make sure of this.
3089 guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps")do { if (!(((j1off ^ j4off) & ~15) != 0)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3089, "guarantee(" "((j1off ^ j4off) & ~15) != 0" ") failed"
, "I$ line of 1st & 4th jumps"); ::breakpoint(); } } while
(0);
3090
3091 // registers used as temp
3092 const Register r11_length = r11; // elements count to copy
3093 const Register r10_src_klass = r10; // array klass
3094
3095 // if (length < 0) return -1;
3096 __masm-> movl(r11_length, length); // length (elements count, 32-bits value)
3097 __masm-> testl(r11_length, r11_length);
3098 __masm-> jccb(Assembler::negative, L_failed_0)jccb_0(Assembler::negative, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3098);
3099
3100 __masm-> load_klass(r10_src_klass, src, rklass_tmp);
3101#ifdef ASSERT1
3102 // assert(src->klass() != NULL);
3103 {
3104 BLOCK_COMMENT("assert klasses not null {")masm-> block_comment("assert klasses not null {");
3105 Label L1, L2;
3106 __masm-> testptr(r10_src_klass, r10_src_klass);
3107 __masm-> jcc(Assembler::notZero, L2); // it is broken if klass is NULL
3108 __masm-> bind(L1);
3109 __masm-> stop("broken null klass");
3110 __masm-> bind(L2);
3111 __masm-> load_klass(rax, dst, rklass_tmp);
3112 __masm-> cmpq(rax, 0);
3113 __masm-> jcc(Assembler::equal, L1); // this would be broken also
3114 BLOCK_COMMENT("} assert klasses not null done")masm-> block_comment("} assert klasses not null done");
3115 }
3116#endif
3117
3118 // Load layout helper (32-bits)
3119 //
3120 // |array_tag| | header_size | element_type | |log2_element_size|
3121 // 32 30 24 16 8 2 0
3122 //
3123 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
3124 //
3125
3126 const int lh_offset = in_bytes(Klass::layout_helper_offset());
3127
3128 // Handle objArrays completely differently...
3129 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3130 __masm-> cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
3131 __masm-> jcc(Assembler::equal, L_objArray);
3132
3133 // if (src->klass() != dst->klass()) return -1;
3134 __masm-> load_klass(rax, dst, rklass_tmp);
3135 __masm-> cmpq(r10_src_klass, rax);
3136 __masm-> jcc(Assembler::notEqual, L_failed);
3137
3138 const Register rax_lh = rax; // layout helper
3139 __masm-> movl(rax_lh, Address(r10_src_klass, lh_offset));
3140
3141 // if (!src->is_Array()) return -1;
3142 __masm-> cmpl(rax_lh, Klass::_lh_neutral_value);
3143 __masm-> jcc(Assembler::greaterEqual, L_failed);
3144
3145 // At this point, it is known to be a typeArray (array_tag 0x3).
3146#ifdef ASSERT1
3147 {
3148 BLOCK_COMMENT("assert primitive array {")masm-> block_comment("assert primitive array {");
3149 Label L;
3150 __masm-> cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
3151 __masm-> jcc(Assembler::greaterEqual, L);
3152 __masm-> stop("must be a primitive array");
3153 __masm-> bind(L);
3154 BLOCK_COMMENT("} assert primitive array done")masm-> block_comment("} assert primitive array done");
3155 }
3156#endif
3157
3158 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
3159 r10, L_failed);
3160
3161 // TypeArrayKlass
3162 //
3163 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
3164 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
3165 //
3166
3167 const Register r10_offset = r10; // array offset
3168 const Register rax_elsize = rax_lh; // element size
3169
3170 __masm-> movl(r10_offset, rax_lh);
3171 __masm-> shrl(r10_offset, Klass::_lh_header_size_shift);
3172 __masm-> andptr(r10_offset, Klass::_lh_header_size_mask); // array_offset
3173 __masm-> addptr(src, r10_offset); // src array offset
3174 __masm-> addptr(dst, r10_offset); // dst array offset
3175 BLOCK_COMMENT("choose copy loop based on element size")masm-> block_comment("choose copy loop based on element size"
);
3176 __masm-> andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
3177
3178#ifdef _WIN64
3179 __masm-> pop(rklass_tmp); // Restore callee-save rdi
3180#endif
3181
3182 // next registers should be set before the jump to corresponding stub
3183 const Register from = c_rarg0; // source array address
3184 const Register to = c_rarg1; // destination array address
3185 const Register count = c_rarg2; // elements count
3186
3187 // 'from', 'to', 'count' registers should be set in such order
3188 // since they are the same as 'src', 'src_pos', 'dst'.
3189
3190 __masm-> cmpl(rax_elsize, 0);
3191 __masm-> jccb(Assembler::notEqual, L_copy_shorts)jccb_0(Assembler::notEqual, L_copy_shorts, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3191);
3192 __masm-> lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
3193 __masm-> lea(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr
3194 __masm-> movl2ptr(count, r11_length); // length
3195 __masm-> jump(RuntimeAddress(byte_copy_entry));
3196
3197 __masm-> BIND(L_copy_shorts)bind(L_copy_shorts); masm-> block_comment("L_copy_shorts" ":"
);
3198 __masm-> cmpl(rax_elsize, LogBytesPerShort);
3199 __masm-> jccb(Assembler::notEqual, L_copy_ints)jccb_0(Assembler::notEqual, L_copy_ints, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3199);
3200 __masm-> lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
3201 __masm-> lea(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr
3202 __masm-> movl2ptr(count, r11_length); // length
3203 __masm-> jump(RuntimeAddress(short_copy_entry));
3204
3205 __masm-> BIND(L_copy_ints)bind(L_copy_ints); masm-> block_comment("L_copy_ints" ":");
3206 __masm-> cmpl(rax_elsize, LogBytesPerInt);
3207 __masm-> jccb(Assembler::notEqual, L_copy_longs)jccb_0(Assembler::notEqual, L_copy_longs, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3207);
3208 __masm-> lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
3209 __masm-> lea(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr
3210 __masm-> movl2ptr(count, r11_length); // length
3211 __masm-> jump(RuntimeAddress(int_copy_entry));
3212
3213 __masm-> BIND(L_copy_longs)bind(L_copy_longs); masm-> block_comment("L_copy_longs" ":"
);
3214#ifdef ASSERT1
3215 {
3216 BLOCK_COMMENT("assert long copy {")masm-> block_comment("assert long copy {");
3217 Label L;
3218 __masm-> cmpl(rax_elsize, LogBytesPerLong);
3219 __masm-> jcc(Assembler::equal, L);
3220 __masm-> stop("must be long copy, but elsize is wrong");
3221 __masm-> bind(L);
3222 BLOCK_COMMENT("} assert long copy done")masm-> block_comment("} assert long copy done");
3223 }
3224#endif
3225 __masm-> lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
3226 __masm-> lea(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr
3227 __masm-> movl2ptr(count, r11_length); // length
3228 __masm-> jump(RuntimeAddress(long_copy_entry));
3229
3230 // ObjArrayKlass
3231 __masm-> BIND(L_objArray)bind(L_objArray); masm-> block_comment("L_objArray" ":");
3232 // live at this point: r10_src_klass, r11_length, src[_pos], dst[_pos]
3233
3234 Label L_plain_copy, L_checkcast_copy;
3235 // test array classes for subtyping
3236 __masm-> load_klass(rax, dst, rklass_tmp);
3237 __masm-> cmpq(r10_src_klass, rax); // usual case is exact equality
3238 __masm-> jcc(Assembler::notEqual, L_checkcast_copy);
3239
3240 // Identically typed arrays can be copied without element-wise checks.
3241 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
3242 r10, L_failed);
3243
3244 __masm-> lea(from, Address(src, src_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8),
3245 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
3246 __masm-> lea(to, Address(dst, dst_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8),
3247 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
3248 __masm-> movl2ptr(count, r11_length); // length
3249 __masm-> BIND(L_plain_copy)bind(L_plain_copy); masm-> block_comment("L_plain_copy" ":"
);
3250#ifdef _WIN64
3251 __masm-> pop(rklass_tmp); // Restore callee-save rdi
3252#endif
3253 __masm-> jump(RuntimeAddress(oop_copy_entry));
3254
3255 __masm-> BIND(L_checkcast_copy)bind(L_checkcast_copy); masm-> block_comment("L_checkcast_copy"
":");
3256 // live at this point: r10_src_klass, r11_length, rax (dst_klass)
3257 {
3258 // Before looking at dst.length, make sure dst is also an objArray.
3259 __masm-> cmpl(Address(rax, lh_offset), objArray_lh);
3260 __masm-> jcc(Assembler::notEqual, L_failed);
3261
3262 // It is safe to examine both src.length and dst.length.
3263 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
3264 rax, L_failed);
3265
3266 const Register r11_dst_klass = r11;
3267 __masm-> load_klass(r11_dst_klass, dst, rklass_tmp); // reload
3268
3269 // Marshal the base address arguments now, freeing registers.
3270 __masm-> lea(from, Address(src, src_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8),
3271 arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
3272 __masm-> lea(to, Address(dst, dst_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8),
3273 arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
3274 __masm-> movl(count, length); // length (reloaded)
3275 Register sco_temp = c_rarg3; // this register is free now
3276 assert_different_registers(from, to, count, sco_temp,
3277 r11_dst_klass, r10_src_klass);
3278 assert_clean_int(count, sco_temp);
3279
3280 // Generate the type check.
3281 const int sco_offset = in_bytes(Klass::super_check_offset_offset());
3282 __masm-> movl(sco_temp, Address(r11_dst_klass, sco_offset));
3283 assert_clean_int(sco_temp, rax);
3284 generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
3285
3286 // Fetch destination element klass from the ObjArrayKlass header.
3287 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
3288 __masm-> movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
3289 __masm-> movl( sco_temp, Address(r11_dst_klass, sco_offset));
3290 assert_clean_int(sco_temp, rax);
3291
3292#ifdef _WIN64
3293 __masm-> pop(rklass_tmp); // Restore callee-save rdi
3294#endif
3295
3296 // the checkcast_copy loop needs two extra arguments:
3297 assert(c_rarg3 == sco_temp, "#3 already in place")do { if (!(c_rarg3 == sco_temp)) { (*g_assert_poison) = 'X';;
report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3297, "assert(" "c_rarg3 == sco_temp" ") failed", "#3 already in place"
); ::breakpoint(); } } while (0);
3298 // Set up arguments for checkcast_copy_entry.
3299 setup_arg_regs(4);
3300 __masm-> movptr(r8, r11_dst_klass); // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
3301 __masm-> jump(RuntimeAddress(checkcast_copy_entry));
3302 }
3303
3304 __masm-> BIND(L_failed)bind(L_failed); masm-> block_comment("L_failed" ":");
3305#ifdef _WIN64
3306 __masm-> pop(rklass_tmp); // Restore callee-save rdi
3307#endif
3308 __masm-> xorptr(rax, rax);
3309 __masm-> notptr(rax); // return -1
3310 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
3311 __masm-> ret(0);
3312
3313 return start;
3314 }
3315
3316 address generate_data_cache_writeback() {
3317 const Register src = c_rarg0; // source address
3318
3319 __masm-> align(CodeEntryAlignment);
3320
3321 StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback");
3322
3323 address start = __masm-> pc();
3324 __masm-> enter();
3325 __masm-> cache_wb(Address(src, 0));
3326 __masm-> leave();
3327 __masm-> ret(0);
3328
3329 return start;
3330 }
3331
3332 address generate_data_cache_writeback_sync() {
3333 const Register is_pre = c_rarg0; // pre or post sync
3334
3335 __masm-> align(CodeEntryAlignment);
3336
3337 StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync");
3338
3339 // pre wbsync is a no-op
3340 // post wbsync translates to an sfence
3341
3342 Label skip;
3343 address start = __masm-> pc();
3344 __masm-> enter();
3345 __masm-> cmpl(is_pre, 0);
3346 __masm-> jcc(Assembler::notEqual, skip);
3347 __masm-> cache_wbsync(false);
3348 __masm-> bind(skip);
3349 __masm-> leave();
3350 __masm-> ret(0);
3351
3352 return start;
3353 }
3354
3355 void generate_arraycopy_stubs() {
3356 address entry;
3357 address entry_jbyte_arraycopy;
3358 address entry_jshort_arraycopy;
3359 address entry_jint_arraycopy;
3360 address entry_oop_arraycopy;
3361 address entry_jlong_arraycopy;
3362 address entry_checkcast_arraycopy;
3363
3364 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry,
3365 "jbyte_disjoint_arraycopy");
3366 StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
3367 "jbyte_arraycopy");
3368
3369 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
3370 "jshort_disjoint_arraycopy");
3371 StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
3372 "jshort_arraycopy");
3373
3374 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, &entry,
3375 "jint_disjoint_arraycopy");
3376 StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, entry,
3377 &entry_jint_arraycopy, "jint_arraycopy");
3378
3379 StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, &entry,
3380 "jlong_disjoint_arraycopy");
3381 StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, entry,
3382 &entry_jlong_arraycopy, "jlong_arraycopy");
3383
3384
3385 if (UseCompressedOops) {
3386 StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, &entry,
3387 "oop_disjoint_arraycopy");
3388 StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, entry,
3389 &entry_oop_arraycopy, "oop_arraycopy");
3390 StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(false, true, &entry,
3391 "oop_disjoint_arraycopy_uninit",
3392 /*dest_uninitialized*/true);
3393 StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(false, true, entry,
3394 NULL__null, "oop_arraycopy_uninit",
3395 /*dest_uninitialized*/true);
3396 } else {
3397 StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, &entry,
3398 "oop_disjoint_arraycopy");
3399 StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, entry,
3400 &entry_oop_arraycopy, "oop_arraycopy");
3401 StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(false, true, &entry,
3402 "oop_disjoint_arraycopy_uninit",
3403 /*dest_uninitialized*/true);
3404 StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(false, true, entry,
3405 NULL__null, "oop_arraycopy_uninit",
3406 /*dest_uninitialized*/true);
3407 }
3408
3409 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
3410 StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL__null,
3411 /*dest_uninitialized*/true);
3412
3413 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy",
3414 entry_jbyte_arraycopy,
3415 entry_jshort_arraycopy,
3416 entry_jint_arraycopy,
3417 entry_jlong_arraycopy);
3418 StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy",
3419 entry_jbyte_arraycopy,
3420 entry_jshort_arraycopy,
3421 entry_jint_arraycopy,
3422 entry_oop_arraycopy,
3423 entry_jlong_arraycopy,
3424 entry_checkcast_arraycopy);
3425
3426 StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
3427 StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
3428 StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
3429 StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
3430 StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
3431 StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
3432
3433 // We don't generate specialized code for HeapWord-aligned source
3434 // arrays, so just use the code we've already generated
3435 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = StubRoutines::_jbyte_disjoint_arraycopy;
3436 StubRoutines::_arrayof_jbyte_arraycopy = StubRoutines::_jbyte_arraycopy;
3437
3438 StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
3439 StubRoutines::_arrayof_jshort_arraycopy = StubRoutines::_jshort_arraycopy;
3440
3441 StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy;
3442 StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
3443
3444 StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy;
3445 StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
3446
3447 StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy;
3448 StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
3449
3450 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
3451 StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
3452 }
3453
3454 // AES intrinsic stubs
3455 enum {AESBlockSize = 16};
3456
3457 address generate_key_shuffle_mask() {
3458 __masm-> align(16);
3459 StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
3460 address start = __masm-> pc();
3461 __masm-> emit_data64( 0x0405060700010203, relocInfo::none );
3462 __masm-> emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
3463 return start;
3464 }
3465
3466 address generate_counter_shuffle_mask() {
3467 __masm-> align(16);
3468 StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
3469 address start = __masm-> pc();
3470 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3471 __masm-> emit_data64(0x0001020304050607, relocInfo::none);
3472 return start;
3473 }
3474
3475 // Utility routine for loading a 128-bit key word in little endian format
3476 // can optionally specify that the shuffle mask is already in an xmmregister
3477 void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL__null) {
3478 __masm-> movdqu(xmmdst, Address(key, offset));
3479 if (xmm_shuf_mask != NULL__null) {
3480 __masm-> pshufb(xmmdst, xmm_shuf_mask);
3481 } else {
3482 __masm-> pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3483 }
3484 }
3485
3486 // Utility routine for increase 128bit counter (iv in CTR mode)
3487 void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
3488 __masm-> pextrq(reg, xmmdst, 0x0);
3489 __masm-> addq(reg, inc_delta);
3490 __masm-> pinsrq(xmmdst, reg, 0x0);
3491 __masm-> jcc(Assembler::carryClear, next_block); // jump if no carry
3492 __masm-> pextrq(reg, xmmdst, 0x01); // Carry
3493 __masm-> addq(reg, 0x01);
3494 __masm-> pinsrq(xmmdst, reg, 0x01); //Carry end
3495 __masm-> BIND(next_block)bind(next_block); masm-> block_comment("next_block" ":"); // next instruction
3496 }
3497
3498 // Arguments:
3499 //
3500 // Inputs:
3501 // c_rarg0 - source byte array address
3502 // c_rarg1 - destination byte array address
3503 // c_rarg2 - K (key) in little endian int array
3504 //
3505 address generate_aescrypt_encryptBlock() {
3506 assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3506, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support"
); ::breakpoint(); } } while (0);
3507 __masm-> align(CodeEntryAlignment);
3508 StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
3509 Label L_doLast;
3510 address start = __masm-> pc();
3511
3512 const Register from = c_rarg0; // source array address
3513 const Register to = c_rarg1; // destination array address
3514 const Register key = c_rarg2; // key array address
3515 const Register keylen = rax;
3516
3517 const XMMRegister xmm_result = xmm0;
3518 const XMMRegister xmm_key_shuf_mask = xmm1;
3519 // On win64 xmm6-xmm15 must be preserved so don't use them.
3520 const XMMRegister xmm_temp1 = xmm2;
3521 const XMMRegister xmm_temp2 = xmm3;
3522 const XMMRegister xmm_temp3 = xmm4;
3523 const XMMRegister xmm_temp4 = xmm5;
3524
3525 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
3526
3527 // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3528 __masm-> movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3529
3530 __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3531 __masm-> movdqu(xmm_result, Address(from, 0)); // get 16 bytes of input
3532
3533 // For encryption, the java expanded key ordering is just what we need
3534 // we don't know if the key is aligned, hence not using load-execute form
3535
3536 load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
3537 __masm-> pxor(xmm_result, xmm_temp1);
3538
3539 load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3540 load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3541 load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3542 load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3543
3544 __masm-> aesenc(xmm_result, xmm_temp1);
3545 __masm-> aesenc(xmm_result, xmm_temp2);
3546 __masm-> aesenc(xmm_result, xmm_temp3);
3547 __masm-> aesenc(xmm_result, xmm_temp4);
3548
3549 load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3550 load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3551 load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3552 load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3553
3554 __masm-> aesenc(xmm_result, xmm_temp1);
3555 __masm-> aesenc(xmm_result, xmm_temp2);
3556 __masm-> aesenc(xmm_result, xmm_temp3);
3557 __masm-> aesenc(xmm_result, xmm_temp4);
3558
3559 load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3560 load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3561
3562 __masm-> cmpl(keylen, 44);
3563 __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3563);
3564
3565 __masm-> aesenc(xmm_result, xmm_temp1);
3566 __masm-> aesenc(xmm_result, xmm_temp2);
3567
3568 load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3569 load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3570
3571 __masm-> cmpl(keylen, 52);
3572 __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3572);
3573
3574 __masm-> aesenc(xmm_result, xmm_temp1);
3575 __masm-> aesenc(xmm_result, xmm_temp2);
3576
3577 load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3578 load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3579
3580 __masm-> BIND(L_doLast)bind(L_doLast); masm-> block_comment("L_doLast" ":");
3581 __masm-> aesenc(xmm_result, xmm_temp1);
3582 __masm-> aesenclast(xmm_result, xmm_temp2);
3583 __masm-> movdqu(Address(to, 0), xmm_result); // store the result
3584 __masm-> xorptr(rax, rax); // return 0
3585 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
3586 __masm-> ret(0);
3587
3588 return start;
3589 }
3590
3591
3592 // Arguments:
3593 //
3594 // Inputs:
3595 // c_rarg0 - source byte array address
3596 // c_rarg1 - destination byte array address
3597 // c_rarg2 - K (key) in little endian int array
3598 //
3599 address generate_aescrypt_decryptBlock() {
3600 assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3600, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support"
); ::breakpoint(); } } while (0);
3601 __masm-> align(CodeEntryAlignment);
3602 StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3603 Label L_doLast;
3604 address start = __masm-> pc();
3605
3606 const Register from = c_rarg0; // source array address
3607 const Register to = c_rarg1; // destination array address
3608 const Register key = c_rarg2; // key array address
3609 const Register keylen = rax;
3610
3611 const XMMRegister xmm_result = xmm0;
3612 const XMMRegister xmm_key_shuf_mask = xmm1;
3613 // On win64 xmm6-xmm15 must be preserved so don't use them.
3614 const XMMRegister xmm_temp1 = xmm2;
3615 const XMMRegister xmm_temp2 = xmm3;
3616 const XMMRegister xmm_temp3 = xmm4;
3617 const XMMRegister xmm_temp4 = xmm5;
3618
3619 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
3620
3621 // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3622 __masm-> movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3623
3624 __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3625 __masm-> movdqu(xmm_result, Address(from, 0));
3626
3627 // for decryption java expanded key ordering is rotated one position from what we want
3628 // so we start from 0x10 here and hit 0x00 last
3629 // we don't know if the key is aligned, hence not using load-execute form
3630 load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3631 load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3632 load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3633 load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3634
3635 __masm-> pxor (xmm_result, xmm_temp1);
3636 __masm-> aesdec(xmm_result, xmm_temp2);
3637 __masm-> aesdec(xmm_result, xmm_temp3);
3638 __masm-> aesdec(xmm_result, xmm_temp4);
3639
3640 load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3641 load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3642 load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3643 load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3644
3645 __masm-> aesdec(xmm_result, xmm_temp1);
3646 __masm-> aesdec(xmm_result, xmm_temp2);
3647 __masm-> aesdec(xmm_result, xmm_temp3);
3648 __masm-> aesdec(xmm_result, xmm_temp4);
3649
3650 load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3651 load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3652 load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
3653
3654 __masm-> cmpl(keylen, 44);
3655 __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3655);
3656
3657 __masm-> aesdec(xmm_result, xmm_temp1);
3658 __masm-> aesdec(xmm_result, xmm_temp2);
3659
3660 load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3661 load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3662
3663 __masm-> cmpl(keylen, 52);
3664 __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3664);
3665
3666 __masm-> aesdec(xmm_result, xmm_temp1);
3667 __masm-> aesdec(xmm_result, xmm_temp2);
3668
3669 load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3670 load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3671
3672 __masm-> BIND(L_doLast)bind(L_doLast); masm-> block_comment("L_doLast" ":");
3673 __masm-> aesdec(xmm_result, xmm_temp1);
3674 __masm-> aesdec(xmm_result, xmm_temp2);
3675
3676 // for decryption the aesdeclast operation is always on key+0x00
3677 __masm-> aesdeclast(xmm_result, xmm_temp3);
3678 __masm-> movdqu(Address(to, 0), xmm_result); // store the result
3679 __masm-> xorptr(rax, rax); // return 0
3680 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
3681 __masm-> ret(0);
3682
3683 return start;
3684 }
3685
3686
3687 // Arguments:
3688 //
3689 // Inputs:
3690 // c_rarg0 - source byte array address
3691 // c_rarg1 - destination byte array address
3692 // c_rarg2 - K (key) in little endian int array
3693 // c_rarg3 - r vector byte array address
3694 // c_rarg4 - input length
3695 //
3696 // Output:
3697 // rax - input length
3698 //
3699 address generate_cipherBlockChaining_encryptAESCrypt() {
3700 assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3700, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support"
); ::breakpoint(); } } while (0);
3701 __masm-> align(CodeEntryAlignment);
3702 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3703 address start = __masm-> pc();
3704
3705 Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
3706 const Register from = c_rarg0; // source array address
3707 const Register to = c_rarg1; // destination array address
3708 const Register key = c_rarg2; // key array address
3709 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
3710 // and left with the results of the last encryption block
3711#ifndef _WIN64
3712 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
3713#else
3714 const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
3715 const Register len_reg = r11; // pick the volatile windows register
3716#endif
3717 const Register pos = rax;
3718
3719 // xmm register assignments for the loops below
3720 const XMMRegister xmm_result = xmm0;
3721 const XMMRegister xmm_temp = xmm1;
3722 // keys 0-10 preloaded into xmm2-xmm12
3723 const int XMM_REG_NUM_KEY_FIRST = 2;
3724 const int XMM_REG_NUM_KEY_LAST = 15;
3725 const XMMRegister xmm_key0 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3726 const XMMRegister xmm_key10 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
3727 const XMMRegister xmm_key11 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
3728 const XMMRegister xmm_key12 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
3729 const XMMRegister xmm_key13 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
3730
3731 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
3732
3733#ifdef _WIN64
3734 // on win64, fill len_reg from stack position
3735 __masm-> movl(len_reg, len_mem);
3736#else
3737 __masm-> push(len_reg); // Save
3738#endif
3739
3740 const XMMRegister xmm_key_shuf_mask = xmm_temp; // used temporarily to swap key bytes up front
3741 __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3742 // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
3743 for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
3744 load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3745 offset += 0x10;
3746 }
3747 __masm-> movdqu(xmm_result, Address(rvec, 0x00)); // initialize xmm_result with r vec
3748
3749 // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3750 __masm-> movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3751 __masm-> cmpl(rax, 44);
3752 __masm-> jcc(Assembler::notEqual, L_key_192_256);
3753
3754 // 128 bit code follows here
3755 __masm-> movptr(pos, 0);
3756 __masm-> align(OptoLoopAlignment);
3757
3758 __masm-> BIND(L_loopTop_128)bind(L_loopTop_128); masm-> block_comment("L_loopTop_128" ":"
);
3759 __masm-> movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
3760 __masm-> pxor (xmm_result, xmm_temp); // xor with the current r vector
3761 __masm-> pxor (xmm_result, xmm_key0); // do the aes rounds
3762 for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
3763 __masm-> aesenc(xmm_result, as_XMMRegister(rnum));
3764 }
3765 __masm-> aesenclast(xmm_result, xmm_key10);
3766 __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3767 // no need to store r to memory until we exit
3768 __masm-> addptr(pos, AESBlockSize);
3769 __masm-> subptr(len_reg, AESBlockSize);
3770 __masm-> jcc(Assembler::notEqual, L_loopTop_128);
3771
3772 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
3773 __masm-> movdqu(Address(rvec, 0), xmm_result); // final value of r stored in rvec of CipherBlockChaining object
3774
3775#ifdef _WIN64
3776 __masm-> movl(rax, len_mem);
3777#else
3778 __masm-> pop(rax); // return length
3779#endif
3780 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
3781 __masm-> ret(0);
3782
3783 __masm-> BIND(L_key_192_256)bind(L_key_192_256); masm-> block_comment("L_key_192_256" ":"
);
3784 // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3785 load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
3786 load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
3787 __masm-> cmpl(rax, 52);
3788 __masm-> jcc(Assembler::notEqual, L_key_256);
3789
3790 // 192-bit code follows here (could be changed to use more xmm registers)
3791 __masm-> movptr(pos, 0);
3792 __masm-> align(OptoLoopAlignment);
3793
3794 __masm-> BIND(L_loopTop_192)bind(L_loopTop_192); masm-> block_comment("L_loopTop_192" ":"
);
3795 __masm-> movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
3796 __masm-> pxor (xmm_result, xmm_temp); // xor with the current r vector
3797 __masm-> pxor (xmm_result, xmm_key0); // do the aes rounds
3798 for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
3799 __masm-> aesenc(xmm_result, as_XMMRegister(rnum));
3800 }
3801 __masm-> aesenclast(xmm_result, xmm_key12);
3802 __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3803 // no need to store r to memory until we exit
3804 __masm-> addptr(pos, AESBlockSize);
3805 __masm-> subptr(len_reg, AESBlockSize);
3806 __masm-> jcc(Assembler::notEqual, L_loopTop_192);
3807 __masm-> jmp(L_exit);
3808
3809 __masm-> BIND(L_key_256)bind(L_key_256); masm-> block_comment("L_key_256" ":");
3810 // 256-bit code follows here (could be changed to use more xmm registers)
3811 load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
3812 __masm-> movptr(pos, 0);
3813 __masm-> align(OptoLoopAlignment);
3814
3815 __masm-> BIND(L_loopTop_256)bind(L_loopTop_256); masm-> block_comment("L_loopTop_256" ":"
);
3816 __masm-> movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input
3817 __masm-> pxor (xmm_result, xmm_temp); // xor with the current r vector
3818 __masm-> pxor (xmm_result, xmm_key0); // do the aes rounds
3819 for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
3820 __masm-> aesenc(xmm_result, as_XMMRegister(rnum));
3821 }
3822 load_key(xmm_temp, key, 0xe0);
3823 __masm-> aesenclast(xmm_result, xmm_temp);
3824 __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3825 // no need to store r to memory until we exit
3826 __masm-> addptr(pos, AESBlockSize);
3827 __masm-> subptr(len_reg, AESBlockSize);
3828 __masm-> jcc(Assembler::notEqual, L_loopTop_256);
3829 __masm-> jmp(L_exit);
3830
3831 return start;
3832 }
3833
3834 // Safefetch stubs.
3835 void generate_safefetch(const char* name, int size, address* entry,
3836 address* fault_pc, address* continuation_pc) {
3837 // safefetch signatures:
3838 // int SafeFetch32(int* adr, int errValue);
3839 // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3840 //
3841 // arguments:
3842 // c_rarg0 = adr
3843 // c_rarg1 = errValue
3844 //
3845 // result:
3846 // PPC_RET = *adr or errValue
3847
3848 StubCodeMark mark(this, "StubRoutines", name);
3849
3850 // Entry point, pc or function descriptor.
3851 *entry = __masm-> pc();
3852
3853 // Load *adr into c_rarg1, may fault.
3854 *fault_pc = __masm-> pc();
3855 switch (size) {
3856 case 4:
3857 // int32_t
3858 __masm-> movl(c_rarg1, Address(c_rarg0, 0));
3859 break;
3860 case 8:
3861 // int64_t
3862 __masm-> movq(c_rarg1, Address(c_rarg0, 0));
3863 break;
3864 default:
3865 ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here(
"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3865); ::breakpoint(); } while (0);
3866 }
3867
3868 // return errValue or *adr
3869 *continuation_pc = __masm-> pc();
3870 __masm-> movq(rax, c_rarg1);
3871 __masm-> ret(0);
3872 }
3873
3874 // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
3875 // to hide instruction latency
3876 //
3877 // Arguments:
3878 //
3879 // Inputs:
3880 // c_rarg0 - source byte array address
3881 // c_rarg1 - destination byte array address
3882 // c_rarg2 - K (key) in little endian int array
3883 // c_rarg3 - r vector byte array address
3884 // c_rarg4 - input length
3885 //
3886 // Output:
3887 // rax - input length
3888 //
3889 address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3890 assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 3890, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support"
); ::breakpoint(); } } while (0);
3891 __masm-> align(CodeEntryAlignment);
3892 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3893 address start = __masm-> pc();
3894
3895 const Register from = c_rarg0; // source array address
3896 const Register to = c_rarg1; // destination array address
3897 const Register key = c_rarg2; // key array address
3898 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
3899 // and left with the results of the last encryption block
3900#ifndef _WIN64
3901 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
3902#else
3903 const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
3904 const Register len_reg = r11; // pick the volatile windows register
3905#endif
3906 const Register pos = rax;
3907
3908 const int PARALLEL_FACTOR = 4;
3909 const int ROUNDS[3] = { 10, 12, 14 }; // aes rounds for key128, key192, key256
3910
3911 Label L_exit;
3912 Label L_singleBlock_loopTopHead[3]; // 128, 192, 256
3913 Label L_singleBlock_loopTopHead2[3]; // 128, 192, 256
3914 Label L_singleBlock_loopTop[3]; // 128, 192, 256
3915 Label L_multiBlock_loopTopHead[3]; // 128, 192, 256
3916 Label L_multiBlock_loopTop[3]; // 128, 192, 256
3917
3918 // keys 0-10 preloaded into xmm5-xmm15
3919 const int XMM_REG_NUM_KEY_FIRST = 5;
3920 const int XMM_REG_NUM_KEY_LAST = 15;
3921 const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3922 const XMMRegister xmm_key_last = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
3923
3924 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
3925
3926#ifdef _WIN64
3927 // on win64, fill len_reg from stack position
3928 __masm-> movl(len_reg, len_mem);
3929#else
3930 __masm-> push(len_reg); // Save
3931#endif
3932 __masm-> push(rbx);
3933 // the java expanded key ordering is rotated one position from what we want
3934 // so we start from 0x10 here and hit 0x00 last
3935 const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front
3936 __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3937 // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
3938 for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
3939 load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3940 offset += 0x10;
3941 }
3942 load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
3943
3944 const XMMRegister xmm_prev_block_cipher = xmm1; // holds cipher of previous block
3945
3946 // registers holding the four results in the parallelized loop
3947 const XMMRegister xmm_result0 = xmm0;
3948 const XMMRegister xmm_result1 = xmm2;
3949 const XMMRegister xmm_result2 = xmm3;
3950 const XMMRegister xmm_result3 = xmm4;
3951
3952 __masm-> movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
3953
3954 __masm-> xorptr(pos, pos);
3955
3956 // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3957 __masm-> movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3958 __masm-> cmpl(rbx, 52);
3959 __masm-> jcc(Assembler::equal, L_multiBlock_loopTopHead[1]);
3960 __masm-> cmpl(rbx, 60);
3961 __masm-> jcc(Assembler::equal, L_multiBlock_loopTopHead[2]);
3962
3963#define DoFour(opc, src_reg)masm-> opc(xmm_result0, src_reg); masm-> opc(xmm_result1
, src_reg); masm-> opc(xmm_result2, src_reg); masm-> opc
(xmm_result3, src_reg); \
3964 __masm-> opc(xmm_result0, src_reg); \
3965 __masm-> opc(xmm_result1, src_reg); \
3966 __masm-> opc(xmm_result2, src_reg); \
3967 __masm-> opc(xmm_result3, src_reg); \
3968
3969 for (int k = 0; k < 3; ++k) {
3970 __masm-> BIND(L_multiBlock_loopTopHead[k])bind(L_multiBlock_loopTopHead[k]); masm-> block_comment("L_multiBlock_loopTopHead[k]"
":");
3971 if (k != 0) {
3972 __masm-> cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3973 __masm-> jcc(Assembler::less, L_singleBlock_loopTopHead2[k]);
3974 }
3975 if (k == 1) {
3976 __masm-> subptr(rsp, 6 * wordSize);
3977 __masm-> movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3978 load_key(xmm15, key, 0xb0); // 0xb0; 192-bit key goes up to 0xc0
3979 __masm-> movdqu(Address(rsp, 2 * wordSize), xmm15);
3980 load_key(xmm1, key, 0xc0); // 0xc0;
3981 __masm-> movdqu(Address(rsp, 4 * wordSize), xmm1);
3982 } else if (k == 2) {
3983 __masm-> subptr(rsp, 10 * wordSize);
3984 __masm-> movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3985 load_key(xmm15, key, 0xd0); // 0xd0; 256-bit key goes upto 0xe0
3986 __masm-> movdqu(Address(rsp, 6 * wordSize), xmm15);
3987 load_key(xmm1, key, 0xe0); // 0xe0;
3988 __masm-> movdqu(Address(rsp, 8 * wordSize), xmm1);
3989 load_key(xmm15, key, 0xb0); // 0xb0;
3990 __masm-> movdqu(Address(rsp, 2 * wordSize), xmm15);
3991 load_key(xmm1, key, 0xc0); // 0xc0;
3992 __masm-> movdqu(Address(rsp, 4 * wordSize), xmm1);
3993 }
3994 __masm-> align(OptoLoopAlignment);
3995 __masm-> BIND(L_multiBlock_loopTop[k])bind(L_multiBlock_loopTop[k]); masm-> block_comment("L_multiBlock_loopTop[k]"
":");
3996 __masm-> cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3997 __masm-> jcc(Assembler::less, L_singleBlock_loopTopHead[k]);
3998
3999 if (k != 0) {
4000 __masm-> movdqu(xmm15, Address(rsp, 2 * wordSize));
4001 __masm-> movdqu(xmm1, Address(rsp, 4 * wordSize));
4002 }
4003
4004 __masm-> movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
4005 __masm-> movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
4006 __masm-> movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
4007 __masm-> movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
4008
4009 DoFour(pxor, xmm_key_first)masm-> pxor(xmm_result0, xmm_key_first); masm-> pxor(xmm_result1
, xmm_key_first); masm-> pxor(xmm_result2, xmm_key_first);
masm-> pxor(xmm_result3, xmm_key_first);;
4010 if (k == 0) {
4011 for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
4012 DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST))masm-> aesdec(xmm_result0, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result1, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result2, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result3, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
));;
4013 }
4014 DoFour(aesdeclast, xmm_key_last)masm-> aesdeclast(xmm_result0, xmm_key_last); masm-> aesdeclast
(xmm_result1, xmm_key_last); masm-> aesdeclast(xmm_result2
, xmm_key_last); masm-> aesdeclast(xmm_result3, xmm_key_last
);;
4015 } else if (k == 1) {
4016 for (int rnum = 1; rnum <= ROUNDS[k]-2; rnum++) {
4017 DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST))masm-> aesdec(xmm_result0, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result1, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result2, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result3, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
));;
4018 }
4019 __masm-> movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
4020 DoFour(aesdec, xmm1)masm-> aesdec(xmm_result0, xmm1); masm-> aesdec(xmm_result1
, xmm1); masm-> aesdec(xmm_result2, xmm1); masm-> aesdec
(xmm_result3, xmm1);; // key : 0xc0
4021 __masm-> movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // xmm1 needs to be loaded again
4022 DoFour(aesdeclast, xmm_key_last)masm-> aesdeclast(xmm_result0, xmm_key_last); masm-> aesdeclast
(xmm_result1, xmm_key_last); masm-> aesdeclast(xmm_result2
, xmm_key_last); masm-> aesdeclast(xmm_result3, xmm_key_last
);;
4023 } else if (k == 2) {
4024 for (int rnum = 1; rnum <= ROUNDS[k] - 4; rnum++) {
4025 DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST))masm-> aesdec(xmm_result0, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result1, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result2, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
)); masm-> aesdec(xmm_result3, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST
));;
4026 }
4027 DoFour(aesdec, xmm1)masm-> aesdec(xmm_result0, xmm1); masm-> aesdec(xmm_result1
, xmm1); masm-> aesdec(xmm_result2, xmm1); masm-> aesdec
(xmm_result3, xmm1);; // key : 0xc0
4028 __masm-> movdqu(xmm15, Address(rsp, 6 * wordSize));
4029 __masm-> movdqu(xmm1, Address(rsp, 8 * wordSize));
4030 DoFour(aesdec, xmm15)masm-> aesdec(xmm_result0, xmm15); masm-> aesdec(xmm_result1
, xmm15); masm-> aesdec(xmm_result2, xmm15); masm-> aesdec
(xmm_result3, xmm15);; // key : 0xd0
4031 __masm-> movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
4032 DoFour(aesdec, xmm1)masm-> aesdec(xmm_result0, xmm1); masm-> aesdec(xmm_result1
, xmm1); masm-> aesdec(xmm_result2, xmm1); masm-> aesdec
(xmm_result3, xmm1);; // key : 0xe0
4033 __masm-> movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // xmm1 needs to be loaded again
4034 DoFour(aesdeclast, xmm_key_last)masm-> aesdeclast(xmm_result0, xmm_key_last); masm-> aesdeclast
(xmm_result1, xmm_key_last); masm-> aesdeclast(xmm_result2
, xmm_key_last); masm-> aesdeclast(xmm_result3, xmm_key_last
);;
4035 }
4036
4037 // for each result, xor with the r vector of previous cipher block
4038 __masm-> pxor(xmm_result0, xmm_prev_block_cipher);
4039 __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4040 __masm-> pxor(xmm_result1, xmm_prev_block_cipher);
4041 __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
4042 __masm-> pxor(xmm_result2, xmm_prev_block_cipher);
4043 __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
4044 __masm-> pxor(xmm_result3, xmm_prev_block_cipher);
4045 __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
4046 if (k != 0) {
4047 __masm-> movdqu(Address(rvec, 0x00), xmm_prev_block_cipher);
4048 }
4049
4050 __masm-> movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0); // store 4 results into the next 64 bytes of output
4051 __masm-> movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
4052 __masm-> movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
4053 __masm-> movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
4054
4055 __masm-> addptr(pos, PARALLEL_FACTOR * AESBlockSize);
4056 __masm-> subptr(len_reg, PARALLEL_FACTOR * AESBlockSize);
4057 __masm-> jmp(L_multiBlock_loopTop[k]);
4058
4059 // registers used in the non-parallelized loops
4060 // xmm register assignments for the loops below
4061 const XMMRegister xmm_result = xmm0;
4062 const XMMRegister xmm_prev_block_cipher_save = xmm2;
4063 const XMMRegister xmm_key11 = xmm3;
4064 const XMMRegister xmm_key12 = xmm4;
4065 const XMMRegister key_tmp = xmm4;
4066
4067 __masm-> BIND(L_singleBlock_loopTopHead[k])bind(L_singleBlock_loopTopHead[k]); masm-> block_comment("L_singleBlock_loopTopHead[k]"
":");
4068 if (k == 1) {
4069 __masm-> addptr(rsp, 6 * wordSize);
4070 } else if (k == 2) {
4071 __masm-> addptr(rsp, 10 * wordSize);
4072 }
4073 __masm-> cmpptr(len_reg, 0); // any blocks left??
4074 __masm-> jcc(Assembler::equal, L_exit);
4075 __masm-> BIND(L_singleBlock_loopTopHead2[k])bind(L_singleBlock_loopTopHead2[k]); masm-> block_comment(
"L_singleBlock_loopTopHead2[k]" ":");
4076 if (k == 1) {
4077 load_key(xmm_key11, key, 0xb0); // 0xb0; 192-bit key goes upto 0xc0
4078 load_key(xmm_key12, key, 0xc0); // 0xc0; 192-bit key goes upto 0xc0
4079 }
4080 if (k == 2) {
4081 load_key(xmm_key11, key, 0xb0); // 0xb0; 256-bit key goes upto 0xe0
4082 }
4083 __masm-> align(OptoLoopAlignment);
4084 __masm-> BIND(L_singleBlock_loopTop[k])bind(L_singleBlock_loopTop[k]); masm-> block_comment("L_singleBlock_loopTop[k]"
":");
4085 __masm-> movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
4086 __masm-> movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
4087 __masm-> pxor(xmm_result, xmm_key_first); // do the aes dec rounds
4088 for (int rnum = 1; rnum <= 9 ; rnum++) {
4089 __masm-> aesdec(xmm_result, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
4090 }
4091 if (k == 1) {
4092 __masm-> aesdec(xmm_result, xmm_key11);
4093 __masm-> aesdec(xmm_result, xmm_key12);
4094 }
4095 if (k == 2) {
4096 __masm-> aesdec(xmm_result, xmm_key11);
4097 load_key(key_tmp, key, 0xc0);
4098 __masm-> aesdec(xmm_result, key_tmp);
4099 load_key(key_tmp, key, 0xd0);
4100 __masm-> aesdec(xmm_result, key_tmp);
4101 load_key(key_tmp, key, 0xe0);
4102 __masm-> aesdec(xmm_result, key_tmp);
4103 }
4104
4105 __masm-> aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0
4106 __masm-> pxor(xmm_result, xmm_prev_block_cipher); // xor with the current r vector
4107 __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
4108 // no need to store r to memory until we exit
4109 __masm-> movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
4110 __masm-> addptr(pos, AESBlockSize);
4111 __masm-> subptr(len_reg, AESBlockSize);
4112 __masm-> jcc(Assembler::notEqual, L_singleBlock_loopTop[k]);
4113 if (k != 2) {
4114 __masm-> jmp(L_exit);
4115 }
4116 } //for 128/192/256
4117
4118 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
4119 __masm-> movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
4120 __masm-> pop(rbx);
4121#ifdef _WIN64
4122 __masm-> movl(rax, len_mem);
4123#else
4124 __masm-> pop(rax); // return length
4125#endif
4126 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
4127 __masm-> ret(0);
4128 return start;
4129}
4130
4131 address generate_electronicCodeBook_encryptAESCrypt() {
4132 __masm-> align(CodeEntryAlignment);
4133 StubCodeMark mark(this, "StubRoutines", "electronicCodeBook_encryptAESCrypt");
4134 address start = __masm-> pc();
4135 const Register from = c_rarg0; // source array address
4136 const Register to = c_rarg1; // destination array address
4137 const Register key = c_rarg2; // key array address
4138 const Register len = c_rarg3; // src len (must be multiple of blocksize 16)
4139 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
4140 __masm-> aesecb_encrypt(from, to, key, len);
4141 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
4142 __masm-> ret(0);
4143 return start;
4144 }
4145
4146 address generate_electronicCodeBook_decryptAESCrypt() {
4147 __masm-> align(CodeEntryAlignment);
4148 StubCodeMark mark(this, "StubRoutines", "electronicCodeBook_decryptAESCrypt");
4149 address start = __masm-> pc();
4150 const Register from = c_rarg0; // source array address
4151 const Register to = c_rarg1; // destination array address
4152 const Register key = c_rarg2; // key array address
4153 const Register len = c_rarg3; // src len (must be multiple of blocksize 16)
4154 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
4155 __masm-> aesecb_decrypt(from, to, key, len);
4156 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
4157 __masm-> ret(0);
4158 return start;
4159 }
4160
4161 // ofs and limit are use for multi-block byte array.
4162 // int com.sun.security.provider.MD5.implCompress(byte[] b, int ofs)
4163 address generate_md5_implCompress(bool multi_block, const char *name) {
4164 __masm-> align(CodeEntryAlignment);
4165 StubCodeMark mark(this, "StubRoutines", name);
4166 address start = __masm-> pc();
4167
4168 const Register buf_param = r15;
4169 const Address state_param(rsp, 0 * wordSize);
4170 const Address ofs_param (rsp, 1 * wordSize );
4171 const Address limit_param(rsp, 1 * wordSize + 4);
4172
4173 __masm-> enter();
4174 __masm-> push(rbx);
4175 __masm-> push(rdi);
4176 __masm-> push(rsi);
4177 __masm-> push(r15);
4178 __masm-> subptr(rsp, 2 * wordSize);
4179
4180 __masm-> movptr(buf_param, c_rarg0);
4181 __masm-> movptr(state_param, c_rarg1);
4182 if (multi_block) {
4183 __masm-> movl(ofs_param, c_rarg2);
4184 __masm-> movl(limit_param, c_rarg3);
4185 }
4186 __masm-> fast_md5(buf_param, state_param, ofs_param, limit_param, multi_block);
4187
4188 __masm-> addptr(rsp, 2 * wordSize);
4189 __masm-> pop(r15);
4190 __masm-> pop(rsi);
4191 __masm-> pop(rdi);
4192 __masm-> pop(rbx);
4193 __masm-> leave();
4194 __masm-> ret(0);
4195 return start;
4196 }
4197
4198 address generate_upper_word_mask() {
4199 __masm-> align64();
4200 StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
4201 address start = __masm-> pc();
4202 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4203 __masm-> emit_data64(0xFFFFFFFF00000000, relocInfo::none);
4204 return start;
4205 }
4206
4207 address generate_shuffle_byte_flip_mask() {
4208 __masm-> align64();
4209 StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
4210 address start = __masm-> pc();
4211 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
4212 __masm-> emit_data64(0x0001020304050607, relocInfo::none);
4213 return start;
4214 }
4215
4216 // ofs and limit are use for multi-block byte array.
4217 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
4218 address generate_sha1_implCompress(bool multi_block, const char *name) {
4219 __masm-> align(CodeEntryAlignment);
4220 StubCodeMark mark(this, "StubRoutines", name);
4221 address start = __masm-> pc();
4222
4223 Register buf = c_rarg0;
4224 Register state = c_rarg1;
4225 Register ofs = c_rarg2;
4226 Register limit = c_rarg3;
4227
4228 const XMMRegister abcd = xmm0;
4229 const XMMRegister e0 = xmm1;
4230 const XMMRegister e1 = xmm2;
4231 const XMMRegister msg0 = xmm3;
4232
4233 const XMMRegister msg1 = xmm4;
4234 const XMMRegister msg2 = xmm5;
4235 const XMMRegister msg3 = xmm6;
4236 const XMMRegister shuf_mask = xmm7;
4237
4238 __masm-> enter();
4239
4240 __masm-> subptr(rsp, 4 * wordSize);
4241
4242 __masm-> fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
4243 buf, state, ofs, limit, rsp, multi_block);
4244
4245 __masm-> addptr(rsp, 4 * wordSize);
4246
4247 __masm-> leave();
4248 __masm-> ret(0);
4249 return start;
4250 }
4251
4252 address generate_pshuffle_byte_flip_mask() {
4253 __masm-> align64();
4254 StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
4255 address start = __masm-> pc();
4256 __masm-> emit_data64(0x0405060700010203, relocInfo::none);
4257 __masm-> emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
4258
4259 if (VM_Version::supports_avx2()) {
4260 __masm-> emit_data64(0x0405060700010203, relocInfo::none); // second copy
4261 __masm-> emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
4262 // _SHUF_00BA
4263 __masm-> emit_data64(0x0b0a090803020100, relocInfo::none);
4264 __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
4265 __masm-> emit_data64(0x0b0a090803020100, relocInfo::none);
4266 __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
4267 // _SHUF_DC00
4268 __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
4269 __masm-> emit_data64(0x0b0a090803020100, relocInfo::none);
4270 __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
4271 __masm-> emit_data64(0x0b0a090803020100, relocInfo::none);
4272 }
4273
4274 return start;
4275 }
4276
4277 //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
4278 address generate_pshuffle_byte_flip_mask_sha512() {
4279 __masm-> align32();
4280 StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512");
4281 address start = __masm-> pc();
4282 if (VM_Version::supports_avx2()) {
4283 __masm-> emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK
4284 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
4285 __masm-> emit_data64(0x1011121314151617, relocInfo::none);
4286 __masm-> emit_data64(0x18191a1b1c1d1e1f, relocInfo::none);
4287 __masm-> emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO
4288 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4289 __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
4290 __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
4291 }
4292
4293 return start;
4294 }
4295
4296// ofs and limit are use for multi-block byte array.
4297// int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
4298 address generate_sha256_implCompress(bool multi_block, const char *name) {
4299 assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "")do { if (!(VM_Version::supports_sha() || VM_Version::supports_avx2
())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 4299, "assert(" "VM_Version::supports_sha() || VM_Version::supports_avx2()"
") failed", ""); ::breakpoint(); } } while (0);
4300 __masm-> align(CodeEntryAlignment);
4301 StubCodeMark mark(this, "StubRoutines", name);
4302 address start = __masm-> pc();
4303
4304 Register buf = c_rarg0;
4305 Register state = c_rarg1;
4306 Register ofs = c_rarg2;
4307 Register limit = c_rarg3;
4308
4309 const XMMRegister msg = xmm0;
4310 const XMMRegister state0 = xmm1;
4311 const XMMRegister state1 = xmm2;
4312 const XMMRegister msgtmp0 = xmm3;
4313
4314 const XMMRegister msgtmp1 = xmm4;
4315 const XMMRegister msgtmp2 = xmm5;
4316 const XMMRegister msgtmp3 = xmm6;
4317 const XMMRegister msgtmp4 = xmm7;
4318
4319 const XMMRegister shuf_mask = xmm8;
4320
4321 __masm-> enter();
4322
4323 __masm-> subptr(rsp, 4 * wordSize);
4324
4325 if (VM_Version::supports_sha()) {
4326 __masm-> fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
4327 buf, state, ofs, limit, rsp, multi_block, shuf_mask);
4328 } else if (VM_Version::supports_avx2()) {
4329 __masm-> sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
4330 buf, state, ofs, limit, rsp, multi_block, shuf_mask);
4331 }
4332 __masm-> addptr(rsp, 4 * wordSize);
4333 __masm-> vzeroupper();
4334 __masm-> leave();
4335 __masm-> ret(0);
4336 return start;
4337 }
4338
4339 address generate_sha512_implCompress(bool multi_block, const char *name) {
4340 assert(VM_Version::supports_avx2(), "")do { if (!(VM_Version::supports_avx2())) { (*g_assert_poison)
= 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 4340, "assert(" "VM_Version::supports_avx2()" ") failed", ""
); ::breakpoint(); } } while (0);
4341 assert(VM_Version::supports_bmi2(), "")do { if (!(VM_Version::supports_bmi2())) { (*g_assert_poison)
= 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 4341, "assert(" "VM_Version::supports_bmi2()" ") failed", ""
); ::breakpoint(); } } while (0);
4342 __masm-> align(CodeEntryAlignment);
4343 StubCodeMark mark(this, "StubRoutines", name);
4344 address start = __masm-> pc();
4345
4346 Register buf = c_rarg0;
4347 Register state = c_rarg1;
4348 Register ofs = c_rarg2;
4349 Register limit = c_rarg3;
4350
4351 const XMMRegister msg = xmm0;
4352 const XMMRegister state0 = xmm1;
4353 const XMMRegister state1 = xmm2;
4354 const XMMRegister msgtmp0 = xmm3;
4355 const XMMRegister msgtmp1 = xmm4;
4356 const XMMRegister msgtmp2 = xmm5;
4357 const XMMRegister msgtmp3 = xmm6;
4358 const XMMRegister msgtmp4 = xmm7;
4359
4360 const XMMRegister shuf_mask = xmm8;
4361
4362 __masm-> enter();
4363
4364 __masm-> sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
4365 buf, state, ofs, limit, rsp, multi_block, shuf_mask);
4366
4367 __masm-> vzeroupper();
4368 __masm-> leave();
4369 __masm-> ret(0);
4370 return start;
4371 }
4372
4373 address ghash_polynomial512_addr() {
4374 __masm-> align(CodeEntryAlignment);
4375 StubCodeMark mark(this, "StubRoutines", "_ghash_poly512_addr");
4376 address start = __masm-> pc();
4377 __masm-> emit_data64(0x00000001C2000000, relocInfo::none); // POLY for reduction
4378 __masm-> emit_data64(0xC200000000000000, relocInfo::none);
4379 __masm-> emit_data64(0x00000001C2000000, relocInfo::none);
4380 __masm-> emit_data64(0xC200000000000000, relocInfo::none);
4381 __masm-> emit_data64(0x00000001C2000000, relocInfo::none);
4382 __masm-> emit_data64(0xC200000000000000, relocInfo::none);
4383 __masm-> emit_data64(0x00000001C2000000, relocInfo::none);
4384 __masm-> emit_data64(0xC200000000000000, relocInfo::none);
4385 __masm-> emit_data64(0x0000000000000001, relocInfo::none); // POLY
4386 __masm-> emit_data64(0xC200000000000000, relocInfo::none);
4387 __masm-> emit_data64(0x0000000000000001, relocInfo::none); // TWOONE
4388 __masm-> emit_data64(0x0000000100000000, relocInfo::none);
4389 return start;
4390}
4391
4392 // Vector AES Galois Counter Mode implementation. Parameters:
4393 // Windows regs | Linux regs
4394 // in = c_rarg0 (rcx) | c_rarg0 (rsi)
4395 // len = c_rarg1 (rdx) | c_rarg1 (rdi)
4396 // ct = c_rarg2 (r8) | c_rarg2 (rdx)
4397 // out = c_rarg3 (r9) | c_rarg3 (rcx)
4398 // key = r10 | c_rarg4 (r8)
4399 // state = r13 | c_rarg5 (r9)
4400 // subkeyHtbl = r14 | r11
4401 // counter = rsi | r12
4402 // return - number of processed bytes
4403 address generate_galoisCounterMode_AESCrypt() {
4404 __masm-> align(CodeEntryAlignment);
4405 StubCodeMark mark(this, "StubRoutines", "galoisCounterMode_AESCrypt");
4406 address start = __masm-> pc();
4407 const Register in = c_rarg0;
4408 const Register len = c_rarg1;
4409 const Register ct = c_rarg2;
4410 const Register out = c_rarg3;
4411 // and updated with the incremented counter in the end
4412#ifndef _WIN64
4413 const Register key = c_rarg4;
4414 const Register state = c_rarg5;
4415 const Address subkeyH_mem(rbp, 2 * wordSize);
4416 const Register subkeyHtbl = r11;
4417 const Address avx512_subkeyH_mem(rbp, 3 * wordSize);
4418 const Register avx512_subkeyHtbl = r13;
4419 const Address counter_mem(rbp, 4 * wordSize);
4420 const Register counter = r12;
4421#else
4422 const Address key_mem(rbp, 6 * wordSize);
4423 const Register key = r10;
4424 const Address state_mem(rbp, 7 * wordSize);
4425 const Register state = r13;
4426 const Address subkeyH_mem(rbp, 8 * wordSize);
4427 const Register subkeyHtbl = r14;
4428 const Address avx512_subkeyH_mem(rbp, 9 * wordSize);
4429 const Register avx512_subkeyHtbl = r12;
4430 const Address counter_mem(rbp, 10 * wordSize);
4431 const Register counter = rsi;
4432#endif
4433 __masm-> enter();
4434 // Save state before entering routine
4435 __masm-> push(r12);
4436 __masm-> push(r13);
4437 __masm-> push(r14);
4438 __masm-> push(r15);
4439 __masm-> push(rbx);
4440#ifdef _WIN64
4441 // on win64, fill len_reg from stack position
4442 __masm-> push(rsi);
4443 __masm-> movptr(key, key_mem);
4444 __masm-> movptr(state, state_mem);
4445#endif
4446 __masm-> movptr(subkeyHtbl, subkeyH_mem);
4447 __masm-> movptr(avx512_subkeyHtbl, avx512_subkeyH_mem);
4448 __masm-> movptr(counter, counter_mem);
4449
4450 __masm-> aesgcm_encrypt(in, len, ct, out, key, state, subkeyHtbl, avx512_subkeyHtbl, counter);
4451
4452 // Restore state before leaving routine
4453#ifdef _WIN64
4454 __masm-> pop(rsi);
4455#endif
4456 __masm-> pop(rbx);
4457 __masm-> pop(r15);
4458 __masm-> pop(r14);
4459 __masm-> pop(r13);
4460 __masm-> pop(r12);
4461
4462 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
4463 __masm-> ret(0);
4464 return start;
4465 }
4466
4467 // This mask is used for incrementing counter value(linc0, linc4, etc.)
4468 address counter_mask_addr() {
4469 __masm-> align64();
4470 StubCodeMark mark(this, "StubRoutines", "counter_mask_addr");
4471 address start = __masm-> pc();
4472 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);//lbswapmask
4473 __masm-> emit_data64(0x0001020304050607, relocInfo::none);
4474 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
4475 __masm-> emit_data64(0x0001020304050607, relocInfo::none);
4476 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
4477 __masm-> emit_data64(0x0001020304050607, relocInfo::none);
4478 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
4479 __masm-> emit_data64(0x0001020304050607, relocInfo::none);
4480 __masm-> emit_data64(0x0000000000000000, relocInfo::none);//linc0 = counter_mask_addr+64
4481 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4482 __masm-> emit_data64(0x0000000000000001, relocInfo::none);//counter_mask_addr() + 80
4483 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4484 __masm-> emit_data64(0x0000000000000002, relocInfo::none);
4485 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4486 __masm-> emit_data64(0x0000000000000003, relocInfo::none);
4487 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4488 __masm-> emit_data64(0x0000000000000004, relocInfo::none);//linc4 = counter_mask_addr() + 128
4489 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4490 __masm-> emit_data64(0x0000000000000004, relocInfo::none);
4491 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4492 __masm-> emit_data64(0x0000000000000004, relocInfo::none);
4493 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4494 __masm-> emit_data64(0x0000000000000004, relocInfo::none);
4495 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4496 __masm-> emit_data64(0x0000000000000008, relocInfo::none);//linc8 = counter_mask_addr() + 192
4497 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4498 __masm-> emit_data64(0x0000000000000008, relocInfo::none);
4499 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4500 __masm-> emit_data64(0x0000000000000008, relocInfo::none);
4501 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4502 __masm-> emit_data64(0x0000000000000008, relocInfo::none);
4503 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4504 __masm-> emit_data64(0x0000000000000020, relocInfo::none);//linc32 = counter_mask_addr() + 256
4505 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4506 __masm-> emit_data64(0x0000000000000020, relocInfo::none);
4507 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4508 __masm-> emit_data64(0x0000000000000020, relocInfo::none);
4509 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4510 __masm-> emit_data64(0x0000000000000020, relocInfo::none);
4511 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4512 __masm-> emit_data64(0x0000000000000010, relocInfo::none);//linc16 = counter_mask_addr() + 320
4513 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4514 __masm-> emit_data64(0x0000000000000010, relocInfo::none);
4515 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4516 __masm-> emit_data64(0x0000000000000010, relocInfo::none);
4517 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4518 __masm-> emit_data64(0x0000000000000010, relocInfo::none);
4519 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
4520 return start;
4521 }
4522
4523 // Vector AES Counter implementation
4524 address generate_counterMode_VectorAESCrypt() {
4525 __masm-> align(CodeEntryAlignment);
4526 StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
4527 address start = __masm-> pc();
4528 const Register from = c_rarg0; // source array address
4529 const Register to = c_rarg1; // destination array address
4530 const Register key = c_rarg2; // key array address r8
4531 const Register counter = c_rarg3; // counter byte array initialized from counter array address
4532 // and updated with the incremented counter in the end
4533#ifndef _WIN64
4534 const Register len_reg = c_rarg4;
4535 const Register saved_encCounter_start = c_rarg5;
4536 const Register used_addr = r10;
4537 const Address used_mem(rbp, 2 * wordSize);
4538 const Register used = r11;
4539#else
4540 const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
4541 const Address saved_encCounter_mem(rbp, 7 * wordSize); // saved encrypted counter is on stack on Win64
4542 const Address used_mem(rbp, 8 * wordSize); // used length is on stack on Win64
4543 const Register len_reg = r10; // pick the first volatile windows register
4544 const Register saved_encCounter_start = r11;
4545 const Register used_addr = r13;
4546 const Register used = r14;
4547#endif
4548 __masm-> enter();
4549 // Save state before entering routine
4550 __masm-> push(r12);
4551 __masm-> push(r13);
4552 __masm-> push(r14);
4553 __masm-> push(r15);
4554#ifdef _WIN64
4555 // on win64, fill len_reg from stack position
4556 __masm-> movl(len_reg, len_mem);
4557 __masm-> movptr(saved_encCounter_start, saved_encCounter_mem);
4558 __masm-> movptr(used_addr, used_mem);
4559 __masm-> movl(used, Address(used_addr, 0));
4560#else
4561 __masm-> push(len_reg); // Save
4562 __masm-> movptr(used_addr, used_mem);
4563 __masm-> movl(used, Address(used_addr, 0));
4564#endif
4565 __masm-> push(rbx);
4566 __masm-> aesctr_encrypt(from, to, key, counter, len_reg, used, used_addr, saved_encCounter_start);
4567 // Restore state before leaving routine
4568 __masm-> pop(rbx);
4569#ifdef _WIN64
4570 __masm-> movl(rax, len_mem); // return length
4571#else
4572 __masm-> pop(rax); // return length
4573#endif
4574 __masm-> pop(r15);
4575 __masm-> pop(r14);
4576 __masm-> pop(r13);
4577 __masm-> pop(r12);
4578
4579 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
4580 __masm-> ret(0);
4581 return start;
4582 }
4583
4584 // This is a version of CTR/AES crypt which does 6 blocks in a loop at a time
4585 // to hide instruction latency
4586 //
4587 // Arguments:
4588 //
4589 // Inputs:
4590 // c_rarg0 - source byte array address
4591 // c_rarg1 - destination byte array address
4592 // c_rarg2 - K (key) in little endian int array
4593 // c_rarg3 - counter vector byte array address
4594 // Linux
4595 // c_rarg4 - input length
4596 // c_rarg5 - saved encryptedCounter start
4597 // rbp + 6 * wordSize - saved used length
4598 // Windows
4599 // rbp + 6 * wordSize - input length
4600 // rbp + 7 * wordSize - saved encryptedCounter start
4601 // rbp + 8 * wordSize - saved used length
4602 //
4603 // Output:
4604 // rax - input length
4605 //
4606 address generate_counterMode_AESCrypt_Parallel() {
4607 assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 4607, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support"
); ::breakpoint(); } } while (0);
4608 __masm-> align(CodeEntryAlignment);
4609 StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
4610 address start = __masm-> pc();
4611 const Register from = c_rarg0; // source array address
4612 const Register to = c_rarg1; // destination array address
4613 const Register key = c_rarg2; // key array address
4614 const Register counter = c_rarg3; // counter byte array initialized from counter array address
4615 // and updated with the incremented counter in the end
4616#ifndef _WIN64
4617 const Register len_reg = c_rarg4;
4618 const Register saved_encCounter_start = c_rarg5;
4619 const Register used_addr = r10;
4620 const Address used_mem(rbp, 2 * wordSize);
4621 const Register used = r11;
4622#else
4623 const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
4624 const Address saved_encCounter_mem(rbp, 7 * wordSize); // length is on stack on Win64
4625 const Address used_mem(rbp, 8 * wordSize); // length is on stack on Win64
4626 const Register len_reg = r10; // pick the first volatile windows register
4627 const Register saved_encCounter_start = r11;
4628 const Register used_addr = r13;
4629 const Register used = r14;
4630#endif
4631 const Register pos = rax;
4632
4633 const int PARALLEL_FACTOR = 6;
4634 const XMMRegister xmm_counter_shuf_mask = xmm0;
4635 const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front
4636 const XMMRegister xmm_curr_counter = xmm2;
4637
4638 const XMMRegister xmm_key_tmp0 = xmm3;
4639 const XMMRegister xmm_key_tmp1 = xmm4;
4640
4641 // registers holding the four results in the parallelized loop
4642 const XMMRegister xmm_result0 = xmm5;
4643 const XMMRegister xmm_result1 = xmm6;
4644 const XMMRegister xmm_result2 = xmm7;
4645 const XMMRegister xmm_result3 = xmm8;
4646 const XMMRegister xmm_result4 = xmm9;
4647 const XMMRegister xmm_result5 = xmm10;
4648
4649 const XMMRegister xmm_from0 = xmm11;
4650 const XMMRegister xmm_from1 = xmm12;
4651 const XMMRegister xmm_from2 = xmm13;
4652 const XMMRegister xmm_from3 = xmm14; //the last one is xmm14. we have to preserve it on WIN64.
4653 const XMMRegister xmm_from4 = xmm3; //reuse xmm3~4. Because xmm_key_tmp0~1 are useless when loading input text
4654 const XMMRegister xmm_from5 = xmm4;
4655
4656 //for key_128, key_192, key_256
4657 const int rounds[3] = {10, 12, 14};
4658 Label L_exit_preLoop, L_preLoop_start;
4659 Label L_multiBlock_loopTop[3];
4660 Label L_singleBlockLoopTop[3];
4661 Label L__incCounter[3][6]; //for 6 blocks
4662 Label L__incCounter_single[3]; //for single block, key128, key192, key256
4663 Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
4664 Label L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
4665
4666 Label L_exit;
4667
4668 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
4669
4670#ifdef _WIN64
4671 // allocate spill slots for r13, r14
4672 enum {
4673 saved_r13_offset,
4674 saved_r14_offset
4675 };
4676 __masm-> subptr(rsp, 2 * wordSize);
4677 __masm-> movptr(Address(rsp, saved_r13_offset * wordSize), r13);
4678 __masm-> movptr(Address(rsp, saved_r14_offset * wordSize), r14);
4679
4680 // on win64, fill len_reg from stack position
4681 __masm-> movl(len_reg, len_mem);
4682 __masm-> movptr(saved_encCounter_start, saved_encCounter_mem);
4683 __masm-> movptr(used_addr, used_mem);
4684 __masm-> movl(used, Address(used_addr, 0));
4685#else
4686 __masm-> push(len_reg); // Save
4687 __masm-> movptr(used_addr, used_mem);
4688 __masm-> movl(used, Address(used_addr, 0));
4689#endif
4690
4691 __masm-> push(rbx); // Save RBX
4692 __masm-> movdqu(xmm_curr_counter, Address(counter, 0x00)); // initialize counter with initial counter
4693 __masm-> movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()), pos); // pos as scratch
4694 __masm-> pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled
4695 __masm-> movptr(pos, 0);
4696
4697 // Use the partially used encrpyted counter from last invocation
4698 __masm-> BIND(L_preLoop_start)bind(L_preLoop_start); masm-> block_comment("L_preLoop_start"
":");
4699 __masm-> cmpptr(used, 16);
4700 __masm-> jcc(Assembler::aboveEqual, L_exit_preLoop);
4701 __masm-> cmpptr(len_reg, 0);
4702 __masm-> jcc(Assembler::lessEqual, L_exit_preLoop);
4703 __masm-> movb(rbx, Address(saved_encCounter_start, used));
4704 __masm-> xorb(rbx, Address(from, pos));
4705 __masm-> movb(Address(to, pos), rbx);
4706 __masm-> addptr(pos, 1);
4707 __masm-> addptr(used, 1);
4708 __masm-> subptr(len_reg, 1);
4709
4710 __masm-> jmp(L_preLoop_start);
4711
4712 __masm-> BIND(L_exit_preLoop)bind(L_exit_preLoop); masm-> block_comment("L_exit_preLoop"
":");
4713 __masm-> movl(Address(used_addr, 0), used);
4714
4715 // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
4716 __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()), rbx); // rbx as scratch
4717 __masm-> movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
4718 __masm-> cmpl(rbx, 52);
4719 __masm-> jcc(Assembler::equal, L_multiBlock_loopTop[1]);
4720 __masm-> cmpl(rbx, 60);
4721 __masm-> jcc(Assembler::equal, L_multiBlock_loopTop[2]);
4722
4723#define CTR_DoSix(opc, src_reg)masm-> opc(xmm_result0, src_reg); masm-> opc(xmm_result1
, src_reg); masm-> opc(xmm_result2, src_reg); masm-> opc
(xmm_result3, src_reg); masm-> opc(xmm_result4, src_reg); masm
-> opc(xmm_result5, src_reg); \
4724 __masm-> opc(xmm_result0, src_reg); \
4725 __masm-> opc(xmm_result1, src_reg); \
4726 __masm-> opc(xmm_result2, src_reg); \
4727 __masm-> opc(xmm_result3, src_reg); \
4728 __masm-> opc(xmm_result4, src_reg); \
4729 __masm-> opc(xmm_result5, src_reg);
4730
4731 // k == 0 : generate code for key_128
4732 // k == 1 : generate code for key_192
4733 // k == 2 : generate code for key_256
4734 for (int k = 0; k < 3; ++k) {
4735 //multi blocks starts here
4736 __masm-> align(OptoLoopAlignment);
4737 __masm-> BIND(L_multiBlock_loopTop[k])bind(L_multiBlock_loopTop[k]); masm-> block_comment("L_multiBlock_loopTop[k]"
":");
4738 __masm-> cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
4739 __masm-> jcc(Assembler::less, L_singleBlockLoopTop[k]);
4740 load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
4741
4742 //load, then increase counters
4743 CTR_DoSix(movdqa, xmm_curr_counter)masm-> movdqa(xmm_result0, xmm_curr_counter); masm-> movdqa
(xmm_result1, xmm_curr_counter); masm-> movdqa(xmm_result2
, xmm_curr_counter); masm-> movdqa(xmm_result3, xmm_curr_counter
); masm-> movdqa(xmm_result4, xmm_curr_counter); masm->
movdqa(xmm_result5, xmm_curr_counter);;
4744 inc_counter(rbx, xmm_result1, 0x01, L__incCounter[k][0]);
4745 inc_counter(rbx, xmm_result2, 0x02, L__incCounter[k][1]);
4746 inc_counter(rbx, xmm_result3, 0x03, L__incCounter[k][2]);
4747 inc_counter(rbx, xmm_result4, 0x04, L__incCounter[k][3]);
4748 inc_counter(rbx, xmm_result5, 0x05, L__incCounter[k][4]);
4749 inc_counter(rbx, xmm_curr_counter, 0x06, L__incCounter[k][5]);
4750 CTR_DoSix(pshufb, xmm_counter_shuf_mask)masm-> pshufb(xmm_result0, xmm_counter_shuf_mask); masm->
pshufb(xmm_result1, xmm_counter_shuf_mask); masm-> pshufb
(xmm_result2, xmm_counter_shuf_mask); masm-> pshufb(xmm_result3
, xmm_counter_shuf_mask); masm-> pshufb(xmm_result4, xmm_counter_shuf_mask
); masm-> pshufb(xmm_result5, xmm_counter_shuf_mask);; // after increased, shuffled counters back for PXOR
4751 CTR_DoSix(pxor, xmm_key_tmp0)masm-> pxor(xmm_result0, xmm_key_tmp0); masm-> pxor(xmm_result1
, xmm_key_tmp0); masm-> pxor(xmm_result2, xmm_key_tmp0); masm
-> pxor(xmm_result3, xmm_key_tmp0); masm-> pxor(xmm_result4
, xmm_key_tmp0); masm-> pxor(xmm_result5, xmm_key_tmp0);; //PXOR with Round 0 key
4752
4753 //load two ROUND_KEYs at a time
4754 for (int i = 1; i < rounds[k]; ) {
4755 load_key(xmm_key_tmp1, key, (0x10 * i), xmm_key_shuf_mask);
4756 load_key(xmm_key_tmp0, key, (0x10 * (i+1)), xmm_key_shuf_mask);
4757 CTR_DoSix(aesenc, xmm_key_tmp1)masm-> aesenc(xmm_result0, xmm_key_tmp1); masm-> aesenc
(xmm_result1, xmm_key_tmp1); masm-> aesenc(xmm_result2, xmm_key_tmp1
); masm-> aesenc(xmm_result3, xmm_key_tmp1); masm-> aesenc
(xmm_result4, xmm_key_tmp1); masm-> aesenc(xmm_result5, xmm_key_tmp1
);;
4758 i++;
4759 if (i != rounds[k]) {
4760 CTR_DoSix(aesenc, xmm_key_tmp0)masm-> aesenc(xmm_result0, xmm_key_tmp0); masm-> aesenc
(xmm_result1, xmm_key_tmp0); masm-> aesenc(xmm_result2, xmm_key_tmp0
); masm-> aesenc(xmm_result3, xmm_key_tmp0); masm-> aesenc
(xmm_result4, xmm_key_tmp0); masm-> aesenc(xmm_result5, xmm_key_tmp0
);;
4761 } else {
4762 CTR_DoSix(aesenclast, xmm_key_tmp0)masm-> aesenclast(xmm_result0, xmm_key_tmp0); masm-> aesenclast
(xmm_result1, xmm_key_tmp0); masm-> aesenclast(xmm_result2
, xmm_key_tmp0); masm-> aesenclast(xmm_result3, xmm_key_tmp0
); masm-> aesenclast(xmm_result4, xmm_key_tmp0); masm->
aesenclast(xmm_result5, xmm_key_tmp0);;
4763 }
4764 i++;
4765 }
4766
4767 // get next PARALLEL_FACTOR blocks into xmm_result registers
4768 __masm-> movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4769 __masm-> movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
4770 __masm-> movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
4771 __masm-> movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
4772 __masm-> movdqu(xmm_from4, Address(from, pos, Address::times_1, 4 * AESBlockSize));
4773 __masm-> movdqu(xmm_from5, Address(from, pos, Address::times_1, 5 * AESBlockSize));
4774
4775 __masm-> pxor(xmm_result0, xmm_from0);
4776 __masm-> pxor(xmm_result1, xmm_from1);
4777 __masm-> pxor(xmm_result2, xmm_from2);
4778 __masm-> pxor(xmm_result3, xmm_from3);
4779 __masm-> pxor(xmm_result4, xmm_from4);
4780 __masm-> pxor(xmm_result5, xmm_from5);
4781
4782 // store 6 results into the next 64 bytes of output
4783 __masm-> movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
4784 __masm-> movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
4785 __masm-> movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
4786 __masm-> movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
4787 __masm-> movdqu(Address(to, pos, Address::times_1, 4 * AESBlockSize), xmm_result4);
4788 __masm-> movdqu(Address(to, pos, Address::times_1, 5 * AESBlockSize), xmm_result5);
4789
4790 __masm-> addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
4791 __masm-> subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
4792 __masm-> jmp(L_multiBlock_loopTop[k]);
4793
4794 // singleBlock starts here
4795 __masm-> align(OptoLoopAlignment);
4796 __masm-> BIND(L_singleBlockLoopTop[k])bind(L_singleBlockLoopTop[k]); masm-> block_comment("L_singleBlockLoopTop[k]"
":");
4797 __masm-> cmpptr(len_reg, 0);
4798 __masm-> jcc(Assembler::lessEqual, L_exit);
4799 load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
4800 __masm-> movdqa(xmm_result0, xmm_curr_counter);
4801 inc_counter(rbx, xmm_curr_counter, 0x01, L__incCounter_single[k]);
4802 __masm-> pshufb(xmm_result0, xmm_counter_shuf_mask);
4803 __masm-> pxor(xmm_result0, xmm_key_tmp0);
4804 for (int i = 1; i < rounds[k]; i++) {
4805 load_key(xmm_key_tmp0, key, (0x10 * i), xmm_key_shuf_mask);
4806 __masm-> aesenc(xmm_result0, xmm_key_tmp0);
4807 }
4808 load_key(xmm_key_tmp0, key, (rounds[k] * 0x10), xmm_key_shuf_mask);
4809 __masm-> aesenclast(xmm_result0, xmm_key_tmp0);
4810 __masm-> cmpptr(len_reg, AESBlockSize);
4811 __masm-> jcc(Assembler::less, L_processTail_insr[k]);
4812 __masm-> movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4813 __masm-> pxor(xmm_result0, xmm_from0);
4814 __masm-> movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
4815 __masm-> addptr(pos, AESBlockSize);
4816 __masm-> subptr(len_reg, AESBlockSize);
4817 __masm-> jmp(L_singleBlockLoopTop[k]);
4818 __masm-> BIND(L_processTail_insr[k])bind(L_processTail_insr[k]); masm-> block_comment("L_processTail_insr[k]"
":"); // Process the tail part of the input array
4819 __masm-> addptr(pos, len_reg); // 1. Insert bytes from src array into xmm_from0 register
4820 __masm-> testptr(len_reg, 8);
4821 __masm-> jcc(Assembler::zero, L_processTail_4_insr[k]);
4822 __masm-> subptr(pos,8);
4823 __masm-> pinsrq(xmm_from0, Address(from, pos), 0);
4824 __masm-> BIND(L_processTail_4_insr[k])bind(L_processTail_4_insr[k]); masm-> block_comment("L_processTail_4_insr[k]"
":");
4825 __masm-> testptr(len_reg, 4);
4826 __masm-> jcc(Assembler::zero, L_processTail_2_insr[k]);
4827 __masm-> subptr(pos,4);
4828 __masm-> pslldq(xmm_from0, 4);
4829 __masm-> pinsrd(xmm_from0, Address(from, pos), 0);
4830 __masm-> BIND(L_processTail_2_insr[k])bind(L_processTail_2_insr[k]); masm-> block_comment("L_processTail_2_insr[k]"
":");
4831 __masm-> testptr(len_reg, 2);
4832 __masm-> jcc(Assembler::zero, L_processTail_1_insr[k]);
4833 __masm-> subptr(pos, 2);
4834 __masm-> pslldq(xmm_from0, 2);
4835 __masm-> pinsrw(xmm_from0, Address(from, pos), 0);
4836 __masm-> BIND(L_processTail_1_insr[k])bind(L_processTail_1_insr[k]); masm-> block_comment("L_processTail_1_insr[k]"
":");
4837 __masm-> testptr(len_reg, 1);
4838 __masm-> jcc(Assembler::zero, L_processTail_exit_insr[k]);
4839 __masm-> subptr(pos, 1);
4840 __masm-> pslldq(xmm_from0, 1);
4841 __masm-> pinsrb(xmm_from0, Address(from, pos), 0);
4842 __masm-> BIND(L_processTail_exit_insr[k])bind(L_processTail_exit_insr[k]); masm-> block_comment("L_processTail_exit_insr[k]"
":");
4843
4844 __masm-> movdqu(Address(saved_encCounter_start, 0), xmm_result0); // 2. Perform pxor of the encrypted counter and plaintext Bytes.
4845 __masm-> pxor(xmm_result0, xmm_from0); // Also the encrypted counter is saved for next invocation.
4846
4847 __masm-> testptr(len_reg, 8);
4848 __masm-> jcc(Assembler::zero, L_processTail_4_extr[k]); // 3. Extract bytes from xmm_result0 into the dest. array
4849 __masm-> pextrq(Address(to, pos), xmm_result0, 0);
4850 __masm-> psrldq(xmm_result0, 8);
4851 __masm-> addptr(pos, 8);
4852 __masm-> BIND(L_processTail_4_extr[k])bind(L_processTail_4_extr[k]); masm-> block_comment("L_processTail_4_extr[k]"
":");
4853 __masm-> testptr(len_reg, 4);
4854 __masm-> jcc(Assembler::zero, L_processTail_2_extr[k]);
4855 __masm-> pextrd(Address(to, pos), xmm_result0, 0);
4856 __masm-> psrldq(xmm_result0, 4);
4857 __masm-> addptr(pos, 4);
4858 __masm-> BIND(L_processTail_2_extr[k])bind(L_processTail_2_extr[k]); masm-> block_comment("L_processTail_2_extr[k]"
":");
4859 __masm-> testptr(len_reg, 2);
4860 __masm-> jcc(Assembler::zero, L_processTail_1_extr[k]);
4861 __masm-> pextrw(Address(to, pos), xmm_result0, 0);
4862 __masm-> psrldq(xmm_result0, 2);
4863 __masm-> addptr(pos, 2);
4864 __masm-> BIND(L_processTail_1_extr[k])bind(L_processTail_1_extr[k]); masm-> block_comment("L_processTail_1_extr[k]"
":");
4865 __masm-> testptr(len_reg, 1);
4866 __masm-> jcc(Assembler::zero, L_processTail_exit_extr[k]);
4867 __masm-> pextrb(Address(to, pos), xmm_result0, 0);
4868
4869 __masm-> BIND(L_processTail_exit_extr[k])bind(L_processTail_exit_extr[k]); masm-> block_comment("L_processTail_exit_extr[k]"
":");
4870 __masm-> movl(Address(used_addr, 0), len_reg);
4871 __masm-> jmp(L_exit);
4872
4873 }
4874
4875 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
4876 __masm-> pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
4877 __masm-> movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
4878 __masm-> pop(rbx); // pop the saved RBX.
4879#ifdef _WIN64
4880 __masm-> movl(rax, len_mem);
4881 __masm-> movptr(r13, Address(rsp, saved_r13_offset * wordSize));
4882 __masm-> movptr(r14, Address(rsp, saved_r14_offset * wordSize));
4883 __masm-> addptr(rsp, 2 * wordSize);
4884#else
4885 __masm-> pop(rax); // return 'len'
4886#endif
4887 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
4888 __masm-> ret(0);
4889 return start;
4890 }
4891
4892void roundDec(XMMRegister xmm_reg) {
4893 __masm-> vaesdec(xmm1, xmm1, xmm_reg, Assembler::AVX_512bit);
4894 __masm-> vaesdec(xmm2, xmm2, xmm_reg, Assembler::AVX_512bit);
4895 __masm-> vaesdec(xmm3, xmm3, xmm_reg, Assembler::AVX_512bit);
4896 __masm-> vaesdec(xmm4, xmm4, xmm_reg, Assembler::AVX_512bit);
4897 __masm-> vaesdec(xmm5, xmm5, xmm_reg, Assembler::AVX_512bit);
4898 __masm-> vaesdec(xmm6, xmm6, xmm_reg, Assembler::AVX_512bit);
4899 __masm-> vaesdec(xmm7, xmm7, xmm_reg, Assembler::AVX_512bit);
4900 __masm-> vaesdec(xmm8, xmm8, xmm_reg, Assembler::AVX_512bit);
4901}
4902
4903void roundDeclast(XMMRegister xmm_reg) {
4904 __masm-> vaesdeclast(xmm1, xmm1, xmm_reg, Assembler::AVX_512bit);
4905 __masm-> vaesdeclast(xmm2, xmm2, xmm_reg, Assembler::AVX_512bit);
4906 __masm-> vaesdeclast(xmm3, xmm3, xmm_reg, Assembler::AVX_512bit);
4907 __masm-> vaesdeclast(xmm4, xmm4, xmm_reg, Assembler::AVX_512bit);
4908 __masm-> vaesdeclast(xmm5, xmm5, xmm_reg, Assembler::AVX_512bit);
4909 __masm-> vaesdeclast(xmm6, xmm6, xmm_reg, Assembler::AVX_512bit);
4910 __masm-> vaesdeclast(xmm7, xmm7, xmm_reg, Assembler::AVX_512bit);
4911 __masm-> vaesdeclast(xmm8, xmm8, xmm_reg, Assembler::AVX_512bit);
4912}
4913
4914 void ev_load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask = NULL__null) {
4915 __masm-> movdqu(xmmdst, Address(key, offset));
4916 if (xmm_shuf_mask != NULL__null) {
4917 __masm-> pshufb(xmmdst, xmm_shuf_mask);
4918 } else {
4919 __masm-> pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
4920 }
4921 __masm-> evshufi64x2(xmmdst, xmmdst, xmmdst, 0x0, Assembler::AVX_512bit);
4922
4923 }
4924
4925address generate_cipherBlockChaining_decryptVectorAESCrypt() {
4926 assert(VM_Version::supports_avx512_vaes(), "need AES instructions and misaligned SSE support")do { if (!(VM_Version::supports_avx512_vaes())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 4926, "assert(" "VM_Version::supports_avx512_vaes()" ") failed"
, "need AES instructions and misaligned SSE support"); ::breakpoint
(); } } while (0);
4927 __masm-> align(CodeEntryAlignment);
4928 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
4929 address start = __masm-> pc();
4930
4931 const Register from = c_rarg0; // source array address
4932 const Register to = c_rarg1; // destination array address
4933 const Register key = c_rarg2; // key array address
4934 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
4935 // and left with the results of the last encryption block
4936#ifndef _WIN64
4937 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
4938#else
4939 const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
4940 const Register len_reg = r11; // pick the volatile windows register
4941#endif
4942
4943 Label Loop, Loop1, L_128, L_256, L_192, KEY_192, KEY_256, Loop2, Lcbc_dec_rem_loop,
4944 Lcbc_dec_rem_last, Lcbc_dec_ret, Lcbc_dec_rem, Lcbc_exit;
4945
4946 __masm-> enter();
4947
4948#ifdef _WIN64
4949 // on win64, fill len_reg from stack position
4950 __masm-> movl(len_reg, len_mem);
4951#else
4952 __masm-> push(len_reg); // Save
4953#endif
4954 __masm-> push(rbx);
4955 __masm-> vzeroupper();
4956
4957 // Temporary variable declaration for swapping key bytes
4958 const XMMRegister xmm_key_shuf_mask = xmm1;
4959 __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
4960
4961 // Calculate number of rounds from key size: 44 for 10-rounds, 52 for 12-rounds, 60 for 14-rounds
4962 const Register rounds = rbx;
4963 __masm-> movl(rounds, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
4964
4965 const XMMRegister IV = xmm0;
4966 // Load IV and broadcast value to 512-bits
4967 __masm-> evbroadcasti64x2(IV, Address(rvec, 0), Assembler::AVX_512bit);
4968
4969 // Temporary variables for storing round keys
4970 const XMMRegister RK0 = xmm30;
4971 const XMMRegister RK1 = xmm9;
4972 const XMMRegister RK2 = xmm18;
4973 const XMMRegister RK3 = xmm19;
4974 const XMMRegister RK4 = xmm20;
4975 const XMMRegister RK5 = xmm21;
4976 const XMMRegister RK6 = xmm22;
4977 const XMMRegister RK7 = xmm23;
4978 const XMMRegister RK8 = xmm24;
4979 const XMMRegister RK9 = xmm25;
4980 const XMMRegister RK10 = xmm26;
4981
4982 // Load and shuffle key
4983 // the java expanded key ordering is rotated one position from what we want
4984 // so we start from 1*16 here and hit 0*16 last
4985 ev_load_key(RK1, key, 1 * 16, xmm_key_shuf_mask);
4986 ev_load_key(RK2, key, 2 * 16, xmm_key_shuf_mask);
4987 ev_load_key(RK3, key, 3 * 16, xmm_key_shuf_mask);
4988 ev_load_key(RK4, key, 4 * 16, xmm_key_shuf_mask);
4989 ev_load_key(RK5, key, 5 * 16, xmm_key_shuf_mask);
4990 ev_load_key(RK6, key, 6 * 16, xmm_key_shuf_mask);
4991 ev_load_key(RK7, key, 7 * 16, xmm_key_shuf_mask);
4992 ev_load_key(RK8, key, 8 * 16, xmm_key_shuf_mask);
4993 ev_load_key(RK9, key, 9 * 16, xmm_key_shuf_mask);
4994 ev_load_key(RK10, key, 10 * 16, xmm_key_shuf_mask);
4995 ev_load_key(RK0, key, 0*16, xmm_key_shuf_mask);
4996
4997 // Variables for storing source cipher text
4998 const XMMRegister S0 = xmm10;
4999 const XMMRegister S1 = xmm11;
5000 const XMMRegister S2 = xmm12;
5001 const XMMRegister S3 = xmm13;
5002 const XMMRegister S4 = xmm14;
5003 const XMMRegister S5 = xmm15;
5004 const XMMRegister S6 = xmm16;
5005 const XMMRegister S7 = xmm17;
5006
5007 // Variables for storing decrypted text
5008 const XMMRegister B0 = xmm1;
5009 const XMMRegister B1 = xmm2;
5010 const XMMRegister B2 = xmm3;
5011 const XMMRegister B3 = xmm4;
5012 const XMMRegister B4 = xmm5;
5013 const XMMRegister B5 = xmm6;
5014 const XMMRegister B6 = xmm7;
5015 const XMMRegister B7 = xmm8;
5016
5017 __masm-> cmpl(rounds, 44);
5018 __masm-> jcc(Assembler::greater, KEY_192);
5019 __masm-> jmp(Loop);
5020
5021 __masm-> BIND(KEY_192)bind(KEY_192); masm-> block_comment("KEY_192" ":");
5022 const XMMRegister RK11 = xmm27;
5023 const XMMRegister RK12 = xmm28;
5024 ev_load_key(RK11, key, 11*16, xmm_key_shuf_mask);
5025 ev_load_key(RK12, key, 12*16, xmm_key_shuf_mask);
5026
5027 __masm-> cmpl(rounds, 52);
5028 __masm-> jcc(Assembler::greater, KEY_256);
5029 __masm-> jmp(Loop);
5030
5031 __masm-> BIND(KEY_256)bind(KEY_256); masm-> block_comment("KEY_256" ":");
5032 const XMMRegister RK13 = xmm29;
5033 const XMMRegister RK14 = xmm31;
5034 ev_load_key(RK13, key, 13*16, xmm_key_shuf_mask);
5035 ev_load_key(RK14, key, 14*16, xmm_key_shuf_mask);
5036
5037 __masm-> BIND(Loop)bind(Loop); masm-> block_comment("Loop" ":");
5038 __masm-> cmpl(len_reg, 512);
5039 __masm-> jcc(Assembler::below, Lcbc_dec_rem);
5040 __masm-> BIND(Loop1)bind(Loop1); masm-> block_comment("Loop1" ":");
5041 __masm-> subl(len_reg, 512);
5042 __masm-> evmovdquq(S0, Address(from, 0 * 64), Assembler::AVX_512bit);
5043 __masm-> evmovdquq(S1, Address(from, 1 * 64), Assembler::AVX_512bit);
5044 __masm-> evmovdquq(S2, Address(from, 2 * 64), Assembler::AVX_512bit);
5045 __masm-> evmovdquq(S3, Address(from, 3 * 64), Assembler::AVX_512bit);
5046 __masm-> evmovdquq(S4, Address(from, 4 * 64), Assembler::AVX_512bit);
5047 __masm-> evmovdquq(S5, Address(from, 5 * 64), Assembler::AVX_512bit);
5048 __masm-> evmovdquq(S6, Address(from, 6 * 64), Assembler::AVX_512bit);
5049 __masm-> evmovdquq(S7, Address(from, 7 * 64), Assembler::AVX_512bit);
5050 __masm-> leaq(from, Address(from, 8 * 64));
5051
5052 __masm-> evpxorq(B0, S0, RK1, Assembler::AVX_512bit);
5053 __masm-> evpxorq(B1, S1, RK1, Assembler::AVX_512bit);
5054 __masm-> evpxorq(B2, S2, RK1, Assembler::AVX_512bit);
5055 __masm-> evpxorq(B3, S3, RK1, Assembler::AVX_512bit);
5056 __masm-> evpxorq(B4, S4, RK1, Assembler::AVX_512bit);
5057 __masm-> evpxorq(B5, S5, RK1, Assembler::AVX_512bit);
5058 __masm-> evpxorq(B6, S6, RK1, Assembler::AVX_512bit);
5059 __masm-> evpxorq(B7, S7, RK1, Assembler::AVX_512bit);
5060
5061 __masm-> evalignq(IV, S0, IV, 0x06);
5062 __masm-> evalignq(S0, S1, S0, 0x06);
5063 __masm-> evalignq(S1, S2, S1, 0x06);
5064 __masm-> evalignq(S2, S3, S2, 0x06);
5065 __masm-> evalignq(S3, S4, S3, 0x06);
5066 __masm-> evalignq(S4, S5, S4, 0x06);
5067 __masm-> evalignq(S5, S6, S5, 0x06);
5068 __masm-> evalignq(S6, S7, S6, 0x06);
5069
5070 roundDec(RK2);
5071 roundDec(RK3);
5072 roundDec(RK4);
5073 roundDec(RK5);
5074 roundDec(RK6);
5075 roundDec(RK7);
5076 roundDec(RK8);
5077 roundDec(RK9);
5078 roundDec(RK10);
5079
5080 __masm-> cmpl(rounds, 44);
5081 __masm-> jcc(Assembler::belowEqual, L_128);
5082 roundDec(RK11);
5083 roundDec(RK12);
5084
5085 __masm-> cmpl(rounds, 52);
5086 __masm-> jcc(Assembler::belowEqual, L_192);
5087 roundDec(RK13);
5088 roundDec(RK14);
5089
5090 __masm-> BIND(L_256)bind(L_256); masm-> block_comment("L_256" ":");
5091 roundDeclast(RK0);
5092 __masm-> jmp(Loop2);
5093
5094 __masm-> BIND(L_128)bind(L_128); masm-> block_comment("L_128" ":");
5095 roundDeclast(RK0);
5096 __masm-> jmp(Loop2);
5097
5098 __masm-> BIND(L_192)bind(L_192); masm-> block_comment("L_192" ":");
5099 roundDeclast(RK0);
5100
5101 __masm-> BIND(Loop2)bind(Loop2); masm-> block_comment("Loop2" ":");
5102 __masm-> evpxorq(B0, B0, IV, Assembler::AVX_512bit);
5103 __masm-> evpxorq(B1, B1, S0, Assembler::AVX_512bit);
5104 __masm-> evpxorq(B2, B2, S1, Assembler::AVX_512bit);
5105 __masm-> evpxorq(B3, B3, S2, Assembler::AVX_512bit);
5106 __masm-> evpxorq(B4, B4, S3, Assembler::AVX_512bit);
5107 __masm-> evpxorq(B5, B5, S4, Assembler::AVX_512bit);
5108 __masm-> evpxorq(B6, B6, S5, Assembler::AVX_512bit);
5109 __masm-> evpxorq(B7, B7, S6, Assembler::AVX_512bit);
5110 __masm-> evmovdquq(IV, S7, Assembler::AVX_512bit);
5111
5112 __masm-> evmovdquq(Address(to, 0 * 64), B0, Assembler::AVX_512bit);
5113 __masm-> evmovdquq(Address(to, 1 * 64), B1, Assembler::AVX_512bit);
5114 __masm-> evmovdquq(Address(to, 2 * 64), B2, Assembler::AVX_512bit);
5115 __masm-> evmovdquq(Address(to, 3 * 64), B3, Assembler::AVX_512bit);
5116 __masm-> evmovdquq(Address(to, 4 * 64), B4, Assembler::AVX_512bit);
5117 __masm-> evmovdquq(Address(to, 5 * 64), B5, Assembler::AVX_512bit);
5118 __masm-> evmovdquq(Address(to, 6 * 64), B6, Assembler::AVX_512bit);
5119 __masm-> evmovdquq(Address(to, 7 * 64), B7, Assembler::AVX_512bit);
5120 __masm-> leaq(to, Address(to, 8 * 64));
5121 __masm-> jmp(Loop);
5122
5123 __masm-> BIND(Lcbc_dec_rem)bind(Lcbc_dec_rem); masm-> block_comment("Lcbc_dec_rem" ":"
);
5124 __masm-> evshufi64x2(IV, IV, IV, 0x03, Assembler::AVX_512bit);
5125
5126 __masm-> BIND(Lcbc_dec_rem_loop)bind(Lcbc_dec_rem_loop); masm-> block_comment("Lcbc_dec_rem_loop"
":");
5127 __masm-> subl(len_reg, 16);
5128 __masm-> jcc(Assembler::carrySet, Lcbc_dec_ret);
5129
5130 __masm-> movdqu(S0, Address(from, 0));
5131 __masm-> evpxorq(B0, S0, RK1, Assembler::AVX_512bit);
5132 __masm-> vaesdec(B0, B0, RK2, Assembler::AVX_512bit);
5133 __masm-> vaesdec(B0, B0, RK3, Assembler::AVX_512bit);
5134 __masm-> vaesdec(B0, B0, RK4, Assembler::AVX_512bit);
5135 __masm-> vaesdec(B0, B0, RK5, Assembler::AVX_512bit);
5136 __masm-> vaesdec(B0, B0, RK6, Assembler::AVX_512bit);
5137 __masm-> vaesdec(B0, B0, RK7, Assembler::AVX_512bit);
5138 __masm-> vaesdec(B0, B0, RK8, Assembler::AVX_512bit);
5139 __masm-> vaesdec(B0, B0, RK9, Assembler::AVX_512bit);
5140 __masm-> vaesdec(B0, B0, RK10, Assembler::AVX_512bit);
5141 __masm-> cmpl(rounds, 44);
5142 __masm-> jcc(Assembler::belowEqual, Lcbc_dec_rem_last);
5143
5144 __masm-> vaesdec(B0, B0, RK11, Assembler::AVX_512bit);
5145 __masm-> vaesdec(B0, B0, RK12, Assembler::AVX_512bit);
5146 __masm-> cmpl(rounds, 52);
5147 __masm-> jcc(Assembler::belowEqual, Lcbc_dec_rem_last);
5148
5149 __masm-> vaesdec(B0, B0, RK13, Assembler::AVX_512bit);
5150 __masm-> vaesdec(B0, B0, RK14, Assembler::AVX_512bit);
5151
5152 __masm-> BIND(Lcbc_dec_rem_last)bind(Lcbc_dec_rem_last); masm-> block_comment("Lcbc_dec_rem_last"
":");
5153 __masm-> vaesdeclast(B0, B0, RK0, Assembler::AVX_512bit);
5154
5155 __masm-> evpxorq(B0, B0, IV, Assembler::AVX_512bit);
5156 __masm-> evmovdquq(IV, S0, Assembler::AVX_512bit);
5157 __masm-> movdqu(Address(to, 0), B0);
5158 __masm-> leaq(from, Address(from, 16));
5159 __masm-> leaq(to, Address(to, 16));
5160 __masm-> jmp(Lcbc_dec_rem_loop);
5161
5162 __masm-> BIND(Lcbc_dec_ret)bind(Lcbc_dec_ret); masm-> block_comment("Lcbc_dec_ret" ":"
);
5163 __masm-> movdqu(Address(rvec, 0), IV);
5164
5165 // Zero out the round keys
5166 __masm-> evpxorq(RK0, RK0, RK0, Assembler::AVX_512bit);
5167 __masm-> evpxorq(RK1, RK1, RK1, Assembler::AVX_512bit);
5168 __masm-> evpxorq(RK2, RK2, RK2, Assembler::AVX_512bit);
5169 __masm-> evpxorq(RK3, RK3, RK3, Assembler::AVX_512bit);
5170 __masm-> evpxorq(RK4, RK4, RK4, Assembler::AVX_512bit);
5171 __masm-> evpxorq(RK5, RK5, RK5, Assembler::AVX_512bit);
5172 __masm-> evpxorq(RK6, RK6, RK6, Assembler::AVX_512bit);
5173 __masm-> evpxorq(RK7, RK7, RK7, Assembler::AVX_512bit);
5174 __masm-> evpxorq(RK8, RK8, RK8, Assembler::AVX_512bit);
5175 __masm-> evpxorq(RK9, RK9, RK9, Assembler::AVX_512bit);
5176 __masm-> evpxorq(RK10, RK10, RK10, Assembler::AVX_512bit);
5177 __masm-> cmpl(rounds, 44);
5178 __masm-> jcc(Assembler::belowEqual, Lcbc_exit);
5179 __masm-> evpxorq(RK11, RK11, RK11, Assembler::AVX_512bit);
5180 __masm-> evpxorq(RK12, RK12, RK12, Assembler::AVX_512bit);
5181 __masm-> cmpl(rounds, 52);
5182 __masm-> jcc(Assembler::belowEqual, Lcbc_exit);
5183 __masm-> evpxorq(RK13, RK13, RK13, Assembler::AVX_512bit);
5184 __masm-> evpxorq(RK14, RK14, RK14, Assembler::AVX_512bit);
5185
5186 __masm-> BIND(Lcbc_exit)bind(Lcbc_exit); masm-> block_comment("Lcbc_exit" ":");
5187 __masm-> pop(rbx);
5188#ifdef _WIN64
5189 __masm-> movl(rax, len_mem);
5190#else
5191 __masm-> pop(rax); // return length
5192#endif
5193 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
5194 __masm-> ret(0);
5195 return start;
5196}
5197
5198// Polynomial x^128+x^127+x^126+x^121+1
5199address ghash_polynomial_addr() {
5200 __masm-> align(CodeEntryAlignment);
5201 StubCodeMark mark(this, "StubRoutines", "_ghash_poly_addr");
5202 address start = __masm-> pc();
5203 __masm-> emit_data64(0x0000000000000001, relocInfo::none);
5204 __masm-> emit_data64(0xc200000000000000, relocInfo::none);
5205 return start;
5206}
5207
5208address ghash_shufflemask_addr() {
5209 __masm-> align(CodeEntryAlignment);
5210 StubCodeMark mark(this, "StubRoutines", "_ghash_shuffmask_addr");
5211 address start = __masm-> pc();
5212 __masm-> emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none);
5213 __masm-> emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none);
5214 return start;
5215}
5216
5217// Ghash single and multi block operations using AVX instructions
5218address generate_avx_ghash_processBlocks() {
5219 __masm-> align(CodeEntryAlignment);
5220
5221 StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
5222 address start = __masm-> pc();
5223
5224 // arguments
5225 const Register state = c_rarg0;
5226 const Register htbl = c_rarg1;
5227 const Register data = c_rarg2;
5228 const Register blocks = c_rarg3;
5229 __masm-> enter();
5230 // Save state before entering routine
5231 __masm-> avx_ghash(state, htbl, data, blocks);
5232 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
5233 __masm-> ret(0);
5234 return start;
5235}
5236
5237 // byte swap x86 long
5238 address generate_ghash_long_swap_mask() {
5239 __masm-> align(CodeEntryAlignment);
5240 StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
5241 address start = __masm-> pc();
5242 __masm-> emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
5243 __masm-> emit_data64(0x0706050403020100, relocInfo::none );
5244 return start;
5245 }
5246
5247 // byte swap x86 byte array
5248 address generate_ghash_byte_swap_mask() {
5249 __masm-> align(CodeEntryAlignment);
5250 StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
5251 address start = __masm-> pc();
5252 __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
5253 __masm-> emit_data64(0x0001020304050607, relocInfo::none );
5254 return start;
5255 }
5256
5257 /* Single and multi-block ghash operations */
5258 address generate_ghash_processBlocks() {
5259 __masm-> align(CodeEntryAlignment);
5260 Label L_ghash_loop, L_exit;
5261 StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
5262 address start = __masm-> pc();
5263
5264 const Register state = c_rarg0;
5265 const Register subkeyH = c_rarg1;
5266 const Register data = c_rarg2;
5267 const Register blocks = c_rarg3;
5268
5269 const XMMRegister xmm_temp0 = xmm0;
5270 const XMMRegister xmm_temp1 = xmm1;
5271 const XMMRegister xmm_temp2 = xmm2;
5272 const XMMRegister xmm_temp3 = xmm3;
5273 const XMMRegister xmm_temp4 = xmm4;
5274 const XMMRegister xmm_temp5 = xmm5;
5275 const XMMRegister xmm_temp6 = xmm6;
5276 const XMMRegister xmm_temp7 = xmm7;
5277 const XMMRegister xmm_temp8 = xmm8;
5278 const XMMRegister xmm_temp9 = xmm9;
5279 const XMMRegister xmm_temp10 = xmm10;
5280
5281 __masm-> enter();
5282
5283 __masm-> movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
5284
5285 __masm-> movdqu(xmm_temp0, Address(state, 0));
5286 __masm-> pshufb(xmm_temp0, xmm_temp10);
5287
5288
5289 __masm-> BIND(L_ghash_loop)bind(L_ghash_loop); masm-> block_comment("L_ghash_loop" ":"
);
5290 __masm-> movdqu(xmm_temp2, Address(data, 0));
5291 __masm-> pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
5292
5293 __masm-> movdqu(xmm_temp1, Address(subkeyH, 0));
5294 __masm-> pshufb(xmm_temp1, xmm_temp10);
5295
5296 __masm-> pxor(xmm_temp0, xmm_temp2);
5297
5298 //
5299 // Multiply with the hash key
5300 //
5301 __masm-> movdqu(xmm_temp3, xmm_temp0);
5302 __masm-> pclmulqdq(xmm_temp3, xmm_temp1, 0); // xmm3 holds a0*b0
5303 __masm-> movdqu(xmm_temp4, xmm_temp0);
5304 __masm-> pclmulqdq(xmm_temp4, xmm_temp1, 16); // xmm4 holds a0*b1
5305
5306 __masm-> movdqu(xmm_temp5, xmm_temp0);
5307 __masm-> pclmulqdq(xmm_temp5, xmm_temp1, 1); // xmm5 holds a1*b0
5308 __masm-> movdqu(xmm_temp6, xmm_temp0);
5309 __masm-> pclmulqdq(xmm_temp6, xmm_temp1, 17); // xmm6 holds a1*b1
5310
5311 __masm-> pxor(xmm_temp4, xmm_temp5); // xmm4 holds a0*b1 + a1*b0
5312
5313 __masm-> movdqu(xmm_temp5, xmm_temp4); // move the contents of xmm4 to xmm5
5314 __masm-> psrldq(xmm_temp4, 8); // shift by xmm4 64 bits to the right
5315 __masm-> pslldq(xmm_temp5, 8); // shift by xmm5 64 bits to the left
5316 __masm-> pxor(xmm_temp3, xmm_temp5);
5317 __masm-> pxor(xmm_temp6, xmm_temp4); // Register pair <xmm6:xmm3> holds the result
5318 // of the carry-less multiplication of
5319 // xmm0 by xmm1.
5320
5321 // We shift the result of the multiplication by one bit position
5322 // to the left to cope for the fact that the bits are reversed.
5323 __masm-> movdqu(xmm_temp7, xmm_temp3);
5324 __masm-> movdqu(xmm_temp8, xmm_temp6);
5325 __masm-> pslld(xmm_temp3, 1);
5326 __masm-> pslld(xmm_temp6, 1);
5327 __masm-> psrld(xmm_temp7, 31);
5328 __masm-> psrld(xmm_temp8, 31);
5329 __masm-> movdqu(xmm_temp9, xmm_temp7);
5330 __masm-> pslldq(xmm_temp8, 4);
5331 __masm-> pslldq(xmm_temp7, 4);
5332 __masm-> psrldq(xmm_temp9, 12);
5333 __masm-> por(xmm_temp3, xmm_temp7);
5334 __masm-> por(xmm_temp6, xmm_temp8);
5335 __masm-> por(xmm_temp6, xmm_temp9);
5336
5337 //
5338 // First phase of the reduction
5339 //
5340 // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
5341 // independently.
5342 __masm-> movdqu(xmm_temp7, xmm_temp3);
5343 __masm-> movdqu(xmm_temp8, xmm_temp3);
5344 __masm-> movdqu(xmm_temp9, xmm_temp3);
5345 __masm-> pslld(xmm_temp7, 31); // packed right shift shifting << 31
5346 __masm-> pslld(xmm_temp8, 30); // packed right shift shifting << 30
5347 __masm-> pslld(xmm_temp9, 25); // packed right shift shifting << 25
5348 __masm-> pxor(xmm_temp7, xmm_temp8); // xor the shifted versions
5349 __masm-> pxor(xmm_temp7, xmm_temp9);
5350 __masm-> movdqu(xmm_temp8, xmm_temp7);
5351 __masm-> pslldq(xmm_temp7, 12);
5352 __masm-> psrldq(xmm_temp8, 4);
5353 __masm-> pxor(xmm_temp3, xmm_temp7); // first phase of the reduction complete
5354
5355 //
5356 // Second phase of the reduction
5357 //
5358 // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
5359 // shift operations.
5360 __masm-> movdqu(xmm_temp2, xmm_temp3);
5361 __masm-> movdqu(xmm_temp4, xmm_temp3);
5362 __masm-> movdqu(xmm_temp5, xmm_temp3);
5363 __masm-> psrld(xmm_temp2, 1); // packed left shifting >> 1
5364 __masm-> psrld(xmm_temp4, 2); // packed left shifting >> 2
5365 __masm-> psrld(xmm_temp5, 7); // packed left shifting >> 7
5366 __masm-> pxor(xmm_temp2, xmm_temp4); // xor the shifted versions
5367 __masm-> pxor(xmm_temp2, xmm_temp5);
5368 __masm-> pxor(xmm_temp2, xmm_temp8);
5369 __masm-> pxor(xmm_temp3, xmm_temp2);
5370 __masm-> pxor(xmm_temp6, xmm_temp3); // the result is in xmm6
5371
5372 __masm-> decrement(blocks);
5373 __masm-> jcc(Assembler::zero, L_exit);
5374 __masm-> movdqu(xmm_temp0, xmm_temp6);
5375 __masm-> addptr(data, 16);
5376 __masm-> jmp(L_ghash_loop);
5377
5378 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
5379 __masm-> pshufb(xmm_temp6, xmm_temp10); // Byte swap 16-byte result
5380 __masm-> movdqu(Address(state, 0), xmm_temp6); // store the result
5381 __masm-> leave();
5382 __masm-> ret(0);
5383 return start;
5384 }
5385
5386 address base64_shuffle_addr()
5387 {
5388 __masm-> align64();
5389 StubCodeMark mark(this, "StubRoutines", "shuffle_base64");
5390 address start = __masm-> pc();
5391 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5392, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5392 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5392, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5393 __masm-> emit_data64(0x0405030401020001, relocInfo::none);
5394 __masm-> emit_data64(0x0a0b090a07080607, relocInfo::none);
5395 __masm-> emit_data64(0x10110f100d0e0c0d, relocInfo::none);
5396 __masm-> emit_data64(0x1617151613141213, relocInfo::none);
5397 __masm-> emit_data64(0x1c1d1b1c191a1819, relocInfo::none);
5398 __masm-> emit_data64(0x222321221f201e1f, relocInfo::none);
5399 __masm-> emit_data64(0x2829272825262425, relocInfo::none);
5400 __masm-> emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none);
5401 return start;
5402 }
5403
5404 address base64_avx2_shuffle_addr()
5405 {
5406 __masm-> align32();
5407 StubCodeMark mark(this, "StubRoutines", "avx2_shuffle_base64");
5408 address start = __masm-> pc();
5409 __masm-> emit_data64(0x0809070805060405, relocInfo::none);
5410 __masm-> emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none);
5411 __masm-> emit_data64(0x0405030401020001, relocInfo::none);
5412 __masm-> emit_data64(0x0a0b090a07080607, relocInfo::none);
5413 return start;
5414 }
5415
5416 address base64_avx2_input_mask_addr()
5417 {
5418 __masm-> align32();
5419 StubCodeMark mark(this, "StubRoutines", "avx2_input_mask_base64");
5420 address start = __masm-> pc();
5421 __masm-> emit_data64(0x8000000000000000, relocInfo::none);
5422 __masm-> emit_data64(0x8000000080000000, relocInfo::none);
5423 __masm-> emit_data64(0x8000000080000000, relocInfo::none);
5424 __masm-> emit_data64(0x8000000080000000, relocInfo::none);
5425 return start;
5426 }
5427
5428 address base64_avx2_lut_addr()
5429 {
5430 __masm-> align32();
5431 StubCodeMark mark(this, "StubRoutines", "avx2_lut_base64");
5432 address start = __masm-> pc();
5433 __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
5434 __masm-> emit_data64(0x0000f0edfcfcfcfc, relocInfo::none);
5435 __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
5436 __masm-> emit_data64(0x0000f0edfcfcfcfc, relocInfo::none);
5437
5438 // URL LUT
5439 __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
5440 __masm-> emit_data64(0x000020effcfcfcfc, relocInfo::none);
5441 __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none);
5442 __masm-> emit_data64(0x000020effcfcfcfc, relocInfo::none);
5443 return start;
5444 }
5445
5446 address base64_encoding_table_addr()
5447 {
5448 __masm-> align64();
5449 StubCodeMark mark(this, "StubRoutines", "encoding_table_base64");
5450 address start = __masm-> pc();
5451 assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5451, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5452 __masm-> emit_data64(0x4847464544434241, relocInfo::none);
5453 __masm-> emit_data64(0x504f4e4d4c4b4a49, relocInfo::none);
5454 __masm-> emit_data64(0x5857565554535251, relocInfo::none);
5455 __masm-> emit_data64(0x6665646362615a59, relocInfo::none);
5456 __masm-> emit_data64(0x6e6d6c6b6a696867, relocInfo::none);
5457 __masm-> emit_data64(0x767574737271706f, relocInfo::none);
5458 __masm-> emit_data64(0x333231307a797877, relocInfo::none);
5459 __masm-> emit_data64(0x2f2b393837363534, relocInfo::none);
5460
5461 // URL table
5462 __masm-> emit_data64(0x4847464544434241, relocInfo::none);
5463 __masm-> emit_data64(0x504f4e4d4c4b4a49, relocInfo::none);
5464 __masm-> emit_data64(0x5857565554535251, relocInfo::none);
5465 __masm-> emit_data64(0x6665646362615a59, relocInfo::none);
5466 __masm-> emit_data64(0x6e6d6c6b6a696867, relocInfo::none);
5467 __masm-> emit_data64(0x767574737271706f, relocInfo::none);
5468 __masm-> emit_data64(0x333231307a797877, relocInfo::none);
5469 __masm-> emit_data64(0x5f2d393837363534, relocInfo::none);
5470 return start;
5471 }
5472
5473 // Code for generating Base64 encoding.
5474 // Intrinsic function prototype in Base64.java:
5475 // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp,
5476 // boolean isURL) {
5477 address generate_base64_encodeBlock()
5478 {
5479 __masm-> align(CodeEntryAlignment);
5480 StubCodeMark mark(this, "StubRoutines", "implEncode");
5481 address start = __masm-> pc();
5482 __masm-> enter();
5483
5484 // Save callee-saved registers before using them
5485 __masm-> push(r12);
5486 __masm-> push(r13);
5487 __masm-> push(r14);
5488 __masm-> push(r15);
5489
5490 // arguments
5491 const Register source = c_rarg0; // Source Array
5492 const Register start_offset = c_rarg1; // start offset
5493 const Register end_offset = c_rarg2; // end offset
5494 const Register dest = c_rarg3; // destination array
5495
5496#ifndef _WIN64
5497 const Register dp = c_rarg4; // Position for writing to dest array
5498 const Register isURL = c_rarg5; // Base64 or URL character set
5499#else
5500 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64
5501 const Address isURL_mem(rbp, 7 * wordSize);
5502 const Register isURL = r10; // pick the volatile windows register
5503 const Register dp = r12;
5504 __masm-> movl(dp, dp_mem);
5505 __masm-> movl(isURL, isURL_mem);
5506#endif
5507
5508 const Register length = r14;
5509 const Register encode_table = r13;
5510 Label L_process3, L_exit, L_processdata, L_vbmiLoop, L_not512, L_32byteLoop;
5511
5512 // calculate length from offsets
5513 __masm-> movl(length, end_offset);
5514 __masm-> subl(length, start_offset);
5515 __masm-> cmpl(length, 0);
5516 __masm-> jcc(Assembler::lessEqual, L_exit);
5517
5518 // Code for 512-bit VBMI encoding. Encodes 48 input bytes into 64
5519 // output bytes. We read 64 input bytes and ignore the last 16, so be
5520 // sure not to read past the end of the input buffer.
5521 if (VM_Version::supports_avx512_vbmi()) {
5522 __masm-> cmpl(length, 64); // Do not overrun input buffer.
5523 __masm-> jcc(Assembler::below, L_not512);
5524
5525 __masm-> shll(isURL, 6); // index into decode table based on isURL
5526 __masm-> lea(encode_table, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr()));
5527 __masm-> addptr(encode_table, isURL);
5528 __masm-> shrl(isURL, 6); // restore isURL
5529
5530 __masm-> mov64(rax, 0x3036242a1016040aull); // Shifts
5531 __masm-> evmovdquq(xmm3, ExternalAddress(StubRoutines::x86::base64_shuffle_addr()), Assembler::AVX_512bit, r15);
5532 __masm-> evmovdquq(xmm2, Address(encode_table, 0), Assembler::AVX_512bit);
5533 __masm-> evpbroadcastq(xmm1, rax, Assembler::AVX_512bit);
5534
5535 __masm-> align32();
5536 __masm-> BIND(L_vbmiLoop)bind(L_vbmiLoop); masm-> block_comment("L_vbmiLoop" ":");
5537
5538 __masm-> vpermb(xmm0, xmm3, Address(source, start_offset), Assembler::AVX_512bit);
5539 __masm-> subl(length, 48);
5540
5541 // Put the input bytes into the proper lanes for writing, then
5542 // encode them.
5543 __masm-> evpmultishiftqb(xmm0, xmm1, xmm0, Assembler::AVX_512bit);
5544 __masm-> vpermb(xmm0, xmm0, xmm2, Assembler::AVX_512bit);
5545
5546 // Write to destination
5547 __masm-> evmovdquq(Address(dest, dp), xmm0, Assembler::AVX_512bit);
5548
5549 __masm-> addptr(dest, 64);
5550 __masm-> addptr(source, 48);
5551 __masm-> cmpl(length, 64);
5552 __masm-> jcc(Assembler::aboveEqual, L_vbmiLoop);
5553
5554 __masm-> vzeroupper();
5555 }
5556
5557 __masm-> BIND(L_not512)bind(L_not512); masm-> block_comment("L_not512" ":");
5558 if (VM_Version::supports_avx2()
5559 && VM_Version::supports_avx512vlbw()) {
5560 /*
5561 ** This AVX2 encoder is based off the paper at:
5562 ** https://dl.acm.org/doi/10.1145/3132709
5563 **
5564 ** We use AVX2 SIMD instructions to encode 24 bytes into 32
5565 ** output bytes.
5566 **
5567 */
5568 // Lengths under 32 bytes are done with scalar routine
5569 __masm-> cmpl(length, 31);
5570 __masm-> jcc(Assembler::belowEqual, L_process3);
5571
5572 // Set up supporting constant table data
5573 __masm-> vmovdqu(xmm9, ExternalAddress(StubRoutines::x86::base64_avx2_shuffle_addr()), rax);
5574 // 6-bit mask for 2nd and 4th (and multiples) 6-bit values
5575 __masm-> movl(rax, 0x0fc0fc00);
5576 __masm-> vmovdqu(xmm1, ExternalAddress(StubRoutines::x86::base64_avx2_input_mask_addr()), rax);
5577 __masm-> evpbroadcastd(xmm8, rax, Assembler::AVX_256bit);
5578
5579 // Multiplication constant for "shifting" right by 6 and 10
5580 // bits
5581 __masm-> movl(rax, 0x04000040);
5582
5583 __masm-> subl(length, 24);
5584 __masm-> evpbroadcastd(xmm7, rax, Assembler::AVX_256bit);
5585
5586 // For the first load, we mask off reading of the first 4
5587 // bytes into the register. This is so we can get 4 3-byte
5588 // chunks into each lane of the register, avoiding having to
5589 // handle end conditions. We then shuffle these bytes into a
5590 // specific order so that manipulation is easier.
5591 //
5592 // The initial read loads the XMM register like this:
5593 //
5594 // Lower 128-bit lane:
5595 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
5596 // | XX | XX | XX | XX | A0 | A1 | A2 | B0 | B1 | B2 | C0 | C1
5597 // | C2 | D0 | D1 | D2 |
5598 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
5599 //
5600 // Upper 128-bit lane:
5601 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
5602 // | E0 | E1 | E2 | F0 | F1 | F2 | G0 | G1 | G2 | H0 | H1 | H2
5603 // | XX | XX | XX | XX |
5604 // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
5605 //
5606 // Where A0 is the first input byte, B0 is the fourth, etc.
5607 // The alphabetical significance denotes the 3 bytes to be
5608 // consumed and encoded into 4 bytes.
5609 //
5610 // We then shuffle the register so each 32-bit word contains
5611 // the sequence:
5612 // A1 A0 A2 A1, B1, B0, B2, B1, etc.
5613 // Each of these byte sequences are then manipulated into 4
5614 // 6-bit values ready for encoding.
5615 //
5616 // If we focus on one set of 3-byte chunks, changing the
5617 // nomenclature such that A0 => a, A1 => b, and A2 => c, we
5618 // shuffle such that each 24-bit chunk contains:
5619 //
5620 // b7 b6 b5 b4 b3 b2 b1 b0 | a7 a6 a5 a4 a3 a2 a1 a0 | c7 c6
5621 // c5 c4 c3 c2 c1 c0 | b7 b6 b5 b4 b3 b2 b1 b0
5622 // Explain this step.
5623 // b3 b2 b1 b0 c5 c4 c3 c2 | c1 c0 d5 d4 d3 d2 d1 d0 | a5 a4
5624 // a3 a2 a1 a0 b5 b4 | b3 b2 b1 b0 c5 c4 c3 c2
5625 //
5626 // W first and off all but bits 4-9 and 16-21 (c5..c0 and
5627 // a5..a0) and shift them using a vector multiplication
5628 // operation (vpmulhuw) which effectively shifts c right by 6
5629 // bits and a right by 10 bits. We similarly mask bits 10-15
5630 // (d5..d0) and 22-27 (b5..b0) and shift them left by 8 and 4
5631 // bits respecively. This is done using vpmullw. We end up
5632 // with 4 6-bit values, thus splitting the 3 input bytes,
5633 // ready for encoding:
5634 // 0 0 d5..d0 0 0 c5..c0 0 0 b5..b0 0 0 a5..a0
5635 //
5636 // For translation, we recognize that there are 5 distinct
5637 // ranges of legal Base64 characters as below:
5638 //
5639 // +-------------+-------------+------------+
5640 // | 6-bit value | ASCII range | offset |
5641 // +-------------+-------------+------------+
5642 // | 0..25 | A..Z | 65 |
5643 // | 26..51 | a..z | 71 |
5644 // | 52..61 | 0..9 | -4 |
5645 // | 62 | + or - | -19 or -17 |
5646 // | 63 | / or _ | -16 or 32 |
5647 // +-------------+-------------+------------+
5648 //
5649 // We note that vpshufb does a parallel lookup in a
5650 // destination register using the lower 4 bits of bytes from a
5651 // source register. If we use a saturated subtraction and
5652 // subtract 51 from each 6-bit value, bytes from [0,51]
5653 // saturate to 0, and [52,63] map to a range of [1,12]. We
5654 // distinguish the [0,25] and [26,51] ranges by assigning a
5655 // value of 13 for all 6-bit values less than 26. We end up
5656 // with:
5657 //
5658 // +-------------+-------------+------------+
5659 // | 6-bit value | Reduced | offset |
5660 // +-------------+-------------+------------+
5661 // | 0..25 | 13 | 65 |
5662 // | 26..51 | 0 | 71 |
5663 // | 52..61 | 0..9 | -4 |
5664 // | 62 | 11 | -19 or -17 |
5665 // | 63 | 12 | -16 or 32 |
5666 // +-------------+-------------+------------+
5667 //
5668 // We then use a final vpshufb to add the appropriate offset,
5669 // translating the bytes.
5670 //
5671 // Load input bytes - only 28 bytes. Mask the first load to
5672 // not load into the full register.
5673 __masm-> vpmaskmovd(xmm1, xmm1, Address(source, start_offset, Address::times_1, -4), Assembler::AVX_256bit);
5674
5675 // Move 3-byte chunks of input (12 bytes) into 16 bytes,
5676 // ordering by:
5677 // 1, 0, 2, 1; 4, 3, 5, 4; etc. This groups 6-bit chunks
5678 // for easy masking
5679 __masm-> vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit);
5680
5681 __masm-> addl(start_offset, 24);
5682
5683 // Load masking register for first and third (and multiples)
5684 // 6-bit values.
5685 __masm-> movl(rax, 0x003f03f0);
5686 __masm-> evpbroadcastd(xmm6, rax, Assembler::AVX_256bit);
5687 // Multiplication constant for "shifting" left by 4 and 8 bits
5688 __masm-> movl(rax, 0x01000010);
5689 __masm-> evpbroadcastd(xmm5, rax, Assembler::AVX_256bit);
5690
5691 // Isolate 6-bit chunks of interest
5692 __masm-> vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit);
5693
5694 // Load constants for encoding
5695 __masm-> movl(rax, 0x19191919);
5696 __masm-> evpbroadcastd(xmm3, rax, Assembler::AVX_256bit);
5697 __masm-> movl(rax, 0x33333333);
5698 __masm-> evpbroadcastd(xmm4, rax, Assembler::AVX_256bit);
5699
5700 // Shift output bytes 0 and 2 into proper lanes
5701 __masm-> vpmulhuw(xmm2, xmm0, xmm7, Assembler::AVX_256bit);
5702
5703 // Mask and shift output bytes 1 and 3 into proper lanes and
5704 // combine
5705 __masm-> vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit);
5706 __masm-> vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit);
5707 __masm-> vpor(xmm0, xmm0, xmm2, Assembler::AVX_256bit);
5708
5709 // Find out which are 0..25. This indicates which input
5710 // values fall in the range of 'A'-'Z', which require an
5711 // additional offset (see comments above)
5712 __masm-> vpcmpgtb(xmm2, xmm0, xmm3, Assembler::AVX_256bit);
5713 __masm-> vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit);
5714 __masm-> vpsubb(xmm1, xmm1, xmm2, Assembler::AVX_256bit);
5715
5716 // Load the proper lookup table
5717 __masm-> lea(r11, ExternalAddress(StubRoutines::x86::base64_avx2_lut_addr()));
5718 __masm-> movl(r15, isURL);
5719 __masm-> shll(r15, 5);
5720 __masm-> vmovdqu(xmm2, Address(r11, r15));
5721
5722 // Shuffle the offsets based on the range calculation done
5723 // above. This allows us to add the correct offset to the
5724 // 6-bit value corresponding to the range documented above.
5725 __masm-> vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit);
5726 __masm-> vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit);
5727
5728 // Store the encoded bytes
5729 __masm-> vmovdqu(Address(dest, dp), xmm0);
5730 __masm-> addl(dp, 32);
5731
5732 __masm-> cmpl(length, 31);
5733 __masm-> jcc(Assembler::belowEqual, L_process3);
5734
5735 __masm-> align32();
5736 __masm-> BIND(L_32byteLoop)bind(L_32byteLoop); masm-> block_comment("L_32byteLoop" ":"
);
5737
5738 // Get next 32 bytes
5739 __masm-> vmovdqu(xmm1, Address(source, start_offset, Address::times_1, -4));
5740
5741 __masm-> subl(length, 24);
5742 __masm-> addl(start_offset, 24);
5743
5744 // This logic is identical to the above, with only constant
5745 // register loads removed. Shuffle the input, mask off 6-bit
5746 // chunks, shift them into place, then add the offset to
5747 // encode.
5748 __masm-> vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit);
5749
5750 __masm-> vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit);
5751 __masm-> vpmulhuw(xmm10, xmm0, xmm7, Assembler::AVX_256bit);
5752 __masm-> vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit);
5753 __masm-> vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit);
5754 __masm-> vpor(xmm0, xmm0, xmm10, Assembler::AVX_256bit);
5755 __masm-> vpcmpgtb(xmm10, xmm0, xmm3, Assembler::AVX_256bit);
5756 __masm-> vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit);
5757 __masm-> vpsubb(xmm1, xmm1, xmm10, Assembler::AVX_256bit);
5758 __masm-> vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit);
5759 __masm-> vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit);
5760
5761 // Store the encoded bytes
5762 __masm-> vmovdqu(Address(dest, dp), xmm0);
5763 __masm-> addl(dp, 32);
5764
5765 __masm-> cmpl(length, 31);
5766 __masm-> jcc(Assembler::above, L_32byteLoop);
5767
5768 __masm-> BIND(L_process3)bind(L_process3); masm-> block_comment("L_process3" ":");
5769 __masm-> vzeroupper();
5770 } else {
5771 __masm-> BIND(L_process3)bind(L_process3); masm-> block_comment("L_process3" ":");
5772 }
5773
5774 __masm-> cmpl(length, 3);
5775 __masm-> jcc(Assembler::below, L_exit);
5776
5777 // Load the encoding table based on isURL
5778 __masm-> lea(r11, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr()));
5779 __masm-> movl(r15, isURL);
5780 __masm-> shll(r15, 6);
5781 __masm-> addptr(r11, r15);
5782
5783 __masm-> BIND(L_processdata)bind(L_processdata); masm-> block_comment("L_processdata" ":"
);
5784
5785 // Load 3 bytes
5786 __masm-> load_unsigned_byte(r15, Address(source, start_offset));
5787 __masm-> load_unsigned_byte(r10, Address(source, start_offset, Address::times_1, 1));
5788 __masm-> load_unsigned_byte(r13, Address(source, start_offset, Address::times_1, 2));
5789
5790 // Build a 32-bit word with bytes 1, 2, 0, 1
5791 __masm-> movl(rax, r10);
5792 __masm-> shll(r10, 24);
5793 __masm-> orl(rax, r10);
5794
5795 __masm-> subl(length, 3);
5796
5797 __masm-> shll(r15, 8);
5798 __masm-> shll(r13, 16);
5799 __masm-> orl(rax, r15);
5800
5801 __masm-> addl(start_offset, 3);
5802
5803 __masm-> orl(rax, r13);
5804 // At this point, rax contains | byte1 | byte2 | byte0 | byte1
5805 // r13 has byte2 << 16 - need low-order 6 bits to translate.
5806 // This translated byte is the fourth output byte.
5807 __masm-> shrl(r13, 16);
5808 __masm-> andl(r13, 0x3f);
5809
5810 // The high-order 6 bits of r15 (byte0) is translated.
5811 // The translated byte is the first output byte.
5812 __masm-> shrl(r15, 10);
5813
5814 __masm-> load_unsigned_byte(r13, Address(r11, r13));
5815 __masm-> load_unsigned_byte(r15, Address(r11, r15));
5816
5817 __masm-> movb(Address(dest, dp, Address::times_1, 3), r13);
5818
5819 // Extract high-order 4 bits of byte1 and low-order 2 bits of byte0.
5820 // This translated byte is the second output byte.
5821 __masm-> shrl(rax, 4);
5822 __masm-> movl(r10, rax);
5823 __masm-> andl(rax, 0x3f);
5824
5825 __masm-> movb(Address(dest, dp, Address::times_1, 0), r15);
5826
5827 __masm-> load_unsigned_byte(rax, Address(r11, rax));
5828
5829 // Extract low-order 2 bits of byte1 and high-order 4 bits of byte2.
5830 // This translated byte is the third output byte.
5831 __masm-> shrl(r10, 18);
5832 __masm-> andl(r10, 0x3f);
5833
5834 __masm-> load_unsigned_byte(r10, Address(r11, r10));
5835
5836 __masm-> movb(Address(dest, dp, Address::times_1, 1), rax);
5837 __masm-> movb(Address(dest, dp, Address::times_1, 2), r10);
5838
5839 __masm-> addl(dp, 4);
5840 __masm-> cmpl(length, 3);
5841 __masm-> jcc(Assembler::aboveEqual, L_processdata);
5842
5843 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
5844 __masm-> pop(r15);
5845 __masm-> pop(r14);
5846 __masm-> pop(r13);
5847 __masm-> pop(r12);
5848 __masm-> leave();
5849 __masm-> ret(0);
5850 return start;
5851 }
5852
5853 // base64 AVX512vbmi tables
5854 address base64_vbmi_lookup_lo_addr() {
5855 __masm-> align64();
5856 StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64");
5857 address start = __masm-> pc();
5858 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5859, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5859 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5859, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5860 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5861 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5862 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5863 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5864 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5865 __masm-> emit_data64(0x3f8080803e808080, relocInfo::none);
5866 __masm-> emit_data64(0x3b3a393837363534, relocInfo::none);
5867 __masm-> emit_data64(0x8080808080803d3c, relocInfo::none);
5868 return start;
5869 }
5870
5871 address base64_vbmi_lookup_hi_addr() {
5872 __masm-> align64();
5873 StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64");
5874 address start = __masm-> pc();
5875 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5876, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5876 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5876, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5877 __masm-> emit_data64(0x0605040302010080, relocInfo::none);
5878 __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
5879 __masm-> emit_data64(0x161514131211100f, relocInfo::none);
5880 __masm-> emit_data64(0x8080808080191817, relocInfo::none);
5881 __masm-> emit_data64(0x201f1e1d1c1b1a80, relocInfo::none);
5882 __masm-> emit_data64(0x2827262524232221, relocInfo::none);
5883 __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
5884 __masm-> emit_data64(0x8080808080333231, relocInfo::none);
5885 return start;
5886 }
5887 address base64_vbmi_lookup_lo_url_addr() {
5888 __masm-> align64();
5889 StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64url");
5890 address start = __masm-> pc();
5891 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5892, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5892 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5892, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5893 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5894 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5895 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5896 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5897 __masm-> emit_data64(0x8080808080808080, relocInfo::none);
5898 __masm-> emit_data64(0x80803e8080808080, relocInfo::none);
5899 __masm-> emit_data64(0x3b3a393837363534, relocInfo::none);
5900 __masm-> emit_data64(0x8080808080803d3c, relocInfo::none);
5901 return start;
5902 }
5903
5904 address base64_vbmi_lookup_hi_url_addr() {
5905 __masm-> align64();
5906 StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64url");
5907 address start = __masm-> pc();
5908 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5909, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5909 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5909, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5910 __masm-> emit_data64(0x0605040302010080, relocInfo::none);
5911 __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
5912 __masm-> emit_data64(0x161514131211100f, relocInfo::none);
5913 __masm-> emit_data64(0x3f80808080191817, relocInfo::none);
5914 __masm-> emit_data64(0x201f1e1d1c1b1a80, relocInfo::none);
5915 __masm-> emit_data64(0x2827262524232221, relocInfo::none);
5916 __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
5917 __masm-> emit_data64(0x8080808080333231, relocInfo::none);
5918 return start;
5919 }
5920
5921 address base64_vbmi_pack_vec_addr() {
5922 __masm-> align64();
5923 StubCodeMark mark(this, "StubRoutines", "pack_vec_base64");
5924 address start = __masm-> pc();
5925 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5926, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5926 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5926, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5927 __masm-> emit_data64(0x090a040506000102, relocInfo::none);
5928 __masm-> emit_data64(0x161011120c0d0e08, relocInfo::none);
5929 __masm-> emit_data64(0x1c1d1e18191a1415, relocInfo::none);
5930 __masm-> emit_data64(0x292a242526202122, relocInfo::none);
5931 __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none);
5932 __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none);
5933 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
5934 __masm-> emit_data64(0x0000000000000000, relocInfo::none);
5935 return start;
5936 }
5937
5938 address base64_vbmi_join_0_1_addr() {
5939 __masm-> align64();
5940 StubCodeMark mark(this, "StubRoutines", "join_0_1_base64");
5941 address start = __masm-> pc();
5942 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5943, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5943 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5943, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5944 __masm-> emit_data64(0x090a040506000102, relocInfo::none);
5945 __masm-> emit_data64(0x161011120c0d0e08, relocInfo::none);
5946 __masm-> emit_data64(0x1c1d1e18191a1415, relocInfo::none);
5947 __masm-> emit_data64(0x292a242526202122, relocInfo::none);
5948 __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none);
5949 __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none);
5950 __masm-> emit_data64(0x494a444546404142, relocInfo::none);
5951 __masm-> emit_data64(0x565051524c4d4e48, relocInfo::none);
5952 return start;
5953 }
5954
5955 address base64_vbmi_join_1_2_addr() {
5956 __masm-> align64();
5957 StubCodeMark mark(this, "StubRoutines", "join_1_2_base64");
5958 address start = __masm-> pc();
5959 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5960, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5960 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5960, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5961 __masm-> emit_data64(0x1c1d1e18191a1415, relocInfo::none);
5962 __masm-> emit_data64(0x292a242526202122, relocInfo::none);
5963 __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none);
5964 __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none);
5965 __masm-> emit_data64(0x494a444546404142, relocInfo::none);
5966 __masm-> emit_data64(0x565051524c4d4e48, relocInfo::none);
5967 __masm-> emit_data64(0x5c5d5e58595a5455, relocInfo::none);
5968 __masm-> emit_data64(0x696a646566606162, relocInfo::none);
5969 return start;
5970 }
5971
5972 address base64_vbmi_join_2_3_addr() {
5973 __masm-> align64();
5974 StubCodeMark mark(this, "StubRoutines", "join_2_3_base64");
5975 address start = __masm-> pc();
5976 assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5977, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0)
5977 "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 5977, "assert(" "((unsigned long long)start & 0x3f) == 0"
") failed", "Alignment problem (0x%08llx)", (unsigned long long
)start); ::breakpoint(); } } while (0);
5978 __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none);
5979 __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none);
5980 __masm-> emit_data64(0x494a444546404142, relocInfo::none);
5981 __masm-> emit_data64(0x565051524c4d4e48, relocInfo::none);
5982 __masm-> emit_data64(0x5c5d5e58595a5455, relocInfo::none);
5983 __masm-> emit_data64(0x696a646566606162, relocInfo::none);
5984 __masm-> emit_data64(0x767071726c6d6e68, relocInfo::none);
5985 __masm-> emit_data64(0x7c7d7e78797a7475, relocInfo::none);
5986 return start;
5987 }
5988
5989 address base64_decoding_table_addr() {
5990 StubCodeMark mark(this, "StubRoutines", "decoding_table_base64");
5991 address start = __masm-> pc();
5992 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
5993 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
5994 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
5995 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
5996 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
5997 __masm-> emit_data64(0x3fffffff3effffff, relocInfo::none);
5998 __masm-> emit_data64(0x3b3a393837363534, relocInfo::none);
5999 __masm-> emit_data64(0xffffffffffff3d3c, relocInfo::none);
6000 __masm-> emit_data64(0x06050403020100ff, relocInfo::none);
6001 __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
6002 __masm-> emit_data64(0x161514131211100f, relocInfo::none);
6003 __masm-> emit_data64(0xffffffffff191817, relocInfo::none);
6004 __masm-> emit_data64(0x201f1e1d1c1b1aff, relocInfo::none);
6005 __masm-> emit_data64(0x2827262524232221, relocInfo::none);
6006 __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
6007 __masm-> emit_data64(0xffffffffff333231, relocInfo::none);
6008 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6009 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6010 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6011 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6012 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6013 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6014 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6015 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6016 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6017 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6018 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6019 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6020 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6021 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6022 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6023 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6024
6025 // URL table
6026 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6027 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6028 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6029 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6030 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6031 __masm-> emit_data64(0xffff3effffffffff, relocInfo::none);
6032 __masm-> emit_data64(0x3b3a393837363534, relocInfo::none);
6033 __masm-> emit_data64(0xffffffffffff3d3c, relocInfo::none);
6034 __masm-> emit_data64(0x06050403020100ff, relocInfo::none);
6035 __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none);
6036 __masm-> emit_data64(0x161514131211100f, relocInfo::none);
6037 __masm-> emit_data64(0x3fffffffff191817, relocInfo::none);
6038 __masm-> emit_data64(0x201f1e1d1c1b1aff, relocInfo::none);
6039 __masm-> emit_data64(0x2827262524232221, relocInfo::none);
6040 __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none);
6041 __masm-> emit_data64(0xffffffffff333231, relocInfo::none);
6042 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6043 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6044 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6045 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6046 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6047 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6048 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6049 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6050 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6051 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6052 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6053 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6054 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6055 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6056 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6057 __masm-> emit_data64(0xffffffffffffffff, relocInfo::none);
6058 return start;
6059 }
6060
6061
6062// Code for generating Base64 decoding.
6063//
6064// Based on the article (and associated code) from https://arxiv.org/abs/1910.05109.
6065//
6066// Intrinsic function prototype in Base64.java:
6067// private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) {
6068 address generate_base64_decodeBlock() {
6069 __masm-> align(CodeEntryAlignment);
6070 StubCodeMark mark(this, "StubRoutines", "implDecode");
6071 address start = __masm-> pc();
6072 __masm-> enter();
6073
6074 // Save callee-saved registers before using them
6075 __masm-> push(r12);
6076 __masm-> push(r13);
6077 __masm-> push(r14);
6078 __masm-> push(r15);
6079 __masm-> push(rbx);
6080
6081 // arguments
6082 const Register source = c_rarg0; // Source Array
6083 const Register start_offset = c_rarg1; // start offset
6084 const Register end_offset = c_rarg2; // end offset
6085 const Register dest = c_rarg3; // destination array
6086 const Register isMIME = rbx;
6087
6088#ifndef _WIN64
6089 const Register dp = c_rarg4; // Position for writing to dest array
6090 const Register isURL = c_rarg5;// Base64 or URL character set
6091 __masm-> movl(isMIME, Address(rbp, 2 * wordSize));
6092#else
6093 const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64
6094 const Address isURL_mem(rbp, 7 * wordSize);
6095 const Register isURL = r10; // pick the volatile windows register
6096 const Register dp = r12;
6097 __masm-> movl(dp, dp_mem);
6098 __masm-> movl(isURL, isURL_mem);
6099 __masm-> movl(isMIME, Address(rbp, 8 * wordSize));
6100#endif
6101
6102 const XMMRegister lookup_lo = xmm5;
6103 const XMMRegister lookup_hi = xmm6;
6104 const XMMRegister errorvec = xmm7;
6105 const XMMRegister pack16_op = xmm9;
6106 const XMMRegister pack32_op = xmm8;
6107 const XMMRegister input0 = xmm3;
6108 const XMMRegister input1 = xmm20;
6109 const XMMRegister input2 = xmm21;
6110 const XMMRegister input3 = xmm19;
6111 const XMMRegister join01 = xmm12;
6112 const XMMRegister join12 = xmm11;
6113 const XMMRegister join23 = xmm10;
6114 const XMMRegister translated0 = xmm2;
6115 const XMMRegister translated1 = xmm1;
6116 const XMMRegister translated2 = xmm0;
6117 const XMMRegister translated3 = xmm4;
6118
6119 const XMMRegister merged0 = xmm2;
6120 const XMMRegister merged1 = xmm1;
6121 const XMMRegister merged2 = xmm0;
6122 const XMMRegister merged3 = xmm4;
6123 const XMMRegister merge_ab_bc0 = xmm2;
6124 const XMMRegister merge_ab_bc1 = xmm1;
6125 const XMMRegister merge_ab_bc2 = xmm0;
6126 const XMMRegister merge_ab_bc3 = xmm4;
6127
6128 const XMMRegister pack24bits = xmm4;
6129
6130 const Register length = r14;
6131 const Register output_size = r13;
6132 const Register output_mask = r15;
6133 const KRegister input_mask = k1;
6134
6135 const XMMRegister input_initial_valid_b64 = xmm0;
6136 const XMMRegister tmp = xmm10;
6137 const XMMRegister mask = xmm0;
6138 const XMMRegister invalid_b64 = xmm1;
6139
6140 Label L_process256, L_process64, L_process64Loop, L_exit, L_processdata, L_loadURL;
6141 Label L_continue, L_finalBit, L_padding, L_donePadding, L_bruteForce;
6142 Label L_forceLoop, L_bottomLoop, L_checkMIME, L_exit_no_vzero;
6143
6144 // calculate length from offsets
6145 __masm-> movl(length, end_offset);
6146 __masm-> subl(length, start_offset);
6147 __masm-> push(dest); // Save for return value calc
6148
6149 // If AVX512 VBMI not supported, just compile non-AVX code
6150 if(VM_Version::supports_avx512_vbmi() &&
6151 VM_Version::supports_avx512bw()) {
6152 __masm-> cmpl(length, 128); // 128-bytes is break-even for AVX-512
6153 __masm-> jcc(Assembler::lessEqual, L_bruteForce);
6154
6155 __masm-> cmpl(isMIME, 0);
6156 __masm-> jcc(Assembler::notEqual, L_bruteForce);
6157
6158 // Load lookup tables based on isURL
6159 __masm-> cmpl(isURL, 0);
6160 __masm-> jcc(Assembler::notZero, L_loadURL);
6161
6162 __masm-> evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_addr()), Assembler::AVX_512bit, r13);
6163 __masm-> evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_addr()), Assembler::AVX_512bit, r13);
6164
6165 __masm-> BIND(L_continue)bind(L_continue); masm-> block_comment("L_continue" ":");
6166
6167 __masm-> movl(r15, 0x01400140);
6168 __masm-> evpbroadcastd(pack16_op, r15, Assembler::AVX_512bit);
6169
6170 __masm-> movl(r15, 0x00011000);
6171 __masm-> evpbroadcastd(pack32_op, r15, Assembler::AVX_512bit);
6172
6173 __masm-> cmpl(length, 0xff);
6174 __masm-> jcc(Assembler::lessEqual, L_process64);
6175
6176 // load masks required for decoding data
6177 __masm-> BIND(L_processdata)bind(L_processdata); masm-> block_comment("L_processdata" ":"
);
6178 __masm-> evmovdquq(join01, ExternalAddress(StubRoutines::x86::base64_vbmi_join_0_1_addr()), Assembler::AVX_512bit,r13);
6179 __masm-> evmovdquq(join12, ExternalAddress(StubRoutines::x86::base64_vbmi_join_1_2_addr()), Assembler::AVX_512bit, r13);
6180 __masm-> evmovdquq(join23, ExternalAddress(StubRoutines::x86::base64_vbmi_join_2_3_addr()), Assembler::AVX_512bit, r13);
6181
6182 __masm-> align32();
6183 __masm-> BIND(L_process256)bind(L_process256); masm-> block_comment("L_process256" ":"
);
6184 // Grab input data
6185 __masm-> evmovdquq(input0, Address(source, start_offset, Address::times_1, 0x00), Assembler::AVX_512bit);
6186 __masm-> evmovdquq(input1, Address(source, start_offset, Address::times_1, 0x40), Assembler::AVX_512bit);
6187 __masm-> evmovdquq(input2, Address(source, start_offset, Address::times_1, 0x80), Assembler::AVX_512bit);
6188 __masm-> evmovdquq(input3, Address(source, start_offset, Address::times_1, 0xc0), Assembler::AVX_512bit);
6189
6190 // Copy the low part of the lookup table into the destination of the permutation
6191 __masm-> evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit);
6192 __masm-> evmovdquq(translated1, lookup_lo, Assembler::AVX_512bit);
6193 __masm-> evmovdquq(translated2, lookup_lo, Assembler::AVX_512bit);
6194 __masm-> evmovdquq(translated3, lookup_lo, Assembler::AVX_512bit);
6195
6196 // Translate the base64 input into "decoded" bytes
6197 __masm-> evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit);
6198 __masm-> evpermt2b(translated1, input1, lookup_hi, Assembler::AVX_512bit);
6199 __masm-> evpermt2b(translated2, input2, lookup_hi, Assembler::AVX_512bit);
6200 __masm-> evpermt2b(translated3, input3, lookup_hi, Assembler::AVX_512bit);
6201
6202 // OR all of the translations together to check for errors (high-order bit of byte set)
6203 __masm-> vpternlogd(input0, 0xfe, input1, input2, Assembler::AVX_512bit);
6204
6205 __masm-> vpternlogd(input3, 0xfe, translated0, translated1, Assembler::AVX_512bit);
6206 __masm-> vpternlogd(input0, 0xfe, translated2, translated3, Assembler::AVX_512bit);
6207 __masm-> vpor(errorvec, input3, input0, Assembler::AVX_512bit);
6208
6209 // Check if there was an error - if so, try 64-byte chunks
6210 __masm-> evpmovb2m(k3, errorvec, Assembler::AVX_512bit);
6211 __masm-> kortestql(k3, k3);
6212 __masm-> jcc(Assembler::notZero, L_process64);
6213
6214 // The merging and shuffling happens here
6215 // We multiply each byte pair [00dddddd | 00cccccc | 00bbbbbb | 00aaaaaa]
6216 // Multiply [00cccccc] by 2^6 added to [00dddddd] to get [0000cccc | ccdddddd]
6217 // The pack16_op is a vector of 0x01400140, so multiply D by 1 and C by 0x40
6218 __masm-> vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit);
6219 __masm-> vpmaddubsw(merge_ab_bc1, translated1, pack16_op, Assembler::AVX_512bit);
6220 __masm-> vpmaddubsw(merge_ab_bc2, translated2, pack16_op, Assembler::AVX_512bit);
6221 __masm-> vpmaddubsw(merge_ab_bc3, translated3, pack16_op, Assembler::AVX_512bit);
6222
6223 // Now do the same with packed 16-bit values.
6224 // We start with [0000cccc | ccdddddd | 0000aaaa | aabbbbbb]
6225 // pack32_op is 0x00011000 (2^12, 1), so this multiplies [0000aaaa | aabbbbbb] by 2^12
6226 // and adds [0000cccc | ccdddddd] to yield [00000000 | aaaaaabb | bbbbcccc | ccdddddd]
6227 __masm-> vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit);
6228 __masm-> vpmaddwd(merged1, merge_ab_bc1, pack32_op, Assembler::AVX_512bit);
6229 __masm-> vpmaddwd(merged2, merge_ab_bc2, pack32_op, Assembler::AVX_512bit);
6230 __masm-> vpmaddwd(merged3, merge_ab_bc3, pack32_op, Assembler::AVX_512bit);
6231
6232 // The join vectors specify which byte from which vector goes into the outputs
6233 // One of every 4 bytes in the extended vector is zero, so we pack them into their
6234 // final positions in the register for storing (256 bytes in, 192 bytes out)
6235 __masm-> evpermt2b(merged0, join01, merged1, Assembler::AVX_512bit);
6236 __masm-> evpermt2b(merged1, join12, merged2, Assembler::AVX_512bit);
6237 __masm-> evpermt2b(merged2, join23, merged3, Assembler::AVX_512bit);
6238
6239 // Store result
6240 __masm-> evmovdquq(Address(dest, dp, Address::times_1, 0x00), merged0, Assembler::AVX_512bit);
6241 __masm-> evmovdquq(Address(dest, dp, Address::times_1, 0x40), merged1, Assembler::AVX_512bit);
6242 __masm-> evmovdquq(Address(dest, dp, Address::times_1, 0x80), merged2, Assembler::AVX_512bit);
6243
6244 __masm-> addptr(source, 0x100);
6245 __masm-> addptr(dest, 0xc0);
6246 __masm-> subl(length, 0x100);
6247 __masm-> cmpl(length, 64 * 4);
6248 __masm-> jcc(Assembler::greaterEqual, L_process256);
6249
6250 // At this point, we've decoded 64 * 4 * n bytes.
6251 // The remaining length will be <= 64 * 4 - 1.
6252 // UNLESS there was an error decoding the first 256-byte chunk. In this
6253 // case, the length will be arbitrarily long.
6254 //
6255 // Note that this will be the path for MIME-encoded strings.
6256
6257 __masm-> BIND(L_process64)bind(L_process64); masm-> block_comment("L_process64" ":");
6258
6259 __masm-> evmovdquq(pack24bits, ExternalAddress(StubRoutines::x86::base64_vbmi_pack_vec_addr()), Assembler::AVX_512bit, r13);
6260
6261 __masm-> cmpl(length, 63);
6262 __masm-> jcc(Assembler::lessEqual, L_finalBit);
6263
6264 __masm-> mov64(rax, 0x0000ffffffffffff);
6265 __masm-> kmovql(k2, rax);
6266
6267 __masm-> align32();
6268 __masm-> BIND(L_process64Loop)bind(L_process64Loop); masm-> block_comment("L_process64Loop"
":");
6269
6270 // Handle first 64-byte block
6271
6272 __masm-> evmovdquq(input0, Address(source, start_offset), Assembler::AVX_512bit);
6273 __masm-> evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit);
6274 __masm-> evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit);
6275
6276 __masm-> vpor(errorvec, translated0, input0, Assembler::AVX_512bit);
6277
6278 // Check for error and bomb out before updating dest
6279 __masm-> evpmovb2m(k3, errorvec, Assembler::AVX_512bit);
6280 __masm-> kortestql(k3, k3);
6281 __masm-> jcc(Assembler::notZero, L_exit);
6282
6283 // Pack output register, selecting correct byte ordering
6284 __masm-> vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit);
6285 __masm-> vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit);
6286 __masm-> vpermb(merged0, pack24bits, merged0, Assembler::AVX_512bit);
6287
6288 __masm-> evmovdqub(Address(dest, dp), k2, merged0, true, Assembler::AVX_512bit);
6289
6290 __masm-> subl(length, 64);
6291 __masm-> addptr(source, 64);
6292 __masm-> addptr(dest, 48);
6293
6294 __masm-> cmpl(length, 64);
6295 __masm-> jcc(Assembler::greaterEqual, L_process64Loop);
6296
6297 __masm-> cmpl(length, 0);
6298 __masm-> jcc(Assembler::lessEqual, L_exit);
6299
6300 __masm-> BIND(L_finalBit)bind(L_finalBit); masm-> block_comment("L_finalBit" ":");
6301 // Now have 1 to 63 bytes left to decode
6302
6303 // I was going to let Java take care of the final fragment
6304 // however it will repeatedly call this routine for every 4 bytes
6305 // of input data, so handle the rest here.
6306 __masm-> movq(rax, -1);
6307 __masm-> bzhiq(rax, rax, length); // Input mask in rax
6308
6309 __masm-> movl(output_size, length);
6310 __masm-> shrl(output_size, 2); // Find (len / 4) * 3 (output length)
6311 __masm-> lea(output_size, Address(output_size, output_size, Address::times_2, 0));
6312 // output_size in r13
6313
6314 // Strip pad characters, if any, and adjust length and mask
6315 __masm-> cmpb(Address(source, length, Address::times_1, -1), '=');
6316 __masm-> jcc(Assembler::equal, L_padding);
6317
6318 __masm-> BIND(L_donePadding)bind(L_donePadding); masm-> block_comment("L_donePadding" ":"
);
6319
6320 // Output size is (64 - output_size), output mask is (all 1s >> output_size).
6321 __masm-> kmovql(input_mask, rax);
6322 __masm-> movq(output_mask, -1);
6323 __masm-> bzhiq(output_mask, output_mask, output_size);
6324
6325 // Load initial input with all valid base64 characters. Will be used
6326 // in merging source bytes to avoid masking when determining if an error occurred.
6327 __masm-> movl(rax, 0x61616161);
6328 __masm-> evpbroadcastd(input_initial_valid_b64, rax, Assembler::AVX_512bit);
6329
6330 // A register containing all invalid base64 decoded values
6331 __masm-> movl(rax, 0x80808080);
6332 __masm-> evpbroadcastd(invalid_b64, rax, Assembler::AVX_512bit);
6333
6334 // input_mask is in k1
6335 // output_size is in r13
6336 // output_mask is in r15
6337 // zmm0 - free
6338 // zmm1 - 0x00011000
6339 // zmm2 - 0x01400140
6340 // zmm3 - errorvec
6341 // zmm4 - pack vector
6342 // zmm5 - lookup_lo
6343 // zmm6 - lookup_hi
6344 // zmm7 - errorvec
6345 // zmm8 - 0x61616161
6346 // zmm9 - 0x80808080
6347
6348 // Load only the bytes from source, merging into our "fully-valid" register
6349 __masm-> evmovdqub(input_initial_valid_b64, input_mask, Address(source, start_offset, Address::times_1, 0x0), true, Assembler::AVX_512bit);
6350
6351 // Decode all bytes within our merged input
6352 __masm-> evmovdquq(tmp, lookup_lo, Assembler::AVX_512bit);
6353 __masm-> evpermt2b(tmp, input_initial_valid_b64, lookup_hi, Assembler::AVX_512bit);
6354 __masm-> vporq(mask, tmp, input_initial_valid_b64, Assembler::AVX_512bit);
6355
6356 // Check for error. Compare (decoded | initial) to all invalid.
6357 // If any bytes have their high-order bit set, then we have an error.
6358 __masm-> evptestmb(k2, mask, invalid_b64, Assembler::AVX_512bit);
6359 __masm-> kortestql(k2, k2);
6360
6361 // If we have an error, use the brute force loop to decode what we can (4-byte chunks).
6362 __masm-> jcc(Assembler::notZero, L_bruteForce);
6363
6364 // Shuffle output bytes
6365 __masm-> vpmaddubsw(tmp, tmp, pack16_op, Assembler::AVX_512bit);
6366 __masm-> vpmaddwd(tmp, tmp, pack32_op, Assembler::AVX_512bit);
6367
6368 __masm-> vpermb(tmp, pack24bits, tmp, Assembler::AVX_512bit);
6369 __masm-> kmovql(k1, output_mask);
6370 __masm-> evmovdqub(Address(dest, dp), k1, tmp, true, Assembler::AVX_512bit);
6371
6372 __masm-> addptr(dest, output_size);
6373
6374 __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":");
6375 __masm-> vzeroupper();
6376 __masm-> pop(rax); // Get original dest value
6377 __masm-> subptr(dest, rax); // Number of bytes converted
6378 __masm-> movptr(rax, dest);
6379 __masm-> pop(rbx);
6380 __masm-> pop(r15);
6381 __masm-> pop(r14);
6382 __masm-> pop(r13);
6383 __masm-> pop(r12);
6384 __masm-> leave();
6385 __masm-> ret(0);
6386
6387 __masm-> BIND(L_loadURL)bind(L_loadURL); masm-> block_comment("L_loadURL" ":");
6388 __masm-> evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_url_addr()), Assembler::AVX_512bit, r13);
6389 __masm-> evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_url_addr()), Assembler::AVX_512bit, r13);
6390 __masm-> jmp(L_continue);
6391
6392 __masm-> BIND(L_padding)bind(L_padding); masm-> block_comment("L_padding" ":");
6393 __masm-> decrementq(output_size, 1);
6394 __masm-> shrq(rax, 1);
6395
6396 __masm-> cmpb(Address(source, length, Address::times_1, -2), '=');
6397 __masm-> jcc(Assembler::notEqual, L_donePadding);
6398
6399 __masm-> decrementq(output_size, 1);
6400 __masm-> shrq(rax, 1);
6401 __masm-> jmp(L_donePadding);
6402
6403 __masm-> align32();
6404 __masm-> BIND(L_bruteForce)bind(L_bruteForce); masm-> block_comment("L_bruteForce" ":"
);
6405 } // End of if(avx512_vbmi)
6406
6407 // Use non-AVX code to decode 4-byte chunks into 3 bytes of output
6408
6409 // Register state (Linux):
6410 // r12-15 - saved on stack
6411 // rdi - src
6412 // rsi - sp
6413 // rdx - sl
6414 // rcx - dst
6415 // r8 - dp
6416 // r9 - isURL
6417
6418 // Register state (Windows):
6419 // r12-15 - saved on stack
6420 // rcx - src
6421 // rdx - sp
6422 // r8 - sl
6423 // r9 - dst
6424 // r12 - dp
6425 // r10 - isURL
6426
6427 // Registers (common):
6428 // length (r14) - bytes in src
6429
6430 const Register decode_table = r11;
6431 const Register out_byte_count = rbx;
6432 const Register byte1 = r13;
6433 const Register byte2 = r15;
6434 const Register byte3 = WINDOWS_ONLY(r8) NOT_WINDOWS(rdx)rdx;
6435 const Register byte4 = WINDOWS_ONLY(r10) NOT_WINDOWS(r9)r9;
6436
6437 __masm-> shrl(length, 2); // Multiple of 4 bytes only - length is # 4-byte chunks
6438 __masm-> cmpl(length, 0);
6439 __masm-> jcc(Assembler::lessEqual, L_exit_no_vzero);
6440
6441 __masm-> shll(isURL, 8); // index into decode table based on isURL
6442 __masm-> lea(decode_table, ExternalAddress(StubRoutines::x86::base64_decoding_table_addr()));
6443 __masm-> addptr(decode_table, isURL);
6444
6445 __masm-> jmp(L_bottomLoop);
6446
6447 __masm-> align32();
6448 __masm-> BIND(L_forceLoop)bind(L_forceLoop); masm-> block_comment("L_forceLoop" ":");
6449 __masm-> shll(byte1, 18);
6450 __masm-> shll(byte2, 12);
6451 __masm-> shll(byte3, 6);
6452 __masm-> orl(byte1, byte2);
6453 __masm-> orl(byte1, byte3);
6454 __masm-> orl(byte1, byte4);
6455
6456 __masm-> addptr(source, 4);
6457
6458 __masm-> movb(Address(dest, dp, Address::times_1, 2), byte1);
6459 __masm-> shrl(byte1, 8);
6460 __masm-> movb(Address(dest, dp, Address::times_1, 1), byte1);
6461 __masm-> shrl(byte1, 8);
6462 __masm-> movb(Address(dest, dp, Address::times_1, 0), byte1);
6463
6464 __masm-> addptr(dest, 3);
6465 __masm-> decrementl(length, 1);
6466 __masm-> jcc(Assembler::zero, L_exit_no_vzero);
6467
6468 __masm-> BIND(L_bottomLoop)bind(L_bottomLoop); masm-> block_comment("L_bottomLoop" ":"
);
6469 __masm-> load_unsigned_byte(byte1, Address(source, start_offset, Address::times_1, 0x00));
6470 __masm-> load_unsigned_byte(byte2, Address(source, start_offset, Address::times_1, 0x01));
6471 __masm-> load_signed_byte(byte1, Address(decode_table, byte1));
6472 __masm-> load_signed_byte(byte2, Address(decode_table, byte2));
6473 __masm-> load_unsigned_byte(byte3, Address(source, start_offset, Address::times_1, 0x02));
6474 __masm-> load_unsigned_byte(byte4, Address(source, start_offset, Address::times_1, 0x03));
6475 __masm-> load_signed_byte(byte3, Address(decode_table, byte3));
6476 __masm-> load_signed_byte(byte4, Address(decode_table, byte4));
6477
6478 __masm-> mov(rax, byte1);
6479 __masm-> orl(rax, byte2);
6480 __masm-> orl(rax, byte3);
6481 __masm-> orl(rax, byte4);
6482 __masm-> jcc(Assembler::positive, L_forceLoop);
6483
6484 __masm-> BIND(L_exit_no_vzero)bind(L_exit_no_vzero); masm-> block_comment("L_exit_no_vzero"
":");
6485 __masm-> pop(rax); // Get original dest value
6486 __masm-> subptr(dest, rax); // Number of bytes converted
6487 __masm-> movptr(rax, dest);
6488 __masm-> pop(rbx);
6489 __masm-> pop(r15);
6490 __masm-> pop(r14);
6491 __masm-> pop(r13);
6492 __masm-> pop(r12);
6493 __masm-> leave();
6494 __masm-> ret(0);
6495
6496 return start;
6497 }
6498
6499
6500 /**
6501 * Arguments:
6502 *
6503 * Inputs:
6504 * c_rarg0 - int crc
6505 * c_rarg1 - byte* buf
6506 * c_rarg2 - int length
6507 *
6508 * Ouput:
6509 * rax - int crc result
6510 */
6511 address generate_updateBytesCRC32() {
6512 assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions")do { if (!(UseCRC32Intrinsics)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 6512, "assert(" "UseCRC32Intrinsics" ") failed", "need AVX and CLMUL instructions"
); ::breakpoint(); } } while (0);
6513
6514 __masm-> align(CodeEntryAlignment);
6515 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
6516
6517 address start = __masm-> pc();
6518 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
6519 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
6520 // rscratch1: r10
6521 const Register crc = c_rarg0; // crc
6522 const Register buf = c_rarg1; // source java byte array address
6523 const Register len = c_rarg2; // length
6524 const Register table = c_rarg3; // crc_table address (reuse register)
6525 const Register tmp1 = r11;
6526 const Register tmp2 = r10;
6527 assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax);
6528
6529 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6530 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6531
6532 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() &&
6533 VM_Version::supports_avx512bw() &&
6534 VM_Version::supports_avx512vl()) {
6535 // The constants used in the CRC32 algorithm requires the 1's compliment of the initial crc value.
6536 // However, the constant table for CRC32-C assumes the original crc value. Account for this
6537 // difference before calling and after returning.
6538 __masm-> lea(table, ExternalAddress(StubRoutines::x86::crc_table_avx512_addr()));
6539 __masm-> notl(crc);
6540 __masm-> kernel_crc32_avx512(crc, buf, len, table, tmp1, tmp2);
6541 __masm-> notl(crc);
6542 } else {
6543 __masm-> kernel_crc32(crc, buf, len, table, tmp1);
6544 }
6545
6546 __masm-> movl(rax, crc);
6547 __masm-> vzeroupper();
6548 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
6549 __masm-> ret(0);
6550
6551 return start;
6552 }
6553
6554 /**
6555 * Arguments:
6556 *
6557 * Inputs:
6558 * c_rarg0 - int crc
6559 * c_rarg1 - byte* buf
6560 * c_rarg2 - long length
6561 * c_rarg3 - table_start - optional (present only when doing a library_call,
6562 * not used by x86 algorithm)
6563 *
6564 * Ouput:
6565 * rax - int crc result
6566 */
6567 address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
6568 assert(UseCRC32CIntrinsics, "need SSE4_2")do { if (!(UseCRC32CIntrinsics)) { (*g_assert_poison) = 'X';;
report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 6568, "assert(" "UseCRC32CIntrinsics" ") failed", "need SSE4_2"
); ::breakpoint(); } } while (0);
6569 __masm-> align(CodeEntryAlignment);
6570 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
6571 address start = __masm-> pc();
6572 //reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs
6573 //Windows RCX RDX R8 R9 none none XMM0..XMM3
6574 //Lin / Sol RDI RSI RDX RCX R8 R9 XMM0..XMM7
6575 const Register crc = c_rarg0; // crc
6576 const Register buf = c_rarg1; // source java byte array address
6577 const Register len = c_rarg2; // length
6578 const Register a = rax;
6579 const Register j = r9;
6580 const Register k = r10;
6581 const Register l = r11;
6582#ifdef _WIN64
6583 const Register y = rdi;
6584 const Register z = rsi;
6585#else
6586 const Register y = rcx;
6587 const Register z = r8;
6588#endif
6589 assert_different_registers(crc, buf, len, a, j, k, l, y, z);
6590
6591 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6592 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6593 if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() &&
6594 VM_Version::supports_avx512bw() &&
6595 VM_Version::supports_avx512vl()) {
6596 __masm-> lea(j, ExternalAddress(StubRoutines::x86::crc32c_table_avx512_addr()));
6597 __masm-> kernel_crc32_avx512(crc, buf, len, j, l, k);
6598 } else {
6599#ifdef _WIN64
6600 __masm-> push(y);
6601 __masm-> push(z);
6602#endif
6603 __masm-> crc32c_ipl_alg2_alt2(crc, buf, len,
6604 a, j, k,
6605 l, y, z,
6606 c_farg0, c_farg1, c_farg2,
6607 is_pclmulqdq_supported);
6608#ifdef _WIN64
6609 __masm-> pop(z);
6610 __masm-> pop(y);
6611#endif
6612 }
6613 __masm-> movl(rax, crc);
6614 __masm-> vzeroupper();
6615 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
6616 __masm-> ret(0);
6617
6618 return start;
6619 }
6620
6621
6622 /***
6623 * Arguments:
6624 *
6625 * Inputs:
6626 * c_rarg0 - int adler
6627 * c_rarg1 - byte* buff
6628 * c_rarg2 - int len
6629 *
6630 * Output:
6631 * rax - int adler result
6632 */
6633
6634 address generate_updateBytesAdler32() {
6635 assert(UseAdler32Intrinsics, "need AVX2")do { if (!(UseAdler32Intrinsics)) { (*g_assert_poison) = 'X';
; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 6635, "assert(" "UseAdler32Intrinsics" ") failed", "need AVX2"
); ::breakpoint(); } } while (0);
6636
6637 __masm-> align(CodeEntryAlignment);
6638 StubCodeMark mark(this, "StubRoutines", "updateBytesAdler32");
6639
6640 address start = __masm-> pc();
6641
6642 const Register data = r9;
6643 const Register size = r10;
6644
6645 const XMMRegister yshuf0 = xmm6;
6646 const XMMRegister yshuf1 = xmm7;
6647 assert_different_registers(c_rarg0, c_rarg1, c_rarg2, data, size);
6648
6649 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6650 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6651
6652 __masm-> vmovdqu(yshuf0, ExternalAddress((address) StubRoutines::x86::_adler32_shuf0_table), r9);
6653 __masm-> vmovdqu(yshuf1, ExternalAddress((address) StubRoutines::x86::_adler32_shuf1_table), r9);
6654 __masm-> movptr(data, c_rarg1); //data
6655 __masm-> movl(size, c_rarg2); //length
6656 __masm-> updateBytesAdler32(c_rarg0, data, size, yshuf0, yshuf1, ExternalAddress((address) StubRoutines::x86::_adler32_ascale_table));
6657 __masm-> leave();
6658 __masm-> ret(0);
6659 return start;
6660 }
6661
6662 /**
6663 * Arguments:
6664 *
6665 * Input:
6666 * c_rarg0 - x address
6667 * c_rarg1 - x length
6668 * c_rarg2 - y address
6669 * c_rarg3 - y length
6670 * not Win64
6671 * c_rarg4 - z address
6672 * c_rarg5 - z length
6673 * Win64
6674 * rsp+40 - z address
6675 * rsp+48 - z length
6676 */
6677 address generate_multiplyToLen() {
6678 __masm-> align(CodeEntryAlignment);
6679 StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
6680
6681 address start = __masm-> pc();
6682 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
6683 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
6684 const Register x = rdi;
6685 const Register xlen = rax;
6686 const Register y = rsi;
6687 const Register ylen = rcx;
6688 const Register z = r8;
6689 const Register zlen = r11;
6690
6691 // Next registers will be saved on stack in multiply_to_len().
6692 const Register tmp1 = r12;
6693 const Register tmp2 = r13;
6694 const Register tmp3 = r14;
6695 const Register tmp4 = r15;
6696 const Register tmp5 = rbx;
6697
6698 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6699 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6700
6701#ifndef _WIN64
6702 __masm-> movptr(zlen, r9); // Save r9 in r11 - zlen
6703#endif
6704 setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
6705 // ylen => rcx, z => r8, zlen => r11
6706 // r9 and r10 may be used to save non-volatile registers
6707#ifdef _WIN64
6708 // last 2 arguments (#4, #5) are on stack on Win64
6709 __masm-> movptr(z, Address(rsp, 6 * wordSize));
6710 __masm-> movptr(zlen, Address(rsp, 7 * wordSize));
6711#endif
6712
6713 __masm-> movptr(xlen, rsi);
6714 __masm-> movptr(y, rdx);
6715 __masm-> multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
6716
6717 restore_arg_regs();
6718
6719 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
6720 __masm-> ret(0);
6721
6722 return start;
6723 }
6724
6725 /**
6726 * Arguments:
6727 *
6728 * Input:
6729 * c_rarg0 - obja address
6730 * c_rarg1 - objb address
6731 * c_rarg3 - length length
6732 * c_rarg4 - scale log2_array_indxscale
6733 *
6734 * Output:
6735 * rax - int >= mismatched index, < 0 bitwise complement of tail
6736 */
6737 address generate_vectorizedMismatch() {
6738 __masm-> align(CodeEntryAlignment);
6739 StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch");
6740 address start = __masm-> pc();
6741
6742 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6743 __masm-> enter();
6744
6745#ifdef _WIN64 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
6746 const Register scale = c_rarg0; //rcx, will exchange with r9
6747 const Register objb = c_rarg1; //rdx
6748 const Register length = c_rarg2; //r8
6749 const Register obja = c_rarg3; //r9
6750 __masm-> xchgq(obja, scale); //now obja and scale contains the correct contents
6751
6752 const Register tmp1 = r10;
6753 const Register tmp2 = r11;
6754#endif
6755#ifndef _WIN64 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
6756 const Register obja = c_rarg0; //U:rdi
6757 const Register objb = c_rarg1; //U:rsi
6758 const Register length = c_rarg2; //U:rdx
6759 const Register scale = c_rarg3; //U:rcx
6760 const Register tmp1 = r8;
6761 const Register tmp2 = r9;
6762#endif
6763 const Register result = rax; //return value
6764 const XMMRegister vec0 = xmm0;
6765 const XMMRegister vec1 = xmm1;
6766 const XMMRegister vec2 = xmm2;
6767
6768 __masm-> vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2);
6769
6770 __masm-> vzeroupper();
6771 __masm-> leave();
6772 __masm-> ret(0);
6773
6774 return start;
6775 }
6776
6777/**
6778 * Arguments:
6779 *
6780 // Input:
6781 // c_rarg0 - x address
6782 // c_rarg1 - x length
6783 // c_rarg2 - z address
6784 // c_rarg3 - z lenth
6785 *
6786 */
6787 address generate_squareToLen() {
6788
6789 __masm-> align(CodeEntryAlignment);
6790 StubCodeMark mark(this, "StubRoutines", "squareToLen");
6791
6792 address start = __masm-> pc();
6793 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
6794 // Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
6795 const Register x = rdi;
6796 const Register len = rsi;
6797 const Register z = r8;
6798 const Register zlen = rcx;
6799
6800 const Register tmp1 = r12;
6801 const Register tmp2 = r13;
6802 const Register tmp3 = r14;
6803 const Register tmp4 = r15;
6804 const Register tmp5 = rbx;
6805
6806 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6807 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6808
6809 setup_arg_regs(4); // x => rdi, len => rsi, z => rdx
6810 // zlen => rcx
6811 // r9 and r10 may be used to save non-volatile registers
6812 __masm-> movptr(r8, rdx);
6813 __masm-> square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
6814
6815 restore_arg_regs();
6816
6817 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
6818 __masm-> ret(0);
6819
6820 return start;
6821 }
6822
6823 address generate_method_entry_barrier() {
6824 __masm-> align(CodeEntryAlignment);
6825 StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
6826
6827 Label deoptimize_label;
6828
6829 address start = __masm-> pc();
6830
6831 __masm-> push(-1); // cookie, this is used for writing the new rsp when deoptimizing
6832
6833 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6834 __masm-> enter(); // save rbp
6835
6836 // save c_rarg0, because we want to use that value.
6837 // We could do without it but then we depend on the number of slots used by pusha
6838 __masm-> push(c_rarg0);
6839
6840 __masm-> lea(c_rarg0, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for c_rarg0 - this should be the return address
6841
6842 __masm-> pusha();
6843
6844 // The method may have floats as arguments, and we must spill them before calling
6845 // the VM runtime.
6846 assert(Argument::n_float_register_parameters_j == 8, "Assumption")do { if (!(Argument::n_float_register_parameters_j == 8)) { (
*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 6846, "assert(" "Argument::n_float_register_parameters_j == 8"
") failed", "Assumption"); ::breakpoint(); } } while (0);
6847 const int xmm_size = wordSize * 2;
6848 const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
6849 __masm-> subptr(rsp, xmm_spill_size);
6850 __masm-> movdqu(Address(rsp, xmm_size * 7), xmm7);
6851 __masm-> movdqu(Address(rsp, xmm_size * 6), xmm6);
6852 __masm-> movdqu(Address(rsp, xmm_size * 5), xmm5);
6853 __masm-> movdqu(Address(rsp, xmm_size * 4), xmm4);
6854 __masm-> movdqu(Address(rsp, xmm_size * 3), xmm3);
6855 __masm-> movdqu(Address(rsp, xmm_size * 2), xmm2);
6856 __masm-> movdqu(Address(rsp, xmm_size * 1), xmm1);
6857 __masm-> movdqu(Address(rsp, xmm_size * 0), xmm0);
6858
6859 __masm-> call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier))((address)((address_word)(static_cast<int (*)(address*)>
(BarrierSetNMethod::nmethod_stub_entry_barrier)))), 1);
6860
6861 __masm-> movdqu(xmm0, Address(rsp, xmm_size * 0));
6862 __masm-> movdqu(xmm1, Address(rsp, xmm_size * 1));
6863 __masm-> movdqu(xmm2, Address(rsp, xmm_size * 2));
6864 __masm-> movdqu(xmm3, Address(rsp, xmm_size * 3));
6865 __masm-> movdqu(xmm4, Address(rsp, xmm_size * 4));
6866 __masm-> movdqu(xmm5, Address(rsp, xmm_size * 5));
6867 __masm-> movdqu(xmm6, Address(rsp, xmm_size * 6));
6868 __masm-> movdqu(xmm7, Address(rsp, xmm_size * 7));
6869 __masm-> addptr(rsp, xmm_spill_size);
6870
6871 __masm-> cmpl(rax, 1); // 1 means deoptimize
6872 __masm-> jcc(Assembler::equal, deoptimize_label);
6873
6874 __masm-> popa();
6875 __masm-> pop(c_rarg0);
6876
6877 __masm-> leave();
6878
6879 __masm-> addptr(rsp, 1 * wordSize); // cookie
6880 __masm-> ret(0);
6881
6882
6883 __masm-> BIND(deoptimize_label)bind(deoptimize_label); masm-> block_comment("deoptimize_label"
":");
6884
6885 __masm-> popa();
6886 __masm-> pop(c_rarg0);
6887
6888 __masm-> leave();
6889
6890 // this can be taken out, but is good for verification purposes. getting a SIGSEGV
6891 // here while still having a correct stack is valuable
6892 __masm-> testptr(rsp, Address(rsp, 0));
6893
6894 __masm-> movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
6895 __masm-> jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
6896
6897 return start;
6898 }
6899
6900 /**
6901 * Arguments:
6902 *
6903 * Input:
6904 * c_rarg0 - out address
6905 * c_rarg1 - in address
6906 * c_rarg2 - offset
6907 * c_rarg3 - len
6908 * not Win64
6909 * c_rarg4 - k
6910 * Win64
6911 * rsp+40 - k
6912 */
6913 address generate_mulAdd() {
6914 __masm-> align(CodeEntryAlignment);
6915 StubCodeMark mark(this, "StubRoutines", "mulAdd");
6916
6917 address start = __masm-> pc();
6918 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
6919 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
6920 const Register out = rdi;
6921 const Register in = rsi;
6922 const Register offset = r11;
6923 const Register len = rcx;
6924 const Register k = r8;
6925
6926 // Next registers will be saved on stack in mul_add().
6927 const Register tmp1 = r12;
6928 const Register tmp2 = r13;
6929 const Register tmp3 = r14;
6930 const Register tmp4 = r15;
6931 const Register tmp5 = rbx;
6932
6933 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6934 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6935
6936 setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx
6937 // len => rcx, k => r8
6938 // r9 and r10 may be used to save non-volatile registers
6939#ifdef _WIN64
6940 // last argument is on stack on Win64
6941 __masm-> movl(k, Address(rsp, 6 * wordSize));
6942#endif
6943 __masm-> movptr(r11, rdx); // move offset in rdx to offset(r11)
6944 __masm-> mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
6945
6946 restore_arg_regs();
6947
6948 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
6949 __masm-> ret(0);
6950
6951 return start;
6952 }
6953
6954 address generate_bigIntegerRightShift() {
6955 __masm-> align(CodeEntryAlignment);
6956 StubCodeMark mark(this, "StubRoutines", "bigIntegerRightShiftWorker");
6957
6958 address start = __masm-> pc();
6959 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit;
6960 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8.
6961 const Register newArr = rdi;
6962 const Register oldArr = rsi;
6963 const Register newIdx = rdx;
6964 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift.
6965 const Register totalNumIter = r8;
6966
6967 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps.
6968 // For everything else, we prefer using r9 and r10 since we do not have to save them before use.
6969 const Register tmp1 = r11; // Caller save.
6970 const Register tmp2 = rax; // Caller save.
35
←
'tmp2' initialized to a null pointer value
→
6971 const Register tmp3 = WINDOWS_ONLY(r12) NOT_WINDOWS(r9)r9; // Windows: Callee save. Linux: Caller save.
6972 const Register tmp4 = WINDOWS_ONLY(r13) NOT_WINDOWS(r10)r10; // Windows: Callee save. Linux: Caller save.
6973 const Register tmp5 = r14; // Callee save.
6974 const Register tmp6 = r15;
6975
6976 const XMMRegister x0 = xmm0;
6977 const XMMRegister x1 = xmm1;
6978 const XMMRegister x2 = xmm2;
6979
6980 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
6981 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
6982
6983#ifdef _WINDOWS
6984 setup_arg_regs(4);
6985 // For windows, since last argument is on stack, we need to move it to the appropriate register.
6986 __masm-> movl(totalNumIter, Address(rsp, 6 * wordSize));
6987 // Save callee save registers.
6988 __masm-> push(tmp3);
6989 __masm-> push(tmp4);
6990#endif
6991 __masm-> push(tmp5);
6992
6993 // Rename temps used throughout the code.
6994 const Register idx = tmp1;
6995 const Register nIdx = tmp2;
36
←
'nIdx' initialized to a null pointer value
→
6996
6997 __masm-> xorl(idx, idx);
6998
6999 // Start right shift from end of the array.
7000 // For example, if #iteration = 4 and newIdx = 1
7001 // then dest[4] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32)
7002 // if #iteration = 4 and newIdx = 0
7003 // then dest[3] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32)
7004 __masm-> movl(idx, totalNumIter);
7005 __masm-> movl(nIdx, idx);
7006 __masm-> addl(nIdx, newIdx);
7007
7008 // If vectorization is enabled, check if the number of iterations is at least 64
7009 // If not, then go to ShifTwo processing 2 iterations
7010 if (VM_Version::supports_avx512_vbmi2()) {
37
←
Calling 'VM_Version::supports_avx512_vbmi2'
→
40
←
Returning from 'VM_Version::supports_avx512_vbmi2'
→
41
←
Taking true branch
→
7011 __masm-> cmpptr(totalNumIter, (AVX3Threshold/64));
7012 __masm-> jcc(Assembler::less, ShiftTwo);
7013
7014 if (AVX3Threshold < 16 * 64) {
42
←
Assuming the condition is false
→
43
←
Taking false branch
→
7015 __masm-> cmpl(totalNumIter, 16);
7016 __masm-> jcc(Assembler::less, ShiftTwo);
7017 }
7018 __masm-> evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit);
7019 __masm-> subl(idx, 16);
7020 __masm-> subl(nIdx, 16);
7021 __masm-> BIND(Shift512Loop)bind(Shift512Loop); masm-> block_comment("Shift512Loop" ":"
);
7022 __masm-> evmovdqul(x2, Address(oldArr, idx, Address::times_4, 4), Assembler::AVX_512bit);
7023 __masm-> evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit);
7024 __masm-> vpshrdvd(x2, x1, x0, Assembler::AVX_512bit);
7025 __masm-> evmovdqul(Address(newArr, nIdx, Address::times_4), x2, Assembler::AVX_512bit);
44
←
Passing null pointer value via 2nd parameter 'index'
→
45
←
Calling constructor for 'Address'
→
7026 __masm-> subl(nIdx, 16);
7027 __masm-> subl(idx, 16);
7028 __masm-> jcc(Assembler::greaterEqual, Shift512Loop);
7029 __masm-> addl(idx, 16);
7030 __masm-> addl(nIdx, 16);
7031 }
7032 __masm-> BIND(ShiftTwo)bind(ShiftTwo); masm-> block_comment("ShiftTwo" ":");
7033 __masm-> cmpl(idx, 2);
7034 __masm-> jcc(Assembler::less, ShiftOne);
7035 __masm-> subl(idx, 2);
7036 __masm-> subl(nIdx, 2);
7037 __masm-> BIND(ShiftTwoLoop)bind(ShiftTwoLoop); masm-> block_comment("ShiftTwoLoop" ":"
);
7038 __masm-> movl(tmp5, Address(oldArr, idx, Address::times_4, 8));
7039 __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 4));
7040 __masm-> movl(tmp3, Address(oldArr, idx, Address::times_4));
7041 __masm-> shrdl(tmp5, tmp4);
7042 __masm-> shrdl(tmp4, tmp3);
7043 __masm-> movl(Address(newArr, nIdx, Address::times_4, 4), tmp5);
7044 __masm-> movl(Address(newArr, nIdx, Address::times_4), tmp4);
7045 __masm-> subl(nIdx, 2);
7046 __masm-> subl(idx, 2);
7047 __masm-> jcc(Assembler::greaterEqual, ShiftTwoLoop);
7048 __masm-> addl(idx, 2);
7049 __masm-> addl(nIdx, 2);
7050
7051 // Do the last iteration
7052 __masm-> BIND(ShiftOne)bind(ShiftOne); masm-> block_comment("ShiftOne" ":");
7053 __masm-> cmpl(idx, 1);
7054 __masm-> jcc(Assembler::less, Exit);
7055 __masm-> subl(idx, 1);
7056 __masm-> subl(nIdx, 1);
7057 __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 4));
7058 __masm-> movl(tmp3, Address(oldArr, idx, Address::times_4));
7059 __masm-> shrdl(tmp4, tmp3);
7060 __masm-> movl(Address(newArr, nIdx, Address::times_4), tmp4);
7061 __masm-> BIND(Exit)bind(Exit); masm-> block_comment("Exit" ":");
7062 // Restore callee save registers.
7063 __masm-> pop(tmp5);
7064#ifdef _WINDOWS
7065 __masm-> pop(tmp4);
7066 __masm-> pop(tmp3);
7067 restore_arg_regs();
7068#endif
7069 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7070 __masm-> ret(0);
7071 return start;
7072 }
7073
7074 /**
7075 * Arguments:
7076 *
7077 * Input:
7078 * c_rarg0 - newArr address
7079 * c_rarg1 - oldArr address
7080 * c_rarg2 - newIdx
7081 * c_rarg3 - shiftCount
7082 * not Win64
7083 * c_rarg4 - numIter
7084 * Win64
7085 * rsp40 - numIter
7086 */
7087 address generate_bigIntegerLeftShift() {
7088 __masm-> align(CodeEntryAlignment);
7089 StubCodeMark mark(this, "StubRoutines", "bigIntegerLeftShiftWorker");
7090 address start = __masm-> pc();
7091 Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit;
7092 // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8.
7093 const Register newArr = rdi;
7094 const Register oldArr = rsi;
7095 const Register newIdx = rdx;
7096 const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift.
7097 const Register totalNumIter = r8;
7098 // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps.
7099 // For everything else, we prefer using r9 and r10 since we do not have to save them before use.
7100 const Register tmp1 = r11; // Caller save.
7101 const Register tmp2 = rax; // Caller save.
7102 const Register tmp3 = WINDOWS_ONLY(r12) NOT_WINDOWS(r9)r9; // Windows: Callee save. Linux: Caller save.
7103 const Register tmp4 = WINDOWS_ONLY(r13) NOT_WINDOWS(r10)r10; // Windows: Callee save. Linux: Caller save.
7104 const Register tmp5 = r14; // Callee save.
7105
7106 const XMMRegister x0 = xmm0;
7107 const XMMRegister x1 = xmm1;
7108 const XMMRegister x2 = xmm2;
7109 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7110 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7111
7112#ifdef _WINDOWS
7113 setup_arg_regs(4);
7114 // For windows, since last argument is on stack, we need to move it to the appropriate register.
7115 __masm-> movl(totalNumIter, Address(rsp, 6 * wordSize));
7116 // Save callee save registers.
7117 __masm-> push(tmp3);
7118 __masm-> push(tmp4);
7119#endif
7120 __masm-> push(tmp5);
7121
7122 // Rename temps used throughout the code
7123 const Register idx = tmp1;
7124 const Register numIterTmp = tmp2;
7125
7126 // Start idx from zero.
7127 __masm-> xorl(idx, idx);
7128 // Compute interior pointer for new array. We do this so that we can use same index for both old and new arrays.
7129 __masm-> lea(newArr, Address(newArr, newIdx, Address::times_4));
7130 __masm-> movl(numIterTmp, totalNumIter);
7131
7132 // If vectorization is enabled, check if the number of iterations is at least 64
7133 // If not, then go to ShiftTwo shifting two numbers at a time
7134 if (VM_Version::supports_avx512_vbmi2()) {
7135 __masm-> cmpl(totalNumIter, (AVX3Threshold/64));
7136 __masm-> jcc(Assembler::less, ShiftTwo);
7137
7138 if (AVX3Threshold < 16 * 64) {
7139 __masm-> cmpl(totalNumIter, 16);
7140 __masm-> jcc(Assembler::less, ShiftTwo);
7141 }
7142 __masm-> evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit);
7143 __masm-> subl(numIterTmp, 16);
7144 __masm-> BIND(Shift512Loop)bind(Shift512Loop); masm-> block_comment("Shift512Loop" ":"
);
7145 __masm-> evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit);
7146 __masm-> evmovdqul(x2, Address(oldArr, idx, Address::times_4, 0x4), Assembler::AVX_512bit);
7147 __masm-> vpshldvd(x1, x2, x0, Assembler::AVX_512bit);
7148 __masm-> evmovdqul(Address(newArr, idx, Address::times_4), x1, Assembler::AVX_512bit);
7149 __masm-> addl(idx, 16);
7150 __masm-> subl(numIterTmp, 16);
7151 __masm-> jcc(Assembler::greaterEqual, Shift512Loop);
7152 __masm-> addl(numIterTmp, 16);
7153 }
7154 __masm-> BIND(ShiftTwo)bind(ShiftTwo); masm-> block_comment("ShiftTwo" ":");
7155 __masm-> cmpl(totalNumIter, 1);
7156 __masm-> jcc(Assembler::less, Exit);
7157 __masm-> movl(tmp3, Address(oldArr, idx, Address::times_4));
7158 __masm-> subl(numIterTmp, 2);
7159 __masm-> jcc(Assembler::less, ShiftOne);
7160
7161 __masm-> BIND(ShiftTwoLoop)bind(ShiftTwoLoop); masm-> block_comment("ShiftTwoLoop" ":"
);
7162 __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4));
7163 __masm-> movl(tmp5, Address(oldArr, idx, Address::times_4, 0x8));
7164 __masm-> shldl(tmp3, tmp4);
7165 __masm-> shldl(tmp4, tmp5);
7166 __masm-> movl(Address(newArr, idx, Address::times_4), tmp3);
7167 __masm-> movl(Address(newArr, idx, Address::times_4, 0x4), tmp4);
7168 __masm-> movl(tmp3, tmp5);
7169 __masm-> addl(idx, 2);
7170 __masm-> subl(numIterTmp, 2);
7171 __masm-> jcc(Assembler::greaterEqual, ShiftTwoLoop);
7172
7173 // Do the last iteration
7174 __masm-> BIND(ShiftOne)bind(ShiftOne); masm-> block_comment("ShiftOne" ":");
7175 __masm-> addl(numIterTmp, 2);
7176 __masm-> cmpl(numIterTmp, 1);
7177 __masm-> jcc(Assembler::less, Exit);
7178 __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4));
7179 __masm-> shldl(tmp3, tmp4);
7180 __masm-> movl(Address(newArr, idx, Address::times_4), tmp3);
7181
7182 __masm-> BIND(Exit)bind(Exit); masm-> block_comment("Exit" ":");
7183 // Restore callee save registers.
7184 __masm-> pop(tmp5);
7185#ifdef _WINDOWS
7186 __masm-> pop(tmp4);
7187 __masm-> pop(tmp3);
7188 restore_arg_regs();
7189#endif
7190 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7191 __masm-> ret(0);
7192 return start;
7193 }
7194
7195 address generate_libmExp() {
7196 StubCodeMark mark(this, "StubRoutines", "libmExp");
7197
7198 address start = __masm-> pc();
7199
7200 const XMMRegister x0 = xmm0;
7201 const XMMRegister x1 = xmm1;
7202 const XMMRegister x2 = xmm2;
7203 const XMMRegister x3 = xmm3;
7204
7205 const XMMRegister x4 = xmm4;
7206 const XMMRegister x5 = xmm5;
7207 const XMMRegister x6 = xmm6;
7208 const XMMRegister x7 = xmm7;
7209
7210 const Register tmp = r11;
7211
7212 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7213 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7214
7215 __masm-> fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
7216
7217 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7218 __masm-> ret(0);
7219
7220 return start;
7221
7222 }
7223
7224 address generate_libmLog() {
7225 StubCodeMark mark(this, "StubRoutines", "libmLog");
7226
7227 address start = __masm-> pc();
7228
7229 const XMMRegister x0 = xmm0;
7230 const XMMRegister x1 = xmm1;
7231 const XMMRegister x2 = xmm2;
7232 const XMMRegister x3 = xmm3;
7233
7234 const XMMRegister x4 = xmm4;
7235 const XMMRegister x5 = xmm5;
7236 const XMMRegister x6 = xmm6;
7237 const XMMRegister x7 = xmm7;
7238
7239 const Register tmp1 = r11;
7240 const Register tmp2 = r8;
7241
7242 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7243 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7244
7245 __masm-> fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2);
7246
7247 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7248 __masm-> ret(0);
7249
7250 return start;
7251
7252 }
7253
7254 address generate_libmLog10() {
7255 StubCodeMark mark(this, "StubRoutines", "libmLog10");
7256
7257 address start = __masm-> pc();
7258
7259 const XMMRegister x0 = xmm0;
7260 const XMMRegister x1 = xmm1;
7261 const XMMRegister x2 = xmm2;
7262 const XMMRegister x3 = xmm3;
7263
7264 const XMMRegister x4 = xmm4;
7265 const XMMRegister x5 = xmm5;
7266 const XMMRegister x6 = xmm6;
7267 const XMMRegister x7 = xmm7;
7268
7269 const Register tmp = r11;
7270
7271 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7272 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7273
7274 __masm-> fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
7275
7276 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7277 __masm-> ret(0);
7278
7279 return start;
7280
7281 }
7282
7283 address generate_libmPow() {
7284 StubCodeMark mark(this, "StubRoutines", "libmPow");
7285
7286 address start = __masm-> pc();
7287
7288 const XMMRegister x0 = xmm0;
7289 const XMMRegister x1 = xmm1;
7290 const XMMRegister x2 = xmm2;
7291 const XMMRegister x3 = xmm3;
7292
7293 const XMMRegister x4 = xmm4;
7294 const XMMRegister x5 = xmm5;
7295 const XMMRegister x6 = xmm6;
7296 const XMMRegister x7 = xmm7;
7297
7298 const Register tmp1 = r8;
7299 const Register tmp2 = r9;
7300 const Register tmp3 = r10;
7301 const Register tmp4 = r11;
7302
7303 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7304 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7305
7306 __masm-> fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
7307
7308 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7309 __masm-> ret(0);
7310
7311 return start;
7312
7313 }
7314
7315 address generate_libmSin() {
7316 StubCodeMark mark(this, "StubRoutines", "libmSin");
7317
7318 address start = __masm-> pc();
7319
7320 const XMMRegister x0 = xmm0;
7321 const XMMRegister x1 = xmm1;
7322 const XMMRegister x2 = xmm2;
7323 const XMMRegister x3 = xmm3;
7324
7325 const XMMRegister x4 = xmm4;
7326 const XMMRegister x5 = xmm5;
7327 const XMMRegister x6 = xmm6;
7328 const XMMRegister x7 = xmm7;
7329
7330 const Register tmp1 = r8;
7331 const Register tmp2 = r9;
7332 const Register tmp3 = r10;
7333 const Register tmp4 = r11;
7334
7335 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7336 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7337
7338#ifdef _WIN64
7339 __masm-> push(rsi);
7340 __masm-> push(rdi);
7341#endif
7342 __masm-> fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
7343
7344#ifdef _WIN64
7345 __masm-> pop(rdi);
7346 __masm-> pop(rsi);
7347#endif
7348
7349 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7350 __masm-> ret(0);
7351
7352 return start;
7353
7354 }
7355
7356 address generate_libmCos() {
7357 StubCodeMark mark(this, "StubRoutines", "libmCos");
7358
7359 address start = __masm-> pc();
7360
7361 const XMMRegister x0 = xmm0;
7362 const XMMRegister x1 = xmm1;
7363 const XMMRegister x2 = xmm2;
7364 const XMMRegister x3 = xmm3;
7365
7366 const XMMRegister x4 = xmm4;
7367 const XMMRegister x5 = xmm5;
7368 const XMMRegister x6 = xmm6;
7369 const XMMRegister x7 = xmm7;
7370
7371 const Register tmp1 = r8;
7372 const Register tmp2 = r9;
7373 const Register tmp3 = r10;
7374 const Register tmp4 = r11;
7375
7376 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7377 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7378
7379#ifdef _WIN64
7380 __masm-> push(rsi);
7381 __masm-> push(rdi);
7382#endif
7383 __masm-> fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
7384
7385#ifdef _WIN64
7386 __masm-> pop(rdi);
7387 __masm-> pop(rsi);
7388#endif
7389
7390 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7391 __masm-> ret(0);
7392
7393 return start;
7394
7395 }
7396
7397 address generate_libmTan() {
7398 StubCodeMark mark(this, "StubRoutines", "libmTan");
7399
7400 address start = __masm-> pc();
7401
7402 const XMMRegister x0 = xmm0;
7403 const XMMRegister x1 = xmm1;
7404 const XMMRegister x2 = xmm2;
7405 const XMMRegister x3 = xmm3;
7406
7407 const XMMRegister x4 = xmm4;
7408 const XMMRegister x5 = xmm5;
7409 const XMMRegister x6 = xmm6;
7410 const XMMRegister x7 = xmm7;
7411
7412 const Register tmp1 = r8;
7413 const Register tmp2 = r9;
7414 const Register tmp3 = r10;
7415 const Register tmp4 = r11;
7416
7417 BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:");
7418 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7419
7420#ifdef _WIN64
7421 __masm-> push(rsi);
7422 __masm-> push(rdi);
7423#endif
7424 __masm-> fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
7425
7426#ifdef _WIN64
7427 __masm-> pop(rdi);
7428 __masm-> pop(rsi);
7429#endif
7430
7431 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7432 __masm-> ret(0);
7433
7434 return start;
7435
7436 }
7437
7438#undef __masm->
7439#define __masm-> masm->
7440
7441 // Continuation point for throwing of implicit exceptions that are
7442 // not handled in the current activation. Fabricates an exception
7443 // oop and initiates normal exception dispatching in this
7444 // frame. Since we need to preserve callee-saved values (currently
7445 // only for C2, but done for C1 as well) we need a callee-saved oop
7446 // map and therefore have to make these stubs into RuntimeStubs
7447 // rather than BufferBlobs. If the compiler needs all registers to
7448 // be preserved between the fault point and the exception handler
7449 // then it must assume responsibility for that in
7450 // AbstractCompiler::continuation_for_implicit_null_exception or
7451 // continuation_for_implicit_division_by_zero_exception. All other
7452 // implicit exceptions (e.g., NullPointerException or
7453 // AbstractMethodError on entry) are either at call sites or
7454 // otherwise assume that stack unwinding will be initiated, so
7455 // caller saved registers were assumed volatile in the compiler.
7456 address generate_throw_exception(const char* name,
7457 address runtime_entry,
7458 Register arg1 = noreg,
7459 Register arg2 = noreg) {
7460 // Information about frame layout at time of blocking runtime call.
7461 // Note that we only have to preserve callee-saved registers since
7462 // the compilers are responsible for supplying a continuation point
7463 // if they expect all registers to be preserved.
7464 enum layout {
7465 rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
7466 rbp_off2,
7467 return_off,
7468 return_off2,
7469 framesize // inclusive of return address
7470 };
7471
7472 int insts_size = 512;
7473 int locs_size = 64;
7474
7475 CodeBuffer code(name, insts_size, locs_size);
7476 OopMapSet* oop_maps = new OopMapSet();
7477 MacroAssembler* masm = new MacroAssembler(&code);
7478
7479 address start = __masm-> pc();
7480
7481 // This is an inlined and slightly modified version of call_VM
7482 // which has the ability to fetch the return PC out of
7483 // thread-local storage and also sets up last_Java_sp slightly
7484 // differently than the real call_VM
7485
7486 __masm-> enter(); // required for proper stackwalking of RuntimeStub frame
7487
7488 assert(is_even(framesize/2), "sp not 16-byte aligned")do { if (!(is_even(framesize/2))) { (*g_assert_poison) = 'X';
; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 7488, "assert(" "is_even(framesize/2)" ") failed", "sp not 16-byte aligned"
); ::breakpoint(); } } while (0);
7489
7490 // return address and rbp are already in place
7491 __masm-> subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
7492
7493 int frame_complete = __masm-> pc() - start;
7494
7495 // Set up last_Java_sp and last_Java_fp
7496 address the_pc = __masm-> pc();
7497 __masm-> set_last_Java_frame(rsp, rbp, the_pc);
7498 __masm-> andptr(rsp, -(StackAlignmentInBytes)); // Align stack
7499
7500 // Call runtime
7501 if (arg1 != noreg) {
7502 assert(arg2 != c_rarg1, "clobbered")do { if (!(arg2 != c_rarg1)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp"
, 7502, "assert(" "arg2 != c_rarg1" ") failed", "clobbered");
::breakpoint(); } } while (0);
7503 __masm-> movptr(c_rarg1, arg1);
7504 }
7505 if (arg2 != noreg) {
7506 __masm-> movptr(c_rarg2, arg2);
7507 }
7508 __masm-> movptr(c_rarg0, r15_thread);
7509 BLOCK_COMMENT("call runtime_entry")masm-> block_comment("call runtime_entry");
7510 __masm-> call(RuntimeAddress(runtime_entry));
7511
7512 // Generate oop map
7513 OopMap* map = new OopMap(framesize, 0);
7514
7515 oop_maps->add_gc_map(the_pc - start, map);
7516
7517 __masm-> reset_last_Java_frame(true);
7518
7519 __masm-> leave(); // required for proper stackwalking of RuntimeStub frame
7520
7521 // check for pending exceptions
7522#ifdef ASSERT1
7523 Label L;
7524 __masm-> cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
7525 (int32_t) NULL_WORD0L);
7526 __masm-> jcc(Assembler::notEqual, L);
7527 __masm-> should_not_reach_here();
7528 __masm-> bind(L);
7529#endif // ASSERT
7530 __masm-> jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
7531
7532
7533 // codeBlob framesize is in words (not VMRegImpl::slot_size)
7534 RuntimeStub* stub =
7535 RuntimeStub::new_runtime_stub(name,
7536 &code,
7537 frame_complete,
7538 (framesize >> (LogBytesPerWord - LogBytesPerInt)),
7539 oop_maps, false);
7540 return stub->entry_point();
7541 }
7542
7543 void create_control_words() {
7544 // Round to nearest, 64-bit mode, exceptions masked
7545 StubRoutines::x86::_mxcsr_std = 0x1F80;
7546 }
7547
7548 // Initialization
7549 void generate_initial() {
7550 // Generates all stubs and initializes the entry points
7551
7552 // This platform-specific settings are needed by generate_call_stub()
7553 create_control_words();
7554
7555 // entry points that exist in all platforms Note: This is code
7556 // that could be shared among different platforms - however the
7557 // benefit seems to be smaller than the disadvantage of having a
7558 // much more complicated generator structure. See also comment in
7559 // stubRoutines.hpp.
7560
7561 StubRoutines::_forward_exception_entry = generate_forward_exception();
7562
7563 StubRoutines::_call_stub_entry =
7564 generate_call_stub(StubRoutines::_call_stub_return_address);
7565
7566 // is referenced by megamorphic call
7567 StubRoutines::_catch_exception_entry = generate_catch_exception();
7568
7569 // atomic calls
7570 StubRoutines::_fence_entry = generate_orderaccess_fence();
7571
7572 // platform dependent
7573 StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
7574
7575 StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
7576
7577 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
7578 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
7579 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
7580 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
7581
7582 StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF);
7583 StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000);
7584 StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
7585 StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
7586
7587 // Build this early so it's available for the interpreter.
7588 StubRoutines::_throw_StackOverflowError_entry =
7589 generate_throw_exception("StackOverflowError throw_exception",
7590 CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_StackOverflowError
)))
7591 SharedRuntime::((address)((address_word)(SharedRuntime:: throw_StackOverflowError
)))
7592 throw_StackOverflowError)((address)((address_word)(SharedRuntime:: throw_StackOverflowError
))));
7593 StubRoutines::_throw_delayed_StackOverflowError_entry =
7594 generate_throw_exception("delayed StackOverflowError throw_exception",
7595 CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_delayed_StackOverflowError
)))
7596 SharedRuntime::((address)((address_word)(SharedRuntime:: throw_delayed_StackOverflowError
)))
7597 throw_delayed_StackOverflowError)((address)((address_word)(SharedRuntime:: throw_delayed_StackOverflowError
))));
7598 if (UseCRC32Intrinsics) {
7599 // set table address before stub generation which use it
7600 StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
7601 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
7602 }
7603
7604 if (UseCRC32CIntrinsics) {
7605 bool supports_clmul = VM_Version::supports_clmul();
7606 StubRoutines::x86::generate_CRC32C_table(supports_clmul);
7607 StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
7608 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
7609 }
7610
7611 if (UseAdler32Intrinsics) {
7612 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
7613 }
7614
7615 if (UseLibmIntrinsic && InlineIntrinsics) {
7616 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
7617 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
7618 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
7619 StubRoutines::x86::_ONEHALF_adr = (address)StubRoutines::x86::_ONEHALF;
7620 StubRoutines::x86::_P_2_adr = (address)StubRoutines::x86::_P_2;
7621 StubRoutines::x86::_SC_4_adr = (address)StubRoutines::x86::_SC_4;
7622 StubRoutines::x86::_Ctable_adr = (address)StubRoutines::x86::_Ctable;
7623 StubRoutines::x86::_SC_2_adr = (address)StubRoutines::x86::_SC_2;
7624 StubRoutines::x86::_SC_3_adr = (address)StubRoutines::x86::_SC_3;
7625 StubRoutines::x86::_SC_1_adr = (address)StubRoutines::x86::_SC_1;
7626 StubRoutines::x86::_PI_INV_TABLE_adr = (address)StubRoutines::x86::_PI_INV_TABLE;
7627 StubRoutines::x86::_PI_4_adr = (address)StubRoutines::x86::_PI_4;
7628 StubRoutines::x86::_PI32INV_adr = (address)StubRoutines::x86::_PI32INV;
7629 StubRoutines::x86::_SIGN_MASK_adr = (address)StubRoutines::x86::_SIGN_MASK;
7630 StubRoutines::x86::_P_1_adr = (address)StubRoutines::x86::_P_1;
7631 StubRoutines::x86::_P_3_adr = (address)StubRoutines::x86::_P_3;
7632 StubRoutines::x86::_NEG_ZERO_adr = (address)StubRoutines::x86::_NEG_ZERO;
7633 }
7634 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
7635 StubRoutines::_dexp = generate_libmExp();
7636 }
7637 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
7638 StubRoutines::_dlog = generate_libmLog();
7639 }
7640 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
7641 StubRoutines::_dlog10 = generate_libmLog10();
7642 }
7643 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
7644 StubRoutines::_dpow = generate_libmPow();
7645 }
7646 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
7647 StubRoutines::_dsin = generate_libmSin();
7648 }
7649 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
7650 StubRoutines::_dcos = generate_libmCos();
7651 }
7652 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
7653 StubRoutines::_dtan = generate_libmTan();
7654 }
7655 }
7656
7657 // Safefetch stubs.
7658 generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
7659 &StubRoutines::_safefetch32_fault_pc,
7660 &StubRoutines::_safefetch32_continuation_pc);
7661 generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
7662 &StubRoutines::_safefetchN_fault_pc,
7663 &StubRoutines::_safefetchN_continuation_pc);
7664 }
7665
7666 void generate_all() {
7667 // Generates all stubs and initializes the entry points
7668
7669 // These entry points require SharedInfo::stack0 to be set up in
7670 // non-core builds and need to be relocatable, so they each
7671 // fabricate a RuntimeStub internally.
7672 StubRoutines::_throw_AbstractMethodError_entry =
7673 generate_throw_exception("AbstractMethodError throw_exception",
7674 CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_AbstractMethodError
)))
7675 SharedRuntime::((address)((address_word)(SharedRuntime:: throw_AbstractMethodError
)))
7676 throw_AbstractMethodError)((address)((address_word)(SharedRuntime:: throw_AbstractMethodError
))));
7677
7678 StubRoutines::_throw_IncompatibleClassChangeError_entry =
7679 generate_throw_exception("IncompatibleClassChangeError throw_exception",
7680 CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_IncompatibleClassChangeError
)))
7681 SharedRuntime::((address)((address_word)(SharedRuntime:: throw_IncompatibleClassChangeError
)))
7682 throw_IncompatibleClassChangeError)((address)((address_word)(SharedRuntime:: throw_IncompatibleClassChangeError
))));
7683
7684 StubRoutines::_throw_NullPointerException_at_call_entry =
7685 generate_throw_exception("NullPointerException at call throw_exception",
7686 CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_NullPointerException_at_call
)))
7687 SharedRuntime::((address)((address_word)(SharedRuntime:: throw_NullPointerException_at_call
)))
7688 throw_NullPointerException_at_call)((address)((address_word)(SharedRuntime:: throw_NullPointerException_at_call
))));
7689
7690 // entry points that are platform specific
7691 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF);
7692 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000);
7693 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF);
7694 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000);
7695 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFFFFFFFFFF);
7696 StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask("vector_int_mask_cmp_bits", 0x0000000100000001);
7697 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff);
7698 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
7699 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff000000ff);
7700 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff0000ffff);
7701 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
7702 0xFFFFFFFF, 0, 0, 0);
7703 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
7704 0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
7705 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x0302010003020100);
7706 StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask");
7707 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x0100010001000100);
7708 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask("vector_long_shuffle_mask", 0x0000000100000000);
7709 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
7710 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
7711
7712 // support for verify_oop (must happen after universe_init)
7713 if (VerifyOops) {
7
←
Assuming 'VerifyOops' is false
→
8
←
Taking false branch
→
7714 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
7715 }
7716
7717 // data cache line writeback
7718 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
7719 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
7720
7721 // arraycopy stubs used by compilers
7722 generate_arraycopy_stubs();
7723
7724 // don't bother generating these AES intrinsic stubs unless global flag is set
7725 if (UseAESIntrinsics) {
9
←
Assuming 'UseAESIntrinsics' is false
→
10
←
Taking false branch
→
7726 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
7727 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
7728 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
7729 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
7730 if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
7731 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
7732 StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt();
7733 StubRoutines::_electronicCodeBook_decryptAESCrypt = generate_electronicCodeBook_decryptAESCrypt();
7734 StubRoutines::x86::_counter_mask_addr = counter_mask_addr();
7735 StubRoutines::x86::_ghash_poly512_addr = ghash_polynomial512_addr();
7736 StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
7737 StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
7738 } else {
7739 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
7740 }
7741 }
7742
7743 if (UseAESCTRIntrinsics) {
11
←
Assuming 'UseAESCTRIntrinsics' is false
→
12
←
Taking false branch
→
7744 if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512bw() && VM_Version::supports_avx512vl()) {
7745 if (StubRoutines::x86::_counter_mask_addr == NULL__null) {
7746 StubRoutines::x86::_counter_mask_addr = counter_mask_addr();
7747 }
7748 StubRoutines::_counterMode_AESCrypt = generate_counterMode_VectorAESCrypt();
7749 } else {
7750 StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
7751 StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
7752 }
7753 }
7754
7755 if (UseMD5Intrinsics) {
13
←
Assuming 'UseMD5Intrinsics' is false
→
14
←
Taking false branch
→
7756 StubRoutines::_md5_implCompress = generate_md5_implCompress(false, "md5_implCompress");
7757 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(true, "md5_implCompressMB");
7758 }
7759 if (UseSHA1Intrinsics) {
15
←
Assuming 'UseSHA1Intrinsics' is false
→
16
←
Taking false branch
→
7760 StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
7761 StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
7762 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
7763 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
7764 }
7765 if (UseSHA256Intrinsics) {
17
←
Assuming 'UseSHA256Intrinsics' is false
→
18
←
Taking false branch
→
7766 StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
7767 char* dst = (char*)StubRoutines::x86::_k256_W;
7768 char* src = (char*)StubRoutines::x86::_k256;
7769 for (int ii = 0; ii < 16; ++ii) {
7770 memcpy(dst + 32 * ii, src + 16 * ii, 16);
7771 memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
7772 }
7773 StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
7774 StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
7775 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
7776 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
7777 }
7778 if (UseSHA512Intrinsics) {
19
←
Assuming 'UseSHA512Intrinsics' is false
→
20
←
Taking false branch
→
7779 StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
7780 StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
7781 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
7782 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
7783 }
7784
7785 // Generate GHASH intrinsics code
7786 if (UseGHASHIntrinsics) {
21
←
Assuming 'UseGHASHIntrinsics' is false
→
22
←
Taking false branch
→
7787 if (StubRoutines::x86::_ghash_long_swap_mask_addr == NULL__null) {
7788 StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
7789 }
7790 StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
7791 if (VM_Version::supports_avx()) {
7792 StubRoutines::x86::_ghash_shuffmask_addr = ghash_shufflemask_addr();
7793 StubRoutines::x86::_ghash_poly_addr = ghash_polynomial_addr();
7794 StubRoutines::_ghash_processBlocks = generate_avx_ghash_processBlocks();
7795 } else {
7796 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
7797 }
7798 }
7799
7800
7801 if (UseBASE64Intrinsics) {
23
←
Assuming 'UseBASE64Intrinsics' is false
→
24
←
Taking false branch
→
7802 if(VM_Version::supports_avx2() &&
7803 VM_Version::supports_avx512bw() &&
7804 VM_Version::supports_avx512vl()) {
7805 StubRoutines::x86::_avx2_shuffle_base64 = base64_avx2_shuffle_addr();
7806 StubRoutines::x86::_avx2_input_mask_base64 = base64_avx2_input_mask_addr();
7807 StubRoutines::x86::_avx2_lut_base64 = base64_avx2_lut_addr();
7808 }
7809 StubRoutines::x86::_encoding_table_base64 = base64_encoding_table_addr();
7810 if (VM_Version::supports_avx512_vbmi()) {
7811 StubRoutines::x86::_shuffle_base64 = base64_shuffle_addr();
7812 StubRoutines::x86::_lookup_lo_base64 = base64_vbmi_lookup_lo_addr();
7813 StubRoutines::x86::_lookup_hi_base64 = base64_vbmi_lookup_hi_addr();
7814 StubRoutines::x86::_lookup_lo_base64url = base64_vbmi_lookup_lo_url_addr();
7815 StubRoutines::x86::_lookup_hi_base64url = base64_vbmi_lookup_hi_url_addr();
7816 StubRoutines::x86::_pack_vec_base64 = base64_vbmi_pack_vec_addr();
7817 StubRoutines::x86::_join_0_1_base64 = base64_vbmi_join_0_1_addr();
7818 StubRoutines::x86::_join_1_2_base64 = base64_vbmi_join_1_2_addr();
7819 StubRoutines::x86::_join_2_3_base64 = base64_vbmi_join_2_3_addr();
7820 }
7821 StubRoutines::x86::_decoding_table_base64 = base64_decoding_table_addr();
7822 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
7823 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
7824 }
7825
7826 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
7827 if (bs_nm != NULL__null) {
25
←
Assuming 'bs_nm' is equal to NULL
→
26
←
Taking false branch
→
7828 StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier();
7829 }
7830#ifdef COMPILER21
7831 if (UseMultiplyToLenIntrinsic) {
27
←
Assuming 'UseMultiplyToLenIntrinsic' is false
→
28
←
Taking false branch
→
7832 StubRoutines::_multiplyToLen = generate_multiplyToLen();
7833 }
7834 if (UseSquareToLenIntrinsic) {
29
←
Assuming 'UseSquareToLenIntrinsic' is false
→
30
←
Taking false branch
→
7835 StubRoutines::_squareToLen = generate_squareToLen();
7836 }
7837 if (UseMulAddIntrinsic) {
31
←
Assuming 'UseMulAddIntrinsic' is false
→
32
←
Taking false branch
→
7838 StubRoutines::_mulAdd = generate_mulAdd();
7839 }
7840 if (VM_Version::supports_avx512_vbmi2()) {
33
←
Taking true branch
→
7841 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
34
←
Calling 'StubGenerator::generate_bigIntegerRightShift'
→
7842 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift();
7843 }
7844 if (UseMontgomeryMultiplyIntrinsic) {
7845 StubRoutines::_montgomeryMultiply
7846 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply)((address)((address_word)(SharedRuntime::montgomery_multiply)
));
7847 }
7848 if (UseMontgomerySquareIntrinsic) {
7849 StubRoutines::_montgomerySquare
7850 = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square)((address)((address_word)(SharedRuntime::montgomery_square)));
7851 }
7852
7853 // Get svml stub routine addresses
7854 void *libjsvml = NULL__null;
7855 char ebuf[1024];
7856 char dll_name[JVM_MAXPATHLEN4096 + 1];
7857 if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "jsvml")) {
7858 libjsvml = os::dll_load(dll_name, ebuf, sizeof ebuf);
7859 }
7860 if (libjsvml != NULL__null) {
7861 // SVML method naming convention
7862 // All the methods are named as __jsvml_op<T><N>_ha_<VV>
7863 // Where:
7864 // ha stands for high accuracy
7865 // <T> is optional to indicate float/double
7866 // Set to f for vector float operation
7867 // Omitted for vector double operation
7868 // <N> is the number of elements in the vector
7869 // 1, 2, 4, 8, 16
7870 // e.g. 128 bit float vector has 4 float elements
7871 // <VV> indicates the avx/sse level:
7872 // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2
7873 // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns
7874 // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns
7875
7876 log_info(library)(!(LogImpl<(LogTag::_library), (LogTag::__NO_TAG), (LogTag
::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::
__NO_TAG)>::is_level(LogLevel::Info))) ? (void)0 : LogImpl
<(LogTag::_library), (LogTag::__NO_TAG), (LogTag::__NO_TAG
), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG)
>::write<LogLevel::Info>("Loaded library %s, handle " INTPTR_FORMAT"0x%016" "l" "x", JNI_LIB_PREFIX"lib" "jsvml" JNI_LIB_SUFFIX".so", p2i(libjsvml));
7877 if (UseAVX > 2) {
7878 for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) {
7879 int vop = VectorSupport::VECTOR_OP_SVML_START + op;
7880 if ((!VM_Version::supports_avx512dq()) &&
7881 (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) {
7882 continue;
7883 }
7884 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::svmlname[op]);
7885 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf);
7886
7887 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::svmlname[op]);
7888 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf);
7889 }
7890 }
7891 const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex");
7892 for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) {
7893 int vop = VectorSupport::VECTOR_OP_SVML_START + op;
7894 if (vop == VectorSupport::VECTOR_OP_POW) {
7895 continue;
7896 }
7897 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
7898 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf);
7899
7900 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
7901 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf);
7902
7903 snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
7904 StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf);
7905
7906 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
7907 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf);
7908
7909 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
7910 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf);
7911
7912 snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
7913 StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf);
7914 }
7915 }
7916#endif // COMPILER2
7917
7918 if (UseVectorizedMismatchIntrinsic) {
7919 StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch();
7920 }
7921 }
7922
7923 public:
7924 StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
7925 if (all) {
4
←
Assuming 'all' is true
→
5
←
Taking true branch
→
7926 generate_all();
6
←
Calling 'StubGenerator::generate_all'
→
7927 } else {
7928 generate_initial();
7929 }
7930 }
7931}; // end class declaration
7932
7933#define UCM_TABLE_MAX_ENTRIES16 16
7934void StubGenerator_generate(CodeBuffer* code, bool all) {
7935 if (UnsafeCopyMemory::_table == NULL__null) {
1
Assuming '_table' is not equal to NULL
→
2
←
Taking false branch
→
7936 UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES16);
7937 }
7938 StubGenerator g(code, all);
3
←
Calling constructor for 'StubGenerator'
→
7939}
←

/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/vm_version_x86.hpp

→
1/*
2 * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#ifndef CPU_X86_VM_VERSION_X86_HPP
26#define CPU_X86_VM_VERSION_X86_HPP
27
28#include "runtime/abstract_vm_version.hpp"
29#include "utilities/macros.hpp"
30#include "utilities/sizes.hpp"
31
32class VM_Version : public Abstract_VM_Version {
33 friend class VMStructs;
34 friend class JVMCIVMStructs;
35
36 public:
37 // cpuid result register layouts. These are all unions of a uint32_t
38 // (in case anyone wants access to the register as a whole) and a bitfield.
39
40 union StdCpuid1Eax {
41 uint32_t value;
42 struct {
43 uint32_t stepping : 4,
44 model : 4,
45 family : 4,
46 proc_type : 2,
47 : 2,
48 ext_model : 4,
49 ext_family : 8,
50 : 4;
51 } bits;
52 };
53
54 union StdCpuid1Ebx { // example, unused
55 uint32_t value;
56 struct {
57 uint32_t brand_id : 8,
58 clflush_size : 8,
59 threads_per_cpu : 8,
60 apic_id : 8;
61 } bits;
62 };
63
64 union StdCpuid1Ecx {
65 uint32_t value;
66 struct {
67 uint32_t sse3 : 1,
68 clmul : 1,
69 : 1,
70 monitor : 1,
71 : 1,
72 vmx : 1,
73 : 1,
74 est : 1,
75 : 1,
76 ssse3 : 1,
77 cid : 1,
78 : 1,
79 fma : 1,
80 cmpxchg16: 1,
81 : 4,
82 dca : 1,
83 sse4_1 : 1,
84 sse4_2 : 1,
85 : 2,
86 popcnt : 1,
87 : 1,
88 aes : 1,
89 : 1,
90 osxsave : 1,
91 avx : 1,
92 : 2,
93 hv : 1;
94 } bits;
95 };
96
97 union StdCpuid1Edx {
98 uint32_t value;
99 struct {
100 uint32_t : 4,
101 tsc : 1,
102 : 3,
103 cmpxchg8 : 1,
104 : 6,
105 cmov : 1,
106 : 3,
107 clflush : 1,
108 : 3,
109 mmx : 1,
110 fxsr : 1,
111 sse : 1,
112 sse2 : 1,
113 : 1,
114 ht : 1,
115 : 3;
116 } bits;
117 };
118
119 union DcpCpuid4Eax {
120 uint32_t value;
121 struct {
122 uint32_t cache_type : 5,
123 : 21,
124 cores_per_cpu : 6;
125 } bits;
126 };
127
128 union DcpCpuid4Ebx {
129 uint32_t value;
130 struct {
131 uint32_t L1_line_size : 12,
132 partitions : 10,
133 associativity : 10;
134 } bits;
135 };
136
137 union TplCpuidBEbx {
138 uint32_t value;
139 struct {
140 uint32_t logical_cpus : 16,
141 : 16;
142 } bits;
143 };
144
145 union ExtCpuid1Ecx {
146 uint32_t value;
147 struct {
148 uint32_t LahfSahf : 1,
149 CmpLegacy : 1,
150 : 3,
151 lzcnt_intel : 1,
152 lzcnt : 1,
153 sse4a : 1,
154 misalignsse : 1,
155 prefetchw : 1,
156 : 22;
157 } bits;
158 };
159
160 union ExtCpuid1Edx {
161 uint32_t value;
162 struct {
163 uint32_t : 22,
164 mmx_amd : 1,
165 mmx : 1,
166 fxsr : 1,
167 : 4,
168 long_mode : 1,
169 tdnow2 : 1,
170 tdnow : 1;
171 } bits;
172 };
173
174 union ExtCpuid5Ex {
175 uint32_t value;
176 struct {
177 uint32_t L1_line_size : 8,
178 L1_tag_lines : 8,
179 L1_assoc : 8,
180 L1_size : 8;
181 } bits;
182 };
183
184 union ExtCpuid7Edx {
185 uint32_t value;
186 struct {
187 uint32_t : 8,
188 tsc_invariance : 1,
189 : 23;
190 } bits;
191 };
192
193 union ExtCpuid8Ecx {
194 uint32_t value;
195 struct {
196 uint32_t cores_per_cpu : 8,
197 : 24;
198 } bits;
199 };
200
201 union SefCpuid7Eax {
202 uint32_t value;
203 };
204
205 union SefCpuid7Ebx {
206 uint32_t value;
207 struct {
208 uint32_t fsgsbase : 1,
209 : 2,
210 bmi1 : 1,
211 : 1,
212 avx2 : 1,
213 : 2,
214 bmi2 : 1,
215 erms : 1,
216 : 1,
217 rtm : 1,
218 : 4,
219 avx512f : 1,
220 avx512dq : 1,
221 : 1,
222 adx : 1,
223 : 3,
224 clflushopt : 1,
225 clwb : 1,
226 : 1,
227 avx512pf : 1,
228 avx512er : 1,
229 avx512cd : 1,
230 sha : 1,
231 avx512bw : 1,
232 avx512vl : 1;
233 } bits;
234 };
235
236 union SefCpuid7Ecx {
237 uint32_t value;
238 struct {
239 uint32_t prefetchwt1 : 1,
240 avx512_vbmi : 1,
241 umip : 1,
242 pku : 1,
243 ospke : 1,
244 : 1,
245 avx512_vbmi2 : 1,
246 : 1,
247 gfni : 1,
248 vaes : 1,
249 avx512_vpclmulqdq : 1,
250 avx512_vnni : 1,
251 avx512_bitalg : 1,
252 : 1,
253 avx512_vpopcntdq : 1,
254 : 17;
255 } bits;
256 };
257
258 union SefCpuid7Edx {
259 uint32_t value;
260 struct {
261 uint32_t : 2,
262 avx512_4vnniw : 1,
263 avx512_4fmaps : 1,
264 : 10,
265 serialize : 1,
266 : 17;
267 } bits;
268 };
269
270 union ExtCpuid1EEbx {
271 uint32_t value;
272 struct {
273 uint32_t : 8,
274 threads_per_core : 8,
275 : 16;
276 } bits;
277 };
278
279 union XemXcr0Eax {
280 uint32_t value;
281 struct {
282 uint32_t x87 : 1,
283 sse : 1,
284 ymm : 1,
285 bndregs : 1,
286 bndcsr : 1,
287 opmask : 1,
288 zmm512 : 1,
289 zmm32 : 1,
290 : 24;
291 } bits;
292 };
293
294protected:
295 static int _cpu;
296 static int _model;
297 static int _stepping;
298
299 static bool _has_intel_jcc_erratum;
300
301 static address _cpuinfo_segv_addr; // address of instruction which causes SEGV
302 static address _cpuinfo_cont_addr; // address of instruction after the one which causes SEGV
303
304 enum Feature_Flag : uint64_t {
305#define CPU_FEATURE_FLAGS(decl)decl(CX8, "cx8", 0) decl(CMOV, "cmov", 1) decl(FXSR, "fxsr", 2
) decl(HT, "ht", 3) decl(MMX, "mmx", 4) decl(3DNOW_PREFETCH, "3dnowpref"
, 5) decl(SSE, "sse", 6) decl(SSE2, "sse2", 7) decl(SSE3, "sse3"
, 8 ) decl(SSSE3, "ssse3", 9 ) decl(SSE4A, "sse4a", 10) decl(
SSE4_1, "sse4.1", 11) decl(SSE4_2, "sse4.2", 12) decl(POPCNT,
"popcnt", 13) decl(LZCNT, "lzcnt", 14) decl(TSC, "tsc", 15) decl
(TSCINV_BIT, "tscinvbit", 16) decl(TSCINV, "tscinv", 17) decl
(AVX, "avx", 18) decl(AVX2, "avx2", 19) decl(AES, "aes", 20) decl
(ERMS, "erms", 21) decl(CLMUL, "clmul", 22) decl(BMI1, "bmi1"
, 23) decl(BMI2, "bmi2", 24) decl(RTM, "rtm", 25) decl(ADX, "adx"
, 26) decl(AVX512F, "avx512f", 27) decl(AVX512DQ, "avx512dq",
28) decl(AVX512PF, "avx512pf", 29) decl(AVX512ER, "avx512er"
, 30) decl(AVX512CD, "avx512cd", 31) decl(AVX512BW, "avx512bw"
, 32) decl(AVX512VL, "avx512vl", 33) decl(SHA, "sha", 34) decl
(FMA, "fma", 35) decl(VZEROUPPER, "vzeroupper", 36) decl(AVX512_VPOPCNTDQ
, "avx512_vpopcntdq", 37) decl(AVX512_VPCLMULQDQ, "avx512_vpclmulqdq"
, 38) decl(AVX512_VAES, "avx512_vaes", 39) decl(AVX512_VNNI, "avx512_vnni"
, 40) decl(FLUSH, "clflush", 41) decl(FLUSHOPT, "clflushopt",
42) decl(CLWB, "clwb", 43) decl(AVX512_VBMI2, "avx512_vbmi2"
, 44) decl(AVX512_VBMI, "avx512_vbmi", 45) decl(HV, "hv", 46)
decl(SERIALIZE, "serialize", 47) \
306 decl(CX8, "cx8", 0) /* next bits are from cpuid 1 (EDX) */ \
307 decl(CMOV, "cmov", 1) \
308 decl(FXSR, "fxsr", 2) \
309 decl(HT, "ht", 3) \
310 \
311 decl(MMX, "mmx", 4) \
312 decl(3DNOW_PREFETCH, "3dnowpref", 5) /* Processor supports 3dnow prefetch and prefetchw instructions */ \
313 /* may not necessarily support other 3dnow instructions */ \
314 decl(SSE, "sse", 6) \
315 decl(SSE2, "sse2", 7) \
316 \
317 decl(SSE3, "sse3", 8 ) /* SSE3 comes from cpuid 1 (ECX) */ \
318 decl(SSSE3, "ssse3", 9 ) \
319 decl(SSE4A, "sse4a", 10) \
320 decl(SSE4_1, "sse4.1", 11) \
321 \
322 decl(SSE4_2, "sse4.2", 12) \
323 decl(POPCNT, "popcnt", 13) \
324 decl(LZCNT, "lzcnt", 14) \
325 decl(TSC, "tsc", 15) \
326 \
327 decl(TSCINV_BIT, "tscinvbit", 16) \
328 decl(TSCINV, "tscinv", 17) \
329 decl(AVX, "avx", 18) \
330 decl(AVX2, "avx2", 19) \
331 \
332 decl(AES, "aes", 20) \
333 decl(ERMS, "erms", 21) /* enhanced 'rep movsb/stosb' instructions */ \
334 decl(CLMUL, "clmul", 22) /* carryless multiply for CRC */ \
335 decl(BMI1, "bmi1", 23) \
336 \
337 decl(BMI2, "bmi2", 24) \
338 decl(RTM, "rtm", 25) /* Restricted Transactional Memory instructions */ \
339 decl(ADX, "adx", 26) \
340 decl(AVX512F, "avx512f", 27) /* AVX 512bit foundation instructions */ \
341 \
342 decl(AVX512DQ, "avx512dq", 28) \
343 decl(AVX512PF, "avx512pf", 29) \
344 decl(AVX512ER, "avx512er", 30) \
345 decl(AVX512CD, "avx512cd", 31) \
346 \
347 decl(AVX512BW, "avx512bw", 32) /* Byte and word vector instructions */ \
348 decl(AVX512VL, "avx512vl", 33) /* EVEX instructions with smaller vector length */ \
349 decl(SHA, "sha", 34) /* SHA instructions */ \
350 decl(FMA, "fma", 35) /* FMA instructions */ \
351 \
352 decl(VZEROUPPER, "vzeroupper", 36) /* Vzeroupper instruction */ \
353 decl(AVX512_VPOPCNTDQ, "avx512_vpopcntdq", 37) /* Vector popcount */ \
354 decl(AVX512_VPCLMULQDQ, "avx512_vpclmulqdq", 38) /* Vector carryless multiplication */ \
355 decl(AVX512_VAES, "avx512_vaes", 39) /* Vector AES instruction */ \
356 \
357 decl(AVX512_VNNI, "avx512_vnni", 40) /* Vector Neural Network Instructions */ \
358 decl(FLUSH, "clflush", 41) /* flush instruction */ \
359 decl(FLUSHOPT, "clflushopt", 42) /* flusopth instruction */ \
360 decl(CLWB, "clwb", 43) /* clwb instruction */ \
361 \
362 decl(AVX512_VBMI2, "avx512_vbmi2", 44) /* VBMI2 shift left double instructions */ \
363 decl(AVX512_VBMI, "avx512_vbmi", 45) /* Vector BMI instructions */ \
364 decl(HV, "hv", 46) /* Hypervisor instructions */ \
365 decl(SERIALIZE, "serialize", 47) /* CPU SERIALIZE */
366
367#define DECLARE_CPU_FEATURE_FLAG(id, name, bit) CPU_##id = (1ULL << bit),
368 CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG)DECLARE_CPU_FEATURE_FLAG(CX8, "cx8", 0) DECLARE_CPU_FEATURE_FLAG
(CMOV, "cmov", 1) DECLARE_CPU_FEATURE_FLAG(FXSR, "fxsr", 2) DECLARE_CPU_FEATURE_FLAG
(HT, "ht", 3) DECLARE_CPU_FEATURE_FLAG(MMX, "mmx", 4) DECLARE_CPU_FEATURE_FLAG
(3DNOW_PREFETCH, "3dnowpref", 5) DECLARE_CPU_FEATURE_FLAG(SSE
, "sse", 6) DECLARE_CPU_FEATURE_FLAG(SSE2, "sse2", 7) DECLARE_CPU_FEATURE_FLAG
(SSE3, "sse3", 8 ) DECLARE_CPU_FEATURE_FLAG(SSSE3, "ssse3", 9
) DECLARE_CPU_FEATURE_FLAG(SSE4A, "sse4a", 10) DECLARE_CPU_FEATURE_FLAG
(SSE4_1, "sse4.1", 11) DECLARE_CPU_FEATURE_FLAG(SSE4_2, "sse4.2"
, 12) DECLARE_CPU_FEATURE_FLAG(POPCNT, "popcnt", 13) DECLARE_CPU_FEATURE_FLAG
(LZCNT, "lzcnt", 14) DECLARE_CPU_FEATURE_FLAG(TSC, "tsc", 15)
DECLARE_CPU_FEATURE_FLAG(TSCINV_BIT, "tscinvbit", 16) DECLARE_CPU_FEATURE_FLAG
(TSCINV, "tscinv", 17) DECLARE_CPU_FEATURE_FLAG(AVX, "avx", 18
) DECLARE_CPU_FEATURE_FLAG(AVX2, "avx2", 19) DECLARE_CPU_FEATURE_FLAG
(AES, "aes", 20) DECLARE_CPU_FEATURE_FLAG(ERMS, "erms", 21) DECLARE_CPU_FEATURE_FLAG
(CLMUL, "clmul", 22) DECLARE_CPU_FEATURE_FLAG(BMI1, "bmi1", 23
) DECLARE_CPU_FEATURE_FLAG(BMI2, "bmi2", 24) DECLARE_CPU_FEATURE_FLAG
(RTM, "rtm", 25) DECLARE_CPU_FEATURE_FLAG(ADX, "adx", 26) DECLARE_CPU_FEATURE_FLAG
(AVX512F, "avx512f", 27) DECLARE_CPU_FEATURE_FLAG(AVX512DQ, "avx512dq"
, 28) DECLARE_CPU_FEATURE_FLAG(AVX512PF, "avx512pf", 29) DECLARE_CPU_FEATURE_FLAG
(AVX512ER, "avx512er", 30) DECLARE_CPU_FEATURE_FLAG(AVX512CD,
"avx512cd", 31) DECLARE_CPU_FEATURE_FLAG(AVX512BW, "avx512bw"
, 32) DECLARE_CPU_FEATURE_FLAG(AVX512VL, "avx512vl", 33) DECLARE_CPU_FEATURE_FLAG
(SHA, "sha", 34) DECLARE_CPU_FEATURE_FLAG(FMA, "fma", 35) DECLARE_CPU_FEATURE_FLAG
(VZEROUPPER, "vzeroupper", 36) DECLARE_CPU_FEATURE_FLAG(AVX512_VPOPCNTDQ
, "avx512_vpopcntdq", 37) DECLARE_CPU_FEATURE_FLAG(AVX512_VPCLMULQDQ
, "avx512_vpclmulqdq", 38) DECLARE_CPU_FEATURE_FLAG(AVX512_VAES
, "avx512_vaes", 39) DECLARE_CPU_FEATURE_FLAG(AVX512_VNNI, "avx512_vnni"
, 40) DECLARE_CPU_FEATURE_FLAG(FLUSH, "clflush", 41) DECLARE_CPU_FEATURE_FLAG
(FLUSHOPT, "clflushopt", 42) DECLARE_CPU_FEATURE_FLAG(CLWB, "clwb"
, 43) DECLARE_CPU_FEATURE_FLAG(AVX512_VBMI2, "avx512_vbmi2", 44
) DECLARE_CPU_FEATURE_FLAG(AVX512_VBMI, "avx512_vbmi", 45) DECLARE_CPU_FEATURE_FLAG
(HV, "hv", 46) DECLARE_CPU_FEATURE_FLAG(SERIALIZE, "serialize"
, 47)
369#undef DECLARE_CPU_FEATURE_FLAG
370 };
371
372 static const char* _features_names[];
373
374 enum Extended_Family {
375 // AMD
376 CPU_FAMILY_AMD_11H = 0x11,
377 // ZX
378 CPU_FAMILY_ZX_CORE_F6 = 6,
379 CPU_FAMILY_ZX_CORE_F7 = 7,
380 // Intel
381 CPU_FAMILY_INTEL_CORE = 6,
382 CPU_MODEL_NEHALEM = 0x1e,
383 CPU_MODEL_NEHALEM_EP = 0x1a,
384 CPU_MODEL_NEHALEM_EX = 0x2e,
385 CPU_MODEL_WESTMERE = 0x25,
386 CPU_MODEL_WESTMERE_EP = 0x2c,
387 CPU_MODEL_WESTMERE_EX = 0x2f,
388 CPU_MODEL_SANDYBRIDGE = 0x2a,
389 CPU_MODEL_SANDYBRIDGE_EP = 0x2d,
390 CPU_MODEL_IVYBRIDGE_EP = 0x3a,
391 CPU_MODEL_HASWELL_E3 = 0x3c,
392 CPU_MODEL_HASWELL_E7 = 0x3f,
393 CPU_MODEL_BROADWELL = 0x3d,
394 CPU_MODEL_SKYLAKE = 0x55
395 };
396
397 // cpuid information block. All info derived from executing cpuid with
398 // various function numbers is stored here. Intel and AMD info is
399 // merged in this block: accessor methods disentangle it.
400 //
401 // The info block is laid out in subblocks of 4 dwords corresponding to
402 // eax, ebx, ecx and edx, whether or not they contain anything useful.
403 struct CpuidInfo {
404 // cpuid function 0
405 uint32_t std_max_function;
406 uint32_t std_vendor_name_0;
407 uint32_t std_vendor_name_1;
408 uint32_t std_vendor_name_2;
409
410 // cpuid function 1
411 StdCpuid1Eax std_cpuid1_eax;
412 StdCpuid1Ebx std_cpuid1_ebx;
413 StdCpuid1Ecx std_cpuid1_ecx;
414 StdCpuid1Edx std_cpuid1_edx;
415
416 // cpuid function 4 (deterministic cache parameters)
417 DcpCpuid4Eax dcp_cpuid4_eax;
418 DcpCpuid4Ebx dcp_cpuid4_ebx;
419 uint32_t dcp_cpuid4_ecx; // unused currently
420 uint32_t dcp_cpuid4_edx; // unused currently
421
422 // cpuid function 7 (structured extended features)
423 SefCpuid7Eax sef_cpuid7_eax;
424 SefCpuid7Ebx sef_cpuid7_ebx;
425 SefCpuid7Ecx sef_cpuid7_ecx;
426 SefCpuid7Edx sef_cpuid7_edx;
427
428 // cpuid function 0xB (processor topology)
429 // ecx = 0
430 uint32_t tpl_cpuidB0_eax;
431 TplCpuidBEbx tpl_cpuidB0_ebx;
432 uint32_t tpl_cpuidB0_ecx; // unused currently
433 uint32_t tpl_cpuidB0_edx; // unused currently
434
435 // ecx = 1
436 uint32_t tpl_cpuidB1_eax;
437 TplCpuidBEbx tpl_cpuidB1_ebx;
438 uint32_t tpl_cpuidB1_ecx; // unused currently
439 uint32_t tpl_cpuidB1_edx; // unused currently
440
441 // ecx = 2
442 uint32_t tpl_cpuidB2_eax;
443 TplCpuidBEbx tpl_cpuidB2_ebx;
444 uint32_t tpl_cpuidB2_ecx; // unused currently
445 uint32_t tpl_cpuidB2_edx; // unused currently
446
447 // cpuid function 0x80000000 // example, unused
448 uint32_t ext_max_function;
449 uint32_t ext_vendor_name_0;
450 uint32_t ext_vendor_name_1;
451 uint32_t ext_vendor_name_2;
452
453 // cpuid function 0x80000001
454 uint32_t ext_cpuid1_eax; // reserved
455 uint32_t ext_cpuid1_ebx; // reserved
456 ExtCpuid1Ecx ext_cpuid1_ecx;
457 ExtCpuid1Edx ext_cpuid1_edx;
458
459 // cpuid functions 0x80000002 thru 0x80000004: example, unused
460 uint32_t proc_name_0, proc_name_1, proc_name_2, proc_name_3;
461 uint32_t proc_name_4, proc_name_5, proc_name_6, proc_name_7;
462 uint32_t proc_name_8, proc_name_9, proc_name_10,proc_name_11;
463
464 // cpuid function 0x80000005 // AMD L1, Intel reserved
465 uint32_t ext_cpuid5_eax; // unused currently
466 uint32_t ext_cpuid5_ebx; // reserved
467 ExtCpuid5Ex ext_cpuid5_ecx; // L1 data cache info (AMD)
468 ExtCpuid5Ex ext_cpuid5_edx; // L1 instruction cache info (AMD)
469
470 // cpuid function 0x80000007
471 uint32_t ext_cpuid7_eax; // reserved
472 uint32_t ext_cpuid7_ebx; // reserved
473 uint32_t ext_cpuid7_ecx; // reserved
474 ExtCpuid7Edx ext_cpuid7_edx; // tscinv
475
476 // cpuid function 0x80000008
477 uint32_t ext_cpuid8_eax; // unused currently
478 uint32_t ext_cpuid8_ebx; // reserved
479 ExtCpuid8Ecx ext_cpuid8_ecx;
480 uint32_t ext_cpuid8_edx; // reserved
481
482 // cpuid function 0x8000001E // AMD 17h
483 uint32_t ext_cpuid1E_eax;
484 ExtCpuid1EEbx ext_cpuid1E_ebx; // threads per core (AMD17h)
485 uint32_t ext_cpuid1E_ecx;
486 uint32_t ext_cpuid1E_edx; // unused currently
487
488 // extended control register XCR0 (the XFEATURE_ENABLED_MASK register)
489 XemXcr0Eax xem_xcr0_eax;
490 uint32_t xem_xcr0_edx; // reserved
491
492 // Space to save ymm registers after signal handle
493 int ymm_save[8*4]; // Save ymm0, ymm7, ymm8, ymm15
494
495 // Space to save zmm registers after signal handle
496 int zmm_save[16*4]; // Save zmm0, zmm7, zmm8, zmm31
497 };
498
499 // The actual cpuid info block
500 static CpuidInfo _cpuid_info;
501
502 // Extractors and predicates
503 static uint32_t extended_cpu_family() {
504 uint32_t result = _cpuid_info.std_cpuid1_eax.bits.family;
505 result += _cpuid_info.std_cpuid1_eax.bits.ext_family;
506 return result;
507 }
508
509 static uint32_t extended_cpu_model() {
510 uint32_t result = _cpuid_info.std_cpuid1_eax.bits.model;
511 result |= _cpuid_info.std_cpuid1_eax.bits.ext_model << 4;
512 return result;
513 }
514
515 static uint32_t cpu_stepping() {
516 uint32_t result = _cpuid_info.std_cpuid1_eax.bits.stepping;
517 return result;
518 }
519
520 static uint logical_processor_count() {
521 uint result = threads_per_core();
522 return result;
523 }
524
525 static bool compute_has_intel_jcc_erratum();
526
527 static uint64_t feature_flags() {
528 uint64_t result = 0;
529 if (_cpuid_info.std_cpuid1_edx.bits.cmpxchg8 != 0)
530 result |= CPU_CX8;
531 if (_cpuid_info.std_cpuid1_edx.bits.cmov != 0)
532 result |= CPU_CMOV;
533 if (_cpuid_info.std_cpuid1_edx.bits.clflush != 0)
534 result |= CPU_FLUSH;
535#ifdef _LP641
536 // clflush should always be available on x86_64
537 // if not we are in real trouble because we rely on it
538 // to flush the code cache.
539 assert ((result & CPU_FLUSH) != 0, "clflush should be available")do { if (!((result & CPU_FLUSH) != 0)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/vm_version_x86.hpp"
, 539, "assert(" "(result & CPU_FLUSH) != 0" ") failed", "clflush should be available"
); ::breakpoint(); } } while (0);
540#endif
541 if (_cpuid_info.std_cpuid1_edx.bits.fxsr != 0 || (is_amd_family() &&
542 _cpuid_info.ext_cpuid1_edx.bits.fxsr != 0))
543 result |= CPU_FXSR;
544 // HT flag is set for multi-core processors also.
545 if (threads_per_core() > 1)
546 result |= CPU_HT;
547 if (_cpuid_info.std_cpuid1_edx.bits.mmx != 0 || (is_amd_family() &&
548 _cpuid_info.ext_cpuid1_edx.bits.mmx != 0))
549 result |= CPU_MMX;
550 if (_cpuid_info.std_cpuid1_edx.bits.sse != 0)
551 result |= CPU_SSE;
552 if (_cpuid_info.std_cpuid1_edx.bits.sse2 != 0)
553 result |= CPU_SSE2;
554 if (_cpuid_info.std_cpuid1_ecx.bits.sse3 != 0)
555 result |= CPU_SSE3;
556 if (_cpuid_info.std_cpuid1_ecx.bits.ssse3 != 0)
557 result |= CPU_SSSE3;
558 if (_cpuid_info.std_cpuid1_ecx.bits.sse4_1 != 0)
559 result |= CPU_SSE4_1;
560 if (_cpuid_info.std_cpuid1_ecx.bits.sse4_2 != 0)
561 result |= CPU_SSE4_2;
562 if (_cpuid_info.std_cpuid1_ecx.bits.popcnt != 0)
563 result |= CPU_POPCNT;
564 if (_cpuid_info.std_cpuid1_ecx.bits.avx != 0 &&
565 _cpuid_info.std_cpuid1_ecx.bits.osxsave != 0 &&
566 _cpuid_info.xem_xcr0_eax.bits.sse != 0 &&
567 _cpuid_info.xem_xcr0_eax.bits.ymm != 0) {
568 result |= CPU_AVX;
569 result |= CPU_VZEROUPPER;
570 if (_cpuid_info.sef_cpuid7_ebx.bits.avx2 != 0)
571 result |= CPU_AVX2;
572 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512f != 0 &&
573 _cpuid_info.xem_xcr0_eax.bits.opmask != 0 &&
574 _cpuid_info.xem_xcr0_eax.bits.zmm512 != 0 &&
575 _cpuid_info.xem_xcr0_eax.bits.zmm32 != 0) {
576 result |= CPU_AVX512F;
577 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512cd != 0)
578 result |= CPU_AVX512CD;
579 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512dq != 0)
580 result |= CPU_AVX512DQ;
581 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512pf != 0)
582 result |= CPU_AVX512PF;
583 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512er != 0)
584 result |= CPU_AVX512ER;
585 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512bw != 0)
586 result |= CPU_AVX512BW;
587 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512vl != 0)
588 result |= CPU_AVX512VL;
589 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
590 result |= CPU_AVX512_VPOPCNTDQ;
591 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
592 result |= CPU_AVX512_VPCLMULQDQ;
593 if (_cpuid_info.sef_cpuid7_ecx.bits.vaes != 0)
594 result |= CPU_AVX512_VAES;
595 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vnni != 0)
596 result |= CPU_AVX512_VNNI;
597 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi != 0)
598 result |= CPU_AVX512_VBMI;
599 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi2 != 0)
600 result |= CPU_AVX512_VBMI2;
601 }
602 }
603 if (_cpuid_info.std_cpuid1_ecx.bits.hv != 0)
604 result |= CPU_HV;
605 if (_cpuid_info.sef_cpuid7_ebx.bits.bmi1 != 0)
606 result |= CPU_BMI1;
607 if (_cpuid_info.std_cpuid1_edx.bits.tsc != 0)
608 result |= CPU_TSC;
609 if (_cpuid_info.ext_cpuid7_edx.bits.tsc_invariance != 0)
610 result |= CPU_TSCINV_BIT;
611 if (_cpuid_info.std_cpuid1_ecx.bits.aes != 0)
612 result |= CPU_AES;
613 if (_cpuid_info.sef_cpuid7_ebx.bits.erms != 0)
614 result |= CPU_ERMS;
615 if (_cpuid_info.std_cpuid1_ecx.bits.clmul != 0)
616 result |= CPU_CLMUL;
617 if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
618 result |= CPU_RTM;
619 if (_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
620 result |= CPU_ADX;
621 if (_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
622 result |= CPU_BMI2;
623 if (_cpuid_info.sef_cpuid7_ebx.bits.sha != 0)
624 result |= CPU_SHA;
625 if (_cpuid_info.std_cpuid1_ecx.bits.fma != 0)
626 result |= CPU_FMA;
627 if (_cpuid_info.sef_cpuid7_ebx.bits.clflushopt != 0)
628 result |= CPU_FLUSHOPT;
629
630 // AMD|Hygon features.
631 if (is_amd_family()) {
632 if ((_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0) ||
633 (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0))
634 result |= CPU_3DNOW_PREFETCH;
635 if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0)
636 result |= CPU_LZCNT;
637 if (_cpuid_info.ext_cpuid1_ecx.bits.sse4a != 0)
638 result |= CPU_SSE4A;
639 }
640
641 // Intel features.
642 if (is_intel()) {
643 if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt_intel != 0)
644 result |= CPU_LZCNT;
645 // for Intel, ecx.bits.misalignsse bit (bit 8) indicates support for prefetchw
646 if (_cpuid_info.ext_cpuid1_ecx.bits.misalignsse != 0) {
647 result |= CPU_3DNOW_PREFETCH;
648 }
649 if (_cpuid_info.sef_cpuid7_ebx.bits.clwb != 0) {
650 result |= CPU_CLWB;
651 }
652 if (_cpuid_info.sef_cpuid7_edx.bits.serialize != 0)
653 result |= CPU_SERIALIZE;
654 }
655
656 // ZX features.
657 if (is_zx()) {
658 if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt_intel != 0)
659 result |= CPU_LZCNT;
660 // for ZX, ecx.bits.misalignsse bit (bit 8) indicates support for prefetchw
661 if (_cpuid_info.ext_cpuid1_ecx.bits.misalignsse != 0) {
662 result |= CPU_3DNOW_PREFETCH;
663 }
664 }
665
666 // Composite features.
667 if (supports_tscinv_bit() &&
668 ((is_amd_family() && !is_amd_Barcelona()) ||
669 is_intel_tsc_synched_at_init())) {
670 result |= CPU_TSCINV;
671 }
672
673 return result;
674 }
675
676 static bool os_supports_avx_vectors() {
677 bool retVal = false;
678 int nreg = 2 LP64_ONLY(+2)+2;
679 if (supports_evex()) {
680 // Verify that OS save/restore all bits of EVEX registers
681 // during signal processing.
682 retVal = true;
683 for (int i = 0; i < 16 * nreg; i++) { // 64 bytes per zmm register
684 if (_cpuid_info.zmm_save[i] != ymm_test_value()) {
685 retVal = false;
686 break;
687 }
688 }
689 } else if (supports_avx()) {
690 // Verify that OS save/restore all bits of AVX registers
691 // during signal processing.
692 retVal = true;
693 for (int i = 0; i < 8 * nreg; i++) { // 32 bytes per ymm register
694 if (_cpuid_info.ymm_save[i] != ymm_test_value()) {
695 retVal = false;
696 break;
697 }
698 }
699 // zmm_save will be set on a EVEX enabled machine even if we choose AVX code gen
700 if (retVal == false) {
701 // Verify that OS save/restore all bits of EVEX registers
702 // during signal processing.
703 retVal = true;
704 for (int i = 0; i < 16 * nreg; i++) { // 64 bytes per zmm register
705 if (_cpuid_info.zmm_save[i] != ymm_test_value()) {
706 retVal = false;
707 break;
708 }
709 }
710 }
711 }
712 return retVal;
713 }
714
715 static void get_processor_features();
716
717public:
718 // Offsets for cpuid asm stub
719 static ByteSize std_cpuid0_offset() { return byte_offset_of(CpuidInfo, std_max_function)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->std_max_function
) - 16)); }
720 static ByteSize std_cpuid1_offset() { return byte_offset_of(CpuidInfo, std_cpuid1_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->std_cpuid1_eax
) - 16)); }
721 static ByteSize dcp_cpuid4_offset() { return byte_offset_of(CpuidInfo, dcp_cpuid4_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->dcp_cpuid4_eax
) - 16)); }
722 static ByteSize sef_cpuid7_offset() { return byte_offset_of(CpuidInfo, sef_cpuid7_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->sef_cpuid7_eax
) - 16)); }
723 static ByteSize ext_cpuid1_offset() { return byte_offset_of(CpuidInfo, ext_cpuid1_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->ext_cpuid1_eax
) - 16)); }
724 static ByteSize ext_cpuid5_offset() { return byte_offset_of(CpuidInfo, ext_cpuid5_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->ext_cpuid5_eax
) - 16)); }
725 static ByteSize ext_cpuid7_offset() { return byte_offset_of(CpuidInfo, ext_cpuid7_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->ext_cpuid7_eax
) - 16)); }
726 static ByteSize ext_cpuid8_offset() { return byte_offset_of(CpuidInfo, ext_cpuid8_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->ext_cpuid8_eax
) - 16)); }
727 static ByteSize ext_cpuid1E_offset() { return byte_offset_of(CpuidInfo, ext_cpuid1E_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->ext_cpuid1E_eax
) - 16)); }
728 static ByteSize tpl_cpuidB0_offset() { return byte_offset_of(CpuidInfo, tpl_cpuidB0_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->tpl_cpuidB0_eax
) - 16)); }
729 static ByteSize tpl_cpuidB1_offset() { return byte_offset_of(CpuidInfo, tpl_cpuidB1_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->tpl_cpuidB1_eax
) - 16)); }
730 static ByteSize tpl_cpuidB2_offset() { return byte_offset_of(CpuidInfo, tpl_cpuidB2_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->tpl_cpuidB2_eax
) - 16)); }
731 static ByteSize xem_xcr0_offset() { return byte_offset_of(CpuidInfo, xem_xcr0_eax)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->xem_xcr0_eax
) - 16)); }
732 static ByteSize ymm_save_offset() { return byte_offset_of(CpuidInfo, ymm_save)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->ymm_save
) - 16)); }
733 static ByteSize zmm_save_offset() { return byte_offset_of(CpuidInfo, zmm_save)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->zmm_save
) - 16)); }
734
735 // The value used to check ymm register after signal handle
736 static int ymm_test_value() { return 0xCAFEBABE; }
737
738 static void get_cpu_info_wrapper();
739 static void set_cpuinfo_segv_addr(address pc) { _cpuinfo_segv_addr = pc; }
740 static bool is_cpuinfo_segv_addr(address pc) { return _cpuinfo_segv_addr == pc; }
741 static void set_cpuinfo_cont_addr(address pc) { _cpuinfo_cont_addr = pc; }
742 static address cpuinfo_cont_addr() { return _cpuinfo_cont_addr; }
743
744 static void clean_cpuFeatures() { _features = 0; }
745 static void set_avx_cpuFeatures() { _features = (CPU_SSE | CPU_SSE2 | CPU_AVX | CPU_VZEROUPPER ); }
746 static void set_evex_cpuFeatures() { _features = (CPU_AVX512F | CPU_SSE | CPU_SSE2 | CPU_VZEROUPPER ); }
747
748
749 // Initialization
750 static void initialize();
751
752 // Override Abstract_VM_Version implementation
753 static void print_platform_virtualization_info(outputStream*);
754
755 // Asserts
756 static void assert_is_initialized() {
757 assert(_cpuid_info.std_cpuid1_eax.bits.family != 0, "VM_Version not initialized")do { if (!(_cpuid_info.std_cpuid1_eax.bits.family != 0)) { (*
g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/vm_version_x86.hpp"
, 757, "assert(" "_cpuid_info.std_cpuid1_eax.bits.family != 0"
") failed", "VM_Version not initialized"); ::breakpoint(); }
} while (0);
758 }
759
760 //
761 // Processor family:
762 // 3 - 386
763 // 4 - 486
764 // 5 - Pentium
765 // 6 - PentiumPro, Pentium II, Celeron, Xeon, Pentium III, Athlon,
766 // Pentium M, Core Solo, Core Duo, Core2 Duo
767 // family 6 model: 9, 13, 14, 15
768 // 0x0f - Pentium 4, Opteron
769 //
770 // Note: The cpu family should be used to select between
771 // instruction sequences which are valid on all Intel
772 // processors. Use the feature test functions below to
773 // determine whether a particular instruction is supported.
774 //
775 static int cpu_family() { return _cpu;}
776 static bool is_P6() { return cpu_family() >= 6; }
777 static bool is_amd() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x68747541; } // 'htuA'
778 static bool is_hygon() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x6F677948; } // 'ogyH'
779 static bool is_amd_family() { return is_amd() || is_hygon(); }
780 static bool is_intel() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x756e6547; } // 'uneG'
781 static bool is_zx() { assert_is_initialized(); return (_cpuid_info.std_vendor_name_0 == 0x746e6543) || (_cpuid_info.std_vendor_name_0 == 0x68532020); } // 'tneC'||'hS '
782 static bool is_atom_family() { return ((cpu_family() == 0x06) && ((extended_cpu_model() == 0x36) || (extended_cpu_model() == 0x37) || (extended_cpu_model() == 0x4D))); } //Silvermont and Centerton
783 static bool is_knights_family() { return UseKNLSetting || ((cpu_family() == 0x06) && ((extended_cpu_model() == 0x57) || (extended_cpu_model() == 0x85))); } // Xeon Phi 3200/5200/7200 and Future Xeon Phi
784
785 static bool supports_processor_topology() {
786 return (_cpuid_info.std_max_function >= 0xB) &&
787 // eax[4:0] | ebx[0:15] == 0 indicates invalid topology level.
788 // Some cpus have max cpuid >= 0xB but do not support processor topology.
789 (((_cpuid_info.tpl_cpuidB0_eax & 0x1f) | _cpuid_info.tpl_cpuidB0_ebx.bits.logical_cpus) != 0);
790 }
791
792 static uint cores_per_cpu() {
793 uint result = 1;
794 if (is_intel()) {
795 bool supports_topology = supports_processor_topology();
796 if (supports_topology) {
797 result = _cpuid_info.tpl_cpuidB1_ebx.bits.logical_cpus /
798 _cpuid_info.tpl_cpuidB0_ebx.bits.logical_cpus;
799 }
800 if (!supports_topology || result == 0) {
801 result = (_cpuid_info.dcp_cpuid4_eax.bits.cores_per_cpu + 1);
802 }
803 } else if (is_amd_family()) {
804 result = (_cpuid_info.ext_cpuid8_ecx.bits.cores_per_cpu + 1);
805 } else if (is_zx()) {
806 bool supports_topology = supports_processor_topology();
807 if (supports_topology) {
808 result = _cpuid_info.tpl_cpuidB1_ebx.bits.logical_cpus /
809 _cpuid_info.tpl_cpuidB0_ebx.bits.logical_cpus;
810 }
811 if (!supports_topology || result == 0) {
812 result = (_cpuid_info.dcp_cpuid4_eax.bits.cores_per_cpu + 1);
813 }
814 }
815 return result;
816 }
817
818 static uint threads_per_core() {
819 uint result = 1;
820 if (is_intel() && supports_processor_topology()) {
821 result = _cpuid_info.tpl_cpuidB0_ebx.bits.logical_cpus;
822 } else if (is_zx() && supports_processor_topology()) {
823 result = _cpuid_info.tpl_cpuidB0_ebx.bits.logical_cpus;
824 } else if (_cpuid_info.std_cpuid1_edx.bits.ht != 0) {
825 if (cpu_family() >= 0x17) {
826 result = _cpuid_info.ext_cpuid1E_ebx.bits.threads_per_core + 1;
827 } else {
828 result = _cpuid_info.std_cpuid1_ebx.bits.threads_per_cpu /
829 cores_per_cpu();
830 }
831 }
832 return (result == 0 ? 1 : result);
833 }
834
835 static intx L1_line_size() {
836 intx result = 0;
837 if (is_intel()) {
838 result = (_cpuid_info.dcp_cpuid4_ebx.bits.L1_line_size + 1);
839 } else if (is_amd_family()) {
840 result = _cpuid_info.ext_cpuid5_ecx.bits.L1_line_size;
841 } else if (is_zx()) {
842 result = (_cpuid_info.dcp_cpuid4_ebx.bits.L1_line_size + 1);
843 }
844 if (result < 32) // not defined ?
845 result = 32; // 32 bytes by default on x86 and other x64
846 return result;
847 }
848
849 static intx prefetch_data_size() {
850 return L1_line_size();
851 }
852
853 //
854 // Feature identification
855 //
856 static bool supports_cpuid() { return _features != 0; }
857 static bool supports_cmpxchg8() { return (_features & CPU_CX8) != 0; }
858 static bool supports_cmov() { return (_features & CPU_CMOV) != 0; }
859 static bool supports_fxsr() { return (_features & CPU_FXSR) != 0; }
860 static bool supports_ht() { return (_features & CPU_HT) != 0; }
861 static bool supports_mmx() { return (_features & CPU_MMX) != 0; }
862 static bool supports_sse() { return (_features & CPU_SSE) != 0; }
863 static bool supports_sse2() { return (_features & CPU_SSE2) != 0; }
864 static bool supports_sse3() { return (_features & CPU_SSE3) != 0; }
865 static bool supports_ssse3() { return (_features & CPU_SSSE3)!= 0; }
866 static bool supports_sse4_1() { return (_features & CPU_SSE4_1) != 0; }
867 static bool supports_sse4_2() { return (_features & CPU_SSE4_2) != 0; }
868 static bool supports_popcnt() { return (_features & CPU_POPCNT) != 0; }
869 static bool supports_avx() { return (_features & CPU_AVX) != 0; }
870 static bool supports_avx2() { return (_features & CPU_AVX2) != 0; }
871 static bool supports_tsc() { return (_features & CPU_TSC) != 0; }
872 static bool supports_aes() { return (_features & CPU_AES) != 0; }
873 static bool supports_erms() { return (_features & CPU_ERMS) != 0; }
874 static bool supports_clmul() { return (_features & CPU_CLMUL) != 0; }
875 static bool supports_rtm() { return (_features & CPU_RTM) != 0; }
876 static bool supports_bmi1() { return (_features & CPU_BMI1) != 0; }
877 static bool supports_bmi2() { return (_features & CPU_BMI2) != 0; }
878 static bool supports_adx() { return (_features & CPU_ADX) != 0; }
879 static bool supports_evex() { return (_features & CPU_AVX512F) != 0; }
880 static bool supports_avx512dq() { return (_features & CPU_AVX512DQ) != 0; }
881 static bool supports_avx512pf() { return (_features & CPU_AVX512PF) != 0; }
882 static bool supports_avx512er() { return (_features & CPU_AVX512ER) != 0; }
883 static bool supports_avx512cd() { return (_features & CPU_AVX512CD) != 0; }
884 static bool supports_avx512bw() { return (_features & CPU_AVX512BW) != 0; }
885 static bool supports_avx512vl() { return (_features & CPU_AVX512VL) != 0; }
886 static bool supports_avx512vlbw() { return (supports_evex() && supports_avx512bw() && supports_avx512vl()); }
887 static bool supports_avx512bwdq() { return (supports_evex() && supports_avx512bw() && supports_avx512dq()); }
888 static bool supports_avx512vldq() { return (supports_evex() && supports_avx512dq() && supports_avx512vl()); }
889 static bool supports_avx512vlbwdq() { return (supports_evex() && supports_avx512vl() &&
890 supports_avx512bw() && supports_avx512dq()); }
891 static bool supports_avx512novl() { return (supports_evex() && !supports_avx512vl()); }
892 static bool supports_avx512nobw() { return (supports_evex() && !supports_avx512bw()); }
893 static bool supports_avx256only() { return (supports_avx2() && !supports_evex()); }
894 static bool supports_avxonly() { return ((supports_avx2() || supports_avx()) && !supports_evex()); }
895 static bool supports_sha() { return (_features & CPU_SHA) != 0; }
896 static bool supports_fma() { return (_features & CPU_FMA) != 0 && supports_avx(); }
897 static bool supports_vzeroupper() { return (_features & CPU_VZEROUPPER) != 0; }
898 static bool supports_avx512_vpopcntdq() { return (_features & CPU_AVX512_VPOPCNTDQ) != 0; }
899 static bool supports_avx512_vpclmulqdq() { return (_features & CPU_AVX512_VPCLMULQDQ) != 0; }
900 static bool supports_avx512_vaes() { return (_features & CPU_AVX512_VAES) != 0; }
901 static bool supports_avx512_vnni() { return (_features & CPU_AVX512_VNNI) != 0; }
902 static bool supports_avx512_vbmi() { return (_features & CPU_AVX512_VBMI) != 0; }
903 static bool supports_avx512_vbmi2() { return (_features & CPU_AVX512_VBMI2) != 0; }
38
←
Assuming the condition is true
→
39
←
Returning the value 1, which participates in a condition later
→
904 static bool supports_hv() { return (_features & CPU_HV) != 0; }
905 static bool supports_serialize() { return (_features & CPU_SERIALIZE) != 0; }
906
907 // Intel features
908 static bool is_intel_family_core() { return is_intel() &&
909 extended_cpu_family() == CPU_FAMILY_INTEL_CORE; }
910
911 static bool is_intel_skylake() { return is_intel_family_core() &&
912 extended_cpu_model() == CPU_MODEL_SKYLAKE; }
913
914 static int avx3_threshold();
915
916 static bool is_intel_tsc_synched_at_init() {
917 if (is_intel_family_core()) {
918 uint32_t ext_model = extended_cpu_model();
919 if (ext_model == CPU_MODEL_NEHALEM_EP ||
920 ext_model == CPU_MODEL_WESTMERE_EP ||
921 ext_model == CPU_MODEL_SANDYBRIDGE_EP ||
922 ext_model == CPU_MODEL_IVYBRIDGE_EP) {
923 // <= 2-socket invariant tsc support. EX versions are usually used
924 // in > 2-socket systems and likely don't synchronize tscs at
925 // initialization.
926 // Code that uses tsc values must be prepared for them to arbitrarily
927 // jump forward or backward.
928 return true;
929 }
930 }
931 return false;
932 }
933
934 // This checks if the JVM is potentially affected by an erratum on Intel CPUs (SKX102)
935 // that causes unpredictable behaviour when jcc crosses 64 byte boundaries. Its microcode
936 // mitigation causes regressions when jumps or fused conditional branches cross or end at
937 // 32 byte boundaries.
938 static bool has_intel_jcc_erratum() { return _has_intel_jcc_erratum; }
939
940 // AMD features
941 static bool supports_3dnow_prefetch() { return (_features & CPU_3DNOW_PREFETCH) != 0; }
942 static bool supports_lzcnt() { return (_features & CPU_LZCNT) != 0; }
943 static bool supports_sse4a() { return (_features & CPU_SSE4A) != 0; }
944
945 static bool is_amd_Barcelona() { return is_amd() &&
946 extended_cpu_family() == CPU_FAMILY_AMD_11H; }
947
948 // Intel and AMD newer cores support fast timestamps well
949 static bool supports_tscinv_bit() {
950 return (_features & CPU_TSCINV_BIT) != 0;
951 }
952 static bool supports_tscinv() {
953 return (_features & CPU_TSCINV) != 0;
954 }
955
956 // Intel Core and newer cpus have fast IDIV instruction (excluding Atom).
957 static bool has_fast_idiv() { return is_intel() && cpu_family() == 6 &&
958 supports_sse3() && _model != 0x1C; }
959
960 static bool supports_compare_and_exchange() { return true; }
961
962 static intx allocate_prefetch_distance(bool use_watermark_prefetch) {
963 // Hardware prefetching (distance/size in bytes):
964 // Pentium 3 - 64 / 32
965 // Pentium 4 - 256 / 128
966 // Athlon - 64 / 32 ????
967 // Opteron - 128 / 64 only when 2 sequential cache lines accessed
968 // Core - 128 / 64
969 //
970 // Software prefetching (distance in bytes / instruction with best score):
971 // Pentium 3 - 128 / prefetchnta
972 // Pentium 4 - 512 / prefetchnta
973 // Athlon - 128 / prefetchnta
974 // Opteron - 256 / prefetchnta
975 // Core - 256 / prefetchnta
976 // It will be used only when AllocatePrefetchStyle > 0
977
978 if (is_amd_family()) { // AMD | Hygon
979 if (supports_sse2()) {
980 return 256; // Opteron
981 } else {
982 return 128; // Athlon
983 }
984 } else { // Intel
985 if (supports_sse3() && cpu_family() == 6) {
986 if (supports_sse4_2() && supports_ht()) { // Nehalem based cpus
987 return 192;
988 } else if (use_watermark_prefetch) { // watermark prefetching on Core
989#ifdef _LP641
990 return 384;
991#else
992 return 320;
993#endif
994 }
995 }
996 if (supports_sse2()) {
997 if (cpu_family() == 6) {
998 return 256; // Pentium M, Core, Core2
999 } else {
1000 return 512; // Pentium 4
1001 }
1002 } else {
1003 return 128; // Pentium 3 (and all other old CPUs)
1004 }
1005 }
1006 }
1007
1008 // SSE2 and later processors implement a 'pause' instruction
1009 // that can be used for efficient implementation of
1010 // the intrinsic for java.lang.Thread.onSpinWait()
1011 static bool supports_on_spin_wait() { return supports_sse2(); }
1012
1013 // x86_64 supports fast class initialization checks for static methods.
1014 static bool supports_fast_class_init_checks() {
1015 return LP64_ONLY(true)true NOT_LP64(false); // not implemented on x86_32
1016 }
1017
1018 constexpr static bool supports_stack_watermark_barrier() {
1019 return true;
1020 }
1021
1022 // there are several insns to force cache line sync to memory which
1023 // we can use to ensure mapped non-volatile memory is up to date with
1024 // pending in-cache changes.
1025 //
1026 // 64 bit cpus always support clflush which writes back and evicts
1027 // on 32 bit cpus support is recorded via a feature flag
1028 //
1029 // clflushopt is optional and acts like clflush except it does
1030 // not synchronize with other memory ops. it needs a preceding
1031 // and trailing StoreStore fence
1032 //
1033 // clwb is an optional intel-specific instruction which
1034 // writes back without evicting the line. it also does not
1035 // synchronize with other memory ops. so, it needs preceding
1036 // and trailing StoreStore fences.
1037
1038#ifdef _LP641
1039
1040 static bool supports_clflush(); // Can't inline due to header file conflict
1041#else
1042 static bool supports_clflush() { return ((_features & CPU_FLUSH) != 0); }
1043#endif // _LP64
1044 // Note: CPU_FLUSHOPT and CPU_CLWB bits should always be zero for 32-bit
1045 static bool supports_clflushopt() { return ((_features & CPU_FLUSHOPT) != 0); }
1046 static bool supports_clwb() { return ((_features & CPU_CLWB) != 0); }
1047
1048#ifdef __APPLE__
1049 // Is the CPU running emulated (for example macOS Rosetta running x86_64 code on M1 ARM (aarch64)
1050 static bool is_cpu_emulated();
1051#endif
1052
1053 // support functions for virtualization detection
1054 private:
1055 static void check_virtualizations();
1056
1057 static const char* cpu_family_description(void);
1058 static const char* cpu_model_description(void);
1059 static const char* cpu_brand(void);
1060 static const char* cpu_brand_string(void);
1061
1062 static int cpu_type_description(char* const buf, size_t buf_len);
1063 static int cpu_detailed_description(char* const buf, size_t buf_len);
1064 static int cpu_extended_brand_string(char* const buf, size_t buf_len);
1065
1066 static bool cpu_is_em64t(void);
1067 static bool is_netburst(void);
1068
1069 // Returns bytes written excluding termninating null byte.
1070 static size_t cpu_write_support_string(char* const buf, size_t buf_len);
1071 static void resolve_cpu_information_details(void);
1072 static int64_t max_qualified_cpu_freq_from_brand_string(void);
1073
1074 public:
1075 // Offsets for cpuid asm stub brand string
1076 static ByteSize proc_name_0_offset() { return byte_offset_of(CpuidInfo, proc_name_0)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_0
) - 16)); }
1077 static ByteSize proc_name_1_offset() { return byte_offset_of(CpuidInfo, proc_name_1)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_1
) - 16)); }
1078 static ByteSize proc_name_2_offset() { return byte_offset_of(CpuidInfo, proc_name_2)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_2
) - 16)); }
1079 static ByteSize proc_name_3_offset() { return byte_offset_of(CpuidInfo, proc_name_3)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_3
) - 16)); }
1080 static ByteSize proc_name_4_offset() { return byte_offset_of(CpuidInfo, proc_name_4)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_4
) - 16)); }
1081 static ByteSize proc_name_5_offset() { return byte_offset_of(CpuidInfo, proc_name_5)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_5
) - 16)); }
1082 static ByteSize proc_name_6_offset() { return byte_offset_of(CpuidInfo, proc_name_6)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_6
) - 16)); }
1083 static ByteSize proc_name_7_offset() { return byte_offset_of(CpuidInfo, proc_name_7)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_7
) - 16)); }
1084 static ByteSize proc_name_8_offset() { return byte_offset_of(CpuidInfo, proc_name_8)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_8
) - 16)); }
1085 static ByteSize proc_name_9_offset() { return byte_offset_of(CpuidInfo, proc_name_9)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_9
) - 16)); }
1086 static ByteSize proc_name_10_offset() { return byte_offset_of(CpuidInfo, proc_name_10)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_10
) - 16)); }
1087 static ByteSize proc_name_11_offset() { return byte_offset_of(CpuidInfo, proc_name_11)in_ByteSize((int)(size_t)((intx)&(((CpuidInfo*)16)->proc_name_11
) - 16)); }
1088
1089 static int64_t maximum_qualified_cpu_frequency(void);
1090
1091 static bool supports_tscinv_ext(void);
1092
1093 static void initialize_tsc();
1094 static void initialize_cpu_information(void);
1095};
1096
1097#endif // CPU_X86_VM_VERSION_X86_HPP
←

/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp

1/*
2 * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#ifndef CPU_X86_ASSEMBLER_X86_HPP
26#define CPU_X86_ASSEMBLER_X86_HPP
27
28#include "asm/register.hpp"
29#include "utilities/powerOfTwo.hpp"
30
31// Contains all the definitions needed for x86 assembly code generation.
32
33// Calling convention
34class Argument {
35 public:
36 enum {
37#ifdef _LP641
38#ifdef _WIN64
39 n_int_register_parameters_c = 4, // rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
40 n_float_register_parameters_c = 4, // xmm0 - xmm3 (c_farg0, c_farg1, ... )
41 n_int_register_returns_c = 1, // rax
42 n_float_register_returns_c = 1, // xmm0
43#else
44 n_int_register_parameters_c = 6, // rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
45 n_float_register_parameters_c = 8, // xmm0 - xmm7 (c_farg0, c_farg1, ... )
46 n_int_register_returns_c = 2, // rax, rdx
47 n_float_register_returns_c = 2, // xmm0, xmm1
48#endif // _WIN64
49 n_int_register_parameters_j = 6, // j_rarg0, j_rarg1, ...
50 n_float_register_parameters_j = 8 // j_farg0, j_farg1, ...
51#else
52 n_register_parameters = 0 // 0 registers used to pass arguments
53#endif // _LP64
54 };
55};
56
57
58#ifdef _LP641
59// Symbolically name the register arguments used by the c calling convention.
60// Windows is different from linux/solaris. So much for standards...
61
62#ifdef _WIN64
63
64REGISTER_DECLARATION(Register, c_rarg0, rcx)const Register c_rarg0 = ((Register)rcx);
65REGISTER_DECLARATION(Register, c_rarg1, rdx)const Register c_rarg1 = ((Register)rdx);
66REGISTER_DECLARATION(Register, c_rarg2, r8)const Register c_rarg2 = ((Register)r8);
67REGISTER_DECLARATION(Register, c_rarg3, r9)const Register c_rarg3 = ((Register)r9);
68
69REGISTER_DECLARATION(XMMRegister, c_farg0, xmm0)const XMMRegister c_farg0 = ((XMMRegister)xmm0);
70REGISTER_DECLARATION(XMMRegister, c_farg1, xmm1)const XMMRegister c_farg1 = ((XMMRegister)xmm1);
71REGISTER_DECLARATION(XMMRegister, c_farg2, xmm2)const XMMRegister c_farg2 = ((XMMRegister)xmm2);
72REGISTER_DECLARATION(XMMRegister, c_farg3, xmm3)const XMMRegister c_farg3 = ((XMMRegister)xmm3);
73
74#else
75
76REGISTER_DECLARATION(Register, c_rarg0, rdi)const Register c_rarg0 = ((Register)rdi);
77REGISTER_DECLARATION(Register, c_rarg1, rsi)const Register c_rarg1 = ((Register)rsi);
78REGISTER_DECLARATION(Register, c_rarg2, rdx)const Register c_rarg2 = ((Register)rdx);
79REGISTER_DECLARATION(Register, c_rarg3, rcx)const Register c_rarg3 = ((Register)rcx);
80REGISTER_DECLARATION(Register, c_rarg4, r8)const Register c_rarg4 = ((Register)r8);
81REGISTER_DECLARATION(Register, c_rarg5, r9)const Register c_rarg5 = ((Register)r9);
82
83REGISTER_DECLARATION(XMMRegister, c_farg0, xmm0)const XMMRegister c_farg0 = ((XMMRegister)xmm0);
84REGISTER_DECLARATION(XMMRegister, c_farg1, xmm1)const XMMRegister c_farg1 = ((XMMRegister)xmm1);
85REGISTER_DECLARATION(XMMRegister, c_farg2, xmm2)const XMMRegister c_farg2 = ((XMMRegister)xmm2);
86REGISTER_DECLARATION(XMMRegister, c_farg3, xmm3)const XMMRegister c_farg3 = ((XMMRegister)xmm3);
87REGISTER_DECLARATION(XMMRegister, c_farg4, xmm4)const XMMRegister c_farg4 = ((XMMRegister)xmm4);
88REGISTER_DECLARATION(XMMRegister, c_farg5, xmm5)const XMMRegister c_farg5 = ((XMMRegister)xmm5);
89REGISTER_DECLARATION(XMMRegister, c_farg6, xmm6)const XMMRegister c_farg6 = ((XMMRegister)xmm6);
90REGISTER_DECLARATION(XMMRegister, c_farg7, xmm7)const XMMRegister c_farg7 = ((XMMRegister)xmm7);
91
92#endif // _WIN64
93
94// Symbolically name the register arguments used by the Java calling convention.
95// We have control over the convention for java so we can do what we please.
96// What pleases us is to offset the java calling convention so that when
97// we call a suitable jni method the arguments are lined up and we don't
98// have to do little shuffling. A suitable jni method is non-static and a
99// small number of arguments (two fewer args on windows)
100//
101// |-------------------------------------------------------|
102// | c_rarg0 c_rarg1 c_rarg2 c_rarg3 c_rarg4 c_rarg5 |
103// |-------------------------------------------------------|
104// | rcx rdx r8 r9 rdi* rsi* | windows (* not a c_rarg)
105// | rdi rsi rdx rcx r8 r9 | solaris/linux
106// |-------------------------------------------------------|
107// | j_rarg5 j_rarg0 j_rarg1 j_rarg2 j_rarg3 j_rarg4 |
108// |-------------------------------------------------------|
109
110REGISTER_DECLARATION(Register, j_rarg0, c_rarg1)const Register j_rarg0 = ((Register)c_rarg1);
111REGISTER_DECLARATION(Register, j_rarg1, c_rarg2)const Register j_rarg1 = ((Register)c_rarg2);
112REGISTER_DECLARATION(Register, j_rarg2, c_rarg3)const Register j_rarg2 = ((Register)c_rarg3);
113// Windows runs out of register args here
114#ifdef _WIN64
115REGISTER_DECLARATION(Register, j_rarg3, rdi)const Register j_rarg3 = ((Register)rdi);
116REGISTER_DECLARATION(Register, j_rarg4, rsi)const Register j_rarg4 = ((Register)rsi);
117#else
118REGISTER_DECLARATION(Register, j_rarg3, c_rarg4)const Register j_rarg3 = ((Register)c_rarg4);
119REGISTER_DECLARATION(Register, j_rarg4, c_rarg5)const Register j_rarg4 = ((Register)c_rarg5);
120#endif /* _WIN64 */
121REGISTER_DECLARATION(Register, j_rarg5, c_rarg0)const Register j_rarg5 = ((Register)c_rarg0);
122
123REGISTER_DECLARATION(XMMRegister, j_farg0, xmm0)const XMMRegister j_farg0 = ((XMMRegister)xmm0);
124REGISTER_DECLARATION(XMMRegister, j_farg1, xmm1)const XMMRegister j_farg1 = ((XMMRegister)xmm1);
125REGISTER_DECLARATION(XMMRegister, j_farg2, xmm2)const XMMRegister j_farg2 = ((XMMRegister)xmm2);
126REGISTER_DECLARATION(XMMRegister, j_farg3, xmm3)const XMMRegister j_farg3 = ((XMMRegister)xmm3);
127REGISTER_DECLARATION(XMMRegister, j_farg4, xmm4)const XMMRegister j_farg4 = ((XMMRegister)xmm4);
128REGISTER_DECLARATION(XMMRegister, j_farg5, xmm5)const XMMRegister j_farg5 = ((XMMRegister)xmm5);
129REGISTER_DECLARATION(XMMRegister, j_farg6, xmm6)const XMMRegister j_farg6 = ((XMMRegister)xmm6);
130REGISTER_DECLARATION(XMMRegister, j_farg7, xmm7)const XMMRegister j_farg7 = ((XMMRegister)xmm7);
131
132REGISTER_DECLARATION(Register, rscratch1, r10)const Register rscratch1 = ((Register)r10); // volatile
133REGISTER_DECLARATION(Register, rscratch2, r11)const Register rscratch2 = ((Register)r11); // volatile
134
135REGISTER_DECLARATION(Register, r12_heapbase, r12)const Register r12_heapbase = ((Register)r12); // callee-saved
136REGISTER_DECLARATION(Register, r15_thread, r15)const Register r15_thread = ((Register)r15); // callee-saved
137
138#else
139// rscratch1 will apear in 32bit code that is dead but of course must compile
140// Using noreg ensures if the dead code is incorrectly live and executed it
141// will cause an assertion failure
142#define rscratch1 noreg
143#define rscratch2 noreg
144
145#endif // _LP64
146
147// JSR 292
148// On x86, the SP does not have to be saved when invoking method handle intrinsics
149// or compiled lambda forms. We indicate that by setting rbp_mh_SP_save to noreg.
150REGISTER_DECLARATION(Register, rbp_mh_SP_save, noreg)const Register rbp_mh_SP_save = ((Register)noreg);
151
152// Address is an abstraction used to represent a memory location
153// using any of the amd64 addressing modes with one object.
154//
155// Note: A register location is represented via a Register, not
156// via an address for efficiency & simplicity reasons.
157
158class ArrayAddress;
159
160class Address {
161 public:
162 enum ScaleFactor {
163 no_scale = -1,
164 times_1 = 0,
165 times_2 = 1,
166 times_4 = 2,
167 times_8 = 3,
168 times_ptr = LP64_ONLY(times_8)times_8 NOT_LP64(times_4)
169 };
170 static ScaleFactor times(int size) {
171 assert(size >= 1 && size <= 8 && is_power_of_2(size), "bad scale size")do { if (!(size >= 1 && size <= 8 && is_power_of_2
(size))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 171, "assert(" "size >= 1 && size <= 8 && is_power_of_2(size)"
") failed", "bad scale size"); ::breakpoint(); } } while (0);
172 if (size == 8) return times_8;
173 if (size == 4) return times_4;
174 if (size == 2) return times_2;
175 return times_1;
176 }
177 static int scale_size(ScaleFactor scale) {
178 assert(scale != no_scale, "")do { if (!(scale != no_scale)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 178, "assert(" "scale != no_scale" ") failed", ""); ::breakpoint
(); } } while (0);
179 assert(((1 << (int)times_1) == 1 &&do { if (!(((1 << (int)times_1) == 1 && (1 <<
(int)times_2) == 2 && (1 << (int)times_4) == 4
&& (1 << (int)times_8) == 8))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)"
") failed", ""); ::breakpoint(); } } while (0)
180 (1 << (int)times_2) == 2 &&do { if (!(((1 << (int)times_1) == 1 && (1 <<
(int)times_2) == 2 && (1 << (int)times_4) == 4
&& (1 << (int)times_8) == 8))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)"
") failed", ""); ::breakpoint(); } } while (0)
181 (1 << (int)times_4) == 4 &&do { if (!(((1 << (int)times_1) == 1 && (1 <<
(int)times_2) == 2 && (1 << (int)times_4) == 4
&& (1 << (int)times_8) == 8))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)"
") failed", ""); ::breakpoint(); } } while (0)
182 (1 << (int)times_8) == 8), "")do { if (!(((1 << (int)times_1) == 1 && (1 <<
(int)times_2) == 2 && (1 << (int)times_4) == 4
&& (1 << (int)times_8) == 8))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)"
") failed", ""); ::breakpoint(); } } while (0);
183 return (1 << (int)scale);
184 }
185
186 private:
187 Register _base;
188 Register _index;
189 XMMRegister _xmmindex;
190 ScaleFactor _scale;
191 int _disp;
192 bool _isxmmindex;
193 RelocationHolder _rspec;
194
195 // Easily misused constructors make them private
196 // %%% can we make these go away?
197 NOT_LP64(Address(address loc, RelocationHolder spec);)
198 Address(int disp, address loc, relocInfo::relocType rtype);
199 Address(int disp, address loc, RelocationHolder spec);
200
201 public:
202
203 int disp() { return _disp; }
204 // creation
205 Address()
206 : _base(noreg),
207 _index(noreg),
208 _xmmindex(xnoreg),
209 _scale(no_scale),
210 _disp(0),
211 _isxmmindex(false){
212 }
213
214 // No default displacement otherwise Register can be implicitly
215 // converted to 0(Register) which is quite a different animal.
216
217 Address(Register base, int disp)
218 : _base(base),
219 _index(noreg),
220 _xmmindex(xnoreg),
221 _scale(no_scale),
222 _disp(disp),
223 _isxmmindex(false){
224 }
225
226 Address(Register base, Register index, ScaleFactor scale, int disp = 0)
227 : _base (base),
228 _index(index),
229 _xmmindex(xnoreg),
230 _scale(scale),
231 _disp (disp),
232 _isxmmindex(false) {
233 assert(!index->is_valid() == (scale == Address::no_scale),do { if (!(!index->is_valid() == (scale == Address::no_scale
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 234, "assert(" "!index->is_valid() == (scale == Address::no_scale)"
") failed", "inconsistent address"); ::breakpoint(); } } while
(0)
46
←
Called C++ object pointer is null
234 "inconsistent address")do { if (!(!index->is_valid() == (scale == Address::no_scale
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 234, "assert(" "!index->is_valid() == (scale == Address::no_scale)"
") failed", "inconsistent address"); ::breakpoint(); } } while
(0);
235 }
236
237 Address(Register base, RegisterOrConstant index, ScaleFactor scale = times_1, int disp = 0)
238 : _base (base),
239 _index(index.register_or_noreg()),
240 _xmmindex(xnoreg),
241 _scale(scale),
242 _disp (disp + (index.constant_or_zero() * scale_size(scale))),
243 _isxmmindex(false){
244 if (!index.is_register()) scale = Address::no_scale;
245 assert(!_index->is_valid() == (scale == Address::no_scale),do { if (!(!_index->is_valid() == (scale == Address::no_scale
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 246, "assert(" "!_index->is_valid() == (scale == Address::no_scale)"
") failed", "inconsistent address"); ::breakpoint(); } } while
(0)
246 "inconsistent address")do { if (!(!_index->is_valid() == (scale == Address::no_scale
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 246, "assert(" "!_index->is_valid() == (scale == Address::no_scale)"
") failed", "inconsistent address"); ::breakpoint(); } } while
(0);
247 }
248
249 Address(Register base, XMMRegister index, ScaleFactor scale, int disp = 0)
250 : _base (base),
251 _index(noreg),
252 _xmmindex(index),
253 _scale(scale),
254 _disp(disp),
255 _isxmmindex(true) {
256 assert(!index->is_valid() == (scale == Address::no_scale),do { if (!(!index->is_valid() == (scale == Address::no_scale
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 257, "assert(" "!index->is_valid() == (scale == Address::no_scale)"
") failed", "inconsistent address"); ::breakpoint(); } } while
(0)
257 "inconsistent address")do { if (!(!index->is_valid() == (scale == Address::no_scale
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 257, "assert(" "!index->is_valid() == (scale == Address::no_scale)"
") failed", "inconsistent address"); ::breakpoint(); } } while
(0);
258 }
259
260 // The following overloads are used in connection with the
261 // ByteSize type (see sizes.hpp). They simplify the use of
262 // ByteSize'd arguments in assembly code.
263
264 Address(Register base, ByteSize disp)
265 : Address(base, in_bytes(disp)) {}
266
267 Address(Register base, Register index, ScaleFactor scale, ByteSize disp)
268 : Address(base, index, scale, in_bytes(disp)) {}
269
270 Address(Register base, RegisterOrConstant index, ScaleFactor scale, ByteSize disp)
271 : Address(base, index, scale, in_bytes(disp)) {}
272
273 Address plus_disp(int disp) const {
274 Address a = (*this);
275 a._disp += disp;
276 return a;
277 }
278 Address plus_disp(RegisterOrConstant disp, ScaleFactor scale = times_1) const {
279 Address a = (*this);
280 a._disp += disp.constant_or_zero() * scale_size(scale);
281 if (disp.is_register()) {
282 assert(!a.index()->is_valid(), "competing indexes")do { if (!(!a.index()->is_valid())) { (*g_assert_poison) =
'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 282, "assert(" "!a.index()->is_valid()" ") failed", "competing indexes"
); ::breakpoint(); } } while (0);
283 a._index = disp.as_register();
284 a._scale = scale;
285 }
286 return a;
287 }
288 bool is_same_address(Address a) const {
289 // disregard _rspec
290 return _base == a._base && _disp == a._disp && _index == a._index && _scale == a._scale;
291 }
292
293 // accessors
294 bool uses(Register reg) const { return _base == reg || _index == reg; }
295 Register base() const { return _base; }
296 Register index() const { return _index; }
297 XMMRegister xmmindex() const { return _xmmindex; }
298 ScaleFactor scale() const { return _scale; }
299 int disp() const { return _disp; }
300 bool isxmmindex() const { return _isxmmindex; }
301
302 // Convert the raw encoding form into the form expected by the constructor for
303 // Address. An index of 4 (rsp) corresponds to having no index, so convert
304 // that to noreg for the Address constructor.
305 static Address make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc);
306
307 static Address make_array(ArrayAddress);
308
309 private:
310 bool base_needs_rex() const {
311 return _base->is_valid() && _base->encoding() >= 8;
312 }
313
314 bool index_needs_rex() const {
315 return _index->is_valid() &&_index->encoding() >= 8;
316 }
317
318 bool xmmindex_needs_rex() const {
319 return _xmmindex->is_valid() && _xmmindex->encoding() >= 8;
320 }
321
322 relocInfo::relocType reloc() const { return _rspec.type(); }
323
324 friend class Assembler;
325 friend class MacroAssembler;
326 friend class LIR_Assembler; // base/index/scale/disp
327};
328
329//
330// AddressLiteral has been split out from Address because operands of this type
331// need to be treated specially on 32bit vs. 64bit platforms. By splitting it out
332// the few instructions that need to deal with address literals are unique and the
333// MacroAssembler does not have to implement every instruction in the Assembler
334// in order to search for address literals that may need special handling depending
335// on the instruction and the platform. As small step on the way to merging i486/amd64
336// directories.
337//
338class AddressLiteral {
339 friend class ArrayAddress;
340 RelocationHolder _rspec;
341 // Typically we use AddressLiterals we want to use their rval
342 // However in some situations we want the lval (effect address) of the item.
343 // We provide a special factory for making those lvals.
344 bool _is_lval;
345
346 // If the target is far we'll need to load the ea of this to
347 // a register to reach it. Otherwise if near we can do rip
348 // relative addressing.
349
350 address _target;
351
352 protected:
353 // creation
354 AddressLiteral()
355 : _is_lval(false),
356 _target(NULL__null)
357 {}
358
359 public:
360
361
362 AddressLiteral(address target, relocInfo::relocType rtype);
363
364 AddressLiteral(address target, RelocationHolder const& rspec)
365 : _rspec(rspec),
366 _is_lval(false),
367 _target(target)
368 {}
369
370 AddressLiteral addr() {
371 AddressLiteral ret = *this;
372 ret._is_lval = true;
373 return ret;
374 }
375
376
377 private:
378
379 address target() { return _target; }
380 bool is_lval() { return _is_lval; }
381
382 relocInfo::relocType reloc() const { return _rspec.type(); }
383 const RelocationHolder& rspec() const { return _rspec; }
384
385 friend class Assembler;
386 friend class MacroAssembler;
387 friend class Address;
388 friend class LIR_Assembler;
389};
390
391// Convience classes
392class RuntimeAddress: public AddressLiteral {
393
394 public:
395
396 RuntimeAddress(address target) : AddressLiteral(target, relocInfo::runtime_call_type) {}
397
398};
399
400class ExternalAddress: public AddressLiteral {
401 private:
402 static relocInfo::relocType reloc_for_target(address target) {
403 // Sometimes ExternalAddress is used for values which aren't
404 // exactly addresses, like the card table base.
405 // external_word_type can't be used for values in the first page
406 // so just skip the reloc in that case.
407 return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none;
408 }
409
410 public:
411
412 ExternalAddress(address target) : AddressLiteral(target, reloc_for_target(target)) {}
413
414};
415
416class InternalAddress: public AddressLiteral {
417
418 public:
419
420 InternalAddress(address target) : AddressLiteral(target, relocInfo::internal_word_type) {}
421
422};
423
424// x86 can do array addressing as a single operation since disp can be an absolute
425// address amd64 can't. We create a class that expresses the concept but does extra
426// magic on amd64 to get the final result
427
428class ArrayAddress {
429 private:
430
431 AddressLiteral _base;
432 Address _index;
433
434 public:
435
436 ArrayAddress() {};
437 ArrayAddress(AddressLiteral base, Address index): _base(base), _index(index) {};
438 AddressLiteral base() { return _base; }
439 Address index() { return _index; }
440
441};
442
443class InstructionAttr;
444
445// 64-bit refect the fxsave size which is 512 bytes and the new xsave area on EVEX which is another 2176 bytes
446// See fxsave and xsave(EVEX enabled) documentation for layout
447const int FPUStateSizeInWords = NOT_LP64(27) LP64_ONLY(2688 / wordSize)2688 / wordSize;
448
449// The Intel x86/Amd64 Assembler: Pure assembler doing NO optimizations on the instruction
450// level (e.g. mov rax, 0 is not translated into xor rax, rax!); i.e., what you write
451// is what you get. The Assembler is generating code into a CodeBuffer.
452
453class Assembler : public AbstractAssembler {
454 friend class AbstractAssembler; // for the non-virtual hack
455 friend class LIR_Assembler; // as_Address()
456 friend class StubGenerator;
457
458 public:
459 enum Condition { // The x86 condition codes used for conditional jumps/moves.
460 zero = 0x4,
461 notZero = 0x5,
462 equal = 0x4,
463 notEqual = 0x5,
464 less = 0xc,
465 lessEqual = 0xe,
466 greater = 0xf,
467 greaterEqual = 0xd,
468 below = 0x2,
469 belowEqual = 0x6,
470 above = 0x7,
471 aboveEqual = 0x3,
472 overflow = 0x0,
473 noOverflow = 0x1,
474 carrySet = 0x2,
475 carryClear = 0x3,
476 negative = 0x8,
477 positive = 0x9,
478 parity = 0xa,
479 noParity = 0xb
480 };
481
482 enum Prefix {
483 // segment overrides
484 CS_segment = 0x2e,
485 SS_segment = 0x36,
486 DS_segment = 0x3e,
487 ES_segment = 0x26,
488 FS_segment = 0x64,
489 GS_segment = 0x65,
490
491 REX = 0x40,
492
493 REX_B = 0x41,
494 REX_X = 0x42,
495 REX_XB = 0x43,
496 REX_R = 0x44,
497 REX_RB = 0x45,
498 REX_RX = 0x46,
499 REX_RXB = 0x47,
500
501 REX_W = 0x48,
502
503 REX_WB = 0x49,
504 REX_WX = 0x4A,
505 REX_WXB = 0x4B,
506 REX_WR = 0x4C,
507 REX_WRB = 0x4D,
508 REX_WRX = 0x4E,
509 REX_WRXB = 0x4F,
510
511 VEX_3bytes = 0xC4,
512 VEX_2bytes = 0xC5,
513 EVEX_4bytes = 0x62,
514 Prefix_EMPTY = 0x0
515 };
516
517 enum VexPrefix {
518 VEX_B = 0x20,
519 VEX_X = 0x40,
520 VEX_R = 0x80,
521 VEX_W = 0x80
522 };
523
524 enum ExexPrefix {
525 EVEX_F = 0x04,
526 EVEX_V = 0x08,
527 EVEX_Rb = 0x10,
528 EVEX_X = 0x40,
529 EVEX_Z = 0x80
530 };
531
532 enum VexSimdPrefix {
533 VEX_SIMD_NONE = 0x0,
534 VEX_SIMD_66 = 0x1,
535 VEX_SIMD_F3 = 0x2,
536 VEX_SIMD_F2 = 0x3
537 };
538
539 enum VexOpcode {
540 VEX_OPCODE_NONE = 0x0,
541 VEX_OPCODE_0F = 0x1,
542 VEX_OPCODE_0F_38 = 0x2,
543 VEX_OPCODE_0F_3A = 0x3,
544 VEX_OPCODE_MASK = 0x1F
545 };
546
547 enum AvxVectorLen {
548 AVX_128bit = 0x0,
549 AVX_256bit = 0x1,
550 AVX_512bit = 0x2,
551 AVX_NoVec = 0x4
552 };
553
554 enum EvexTupleType {
555 EVEX_FV = 0,
556 EVEX_HV = 4,
557 EVEX_FVM = 6,
558 EVEX_T1S = 7,
559 EVEX_T1F = 11,
560 EVEX_T2 = 13,
561 EVEX_T4 = 15,
562 EVEX_T8 = 17,
563 EVEX_HVM = 18,
564 EVEX_QVM = 19,
565 EVEX_OVM = 20,
566 EVEX_M128 = 21,
567 EVEX_DUP = 22,
568 EVEX_ETUP = 23
569 };
570
571 enum EvexInputSizeInBits {
572 EVEX_8bit = 0,
573 EVEX_16bit = 1,
574 EVEX_32bit = 2,
575 EVEX_64bit = 3,
576 EVEX_NObit = 4
577 };
578
579 enum WhichOperand {
580 // input to locate_operand, and format code for relocations
581 imm_operand = 0, // embedded 32-bit|64-bit immediate operand
582 disp32_operand = 1, // embedded 32-bit displacement or address
583 call32_operand = 2, // embedded 32-bit self-relative displacement
584#ifndef _LP641
585 _WhichOperand_limit = 3
586#else
587 narrow_oop_operand = 3, // embedded 32-bit immediate narrow oop
588 _WhichOperand_limit = 4
589#endif
590 };
591
592 // Comparison predicates for integral types & FP types when using SSE
593 enum ComparisonPredicate {
594 eq = 0,
595 lt = 1,
596 le = 2,
597 _false = 3,
598 neq = 4,
599 nlt = 5,
600 nle = 6,
601 _true = 7
602 };
603
604 // Comparison predicates for FP types when using AVX
605 // O means ordered. U is unordered. When using ordered, any NaN comparison is false. Otherwise, it is true.
606 // S means signaling. Q means non-signaling. When signaling is true, instruction signals #IA on NaN.
607 enum ComparisonPredicateFP {
608 EQ_OQ = 0,
609 LT_OS = 1,
610 LE_OS = 2,
611 UNORD_Q = 3,
612 NEQ_UQ = 4,
613 NLT_US = 5,
614 NLE_US = 6,
615 ORD_Q = 7,
616 EQ_UQ = 8,
617 NGE_US = 9,
618 NGT_US = 0xA,
619 FALSE_OQ = 0XB,
620 NEQ_OQ = 0xC,
621 GE_OS = 0xD,
622 GT_OS = 0xE,
623 TRUE_UQ = 0xF,
624 EQ_OS = 0x10,
625 LT_OQ = 0x11,
626 LE_OQ = 0x12,
627 UNORD_S = 0x13,
628 NEQ_US = 0x14,
629 NLT_UQ = 0x15,
630 NLE_UQ = 0x16,
631 ORD_S = 0x17,
632 EQ_US = 0x18,
633 NGE_UQ = 0x19,
634 NGT_UQ = 0x1A,
635 FALSE_OS = 0x1B,
636 NEQ_OS = 0x1C,
637 GE_OQ = 0x1D,
638 GT_OQ = 0x1E,
639 TRUE_US =0x1F
640 };
641
642 enum Width {
643 B = 0,
644 W = 1,
645 D = 2,
646 Q = 3
647 };
648
649 //---< calculate length of instruction >---
650 // As instruction size can't be found out easily on x86/x64,
651 // we just use '4' for len and maxlen.
652 // instruction must start at passed address
653 static unsigned int instr_len(unsigned char *instr) { return 4; }
654
655 //---< longest instructions >---
656 // Max instruction length is not specified in architecture documentation.
657 // We could use a "safe enough" estimate (15), but just default to
658 // instruction length guess from above.
659 static unsigned int instr_maxlen() { return 4; }
660
661 // NOTE: The general philopsophy of the declarations here is that 64bit versions
662 // of instructions are freely declared without the need for wrapping them an ifdef.
663 // (Some dangerous instructions are ifdef's out of inappropriate jvm's.)
664 // In the .cpp file the implementations are wrapped so that they are dropped out
665 // of the resulting jvm. This is done mostly to keep the footprint of MINIMAL
666 // to the size it was prior to merging up the 32bit and 64bit assemblers.
667 //
668 // This does mean you'll get a linker/runtime error if you use a 64bit only instruction
669 // in a 32bit vm. This is somewhat unfortunate but keeps the ifdef noise down.
670
671private:
672
673 bool _legacy_mode_bw;
674 bool _legacy_mode_dq;
675 bool _legacy_mode_vl;
676 bool _legacy_mode_vlbw;
677 NOT_LP64(bool _is_managed;)
678
679 class InstructionAttr *_attributes;
680
681 // 64bit prefixes
682 void prefix(Register reg);
683 void prefix(Register dst, Register src, Prefix p);
684 void prefix(Register dst, Address adr, Prefix p);
685
686 void prefix(Address adr);
687 void prefix(Address adr, Register reg, bool byteinst = false);
688 void prefix(Address adr, XMMRegister reg);
689
690 int prefix_and_encode(int reg_enc, bool byteinst = false);
691 int prefix_and_encode(int dst_enc, int src_enc) {
692 return prefix_and_encode(dst_enc, false, src_enc, false);
693 }
694 int prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte);
695
696 // Some prefixq variants always emit exactly one prefix byte, so besides a
697 // prefix-emitting method we provide a method to get the prefix byte to emit,
698 // which can then be folded into a byte stream.
699 int8_t get_prefixq(Address adr);
700 int8_t get_prefixq(Address adr, Register reg);
701
702 void prefixq(Address adr);
703 void prefixq(Address adr, Register reg);
704 void prefixq(Address adr, XMMRegister reg);
705
706 int prefixq_and_encode(int reg_enc);
707 int prefixq_and_encode(int dst_enc, int src_enc);
708
709 void rex_prefix(Address adr, XMMRegister xreg,
710 VexSimdPrefix pre, VexOpcode opc, bool rex_w);
711 int rex_prefix_and_encode(int dst_enc, int src_enc,
712 VexSimdPrefix pre, VexOpcode opc, bool rex_w);
713
714 void vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc);
715
716 void evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v,
717 int nds_enc, VexSimdPrefix pre, VexOpcode opc);
718
719 void vex_prefix(Address adr, int nds_enc, int xreg_enc,
720 VexSimdPrefix pre, VexOpcode opc,
721 InstructionAttr *attributes);
722
723 int vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc,
724 VexSimdPrefix pre, VexOpcode opc,
725 InstructionAttr *attributes);
726
727 void simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
728 VexOpcode opc, InstructionAttr *attributes);
729
730 int simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
731 VexOpcode opc, InstructionAttr *attributes);
732
733 // Helper functions for groups of instructions
734 void emit_arith_b(int op1, int op2, Register dst, int imm8);
735
736 void emit_arith(int op1, int op2, Register dst, int32_t imm32);
737 // Force generation of a 4 byte immediate value even if it fits into 8bit
738 void emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32);
739 void emit_arith(int op1, int op2, Register dst, Register src);
740
741 bool emit_compressed_disp_byte(int &disp);
742
743 void emit_modrm(int mod, int dst_enc, int src_enc);
744 void emit_modrm_disp8(int mod, int dst_enc, int src_enc,
745 int disp);
746 void emit_modrm_sib(int mod, int dst_enc, int src_enc,
747 Address::ScaleFactor scale, int index_enc, int base_enc);
748 void emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
749 Address::ScaleFactor scale, int index_enc, int base_enc,
750 int disp);
751
752 void emit_operand_helper(int reg_enc,
753 int base_enc, int index_enc, Address::ScaleFactor scale,
754 int disp,
755 RelocationHolder const& rspec,
756 int rip_relative_correction = 0);
757
758 void emit_operand(Register reg,
759 Register base, Register index, Address::ScaleFactor scale,
760 int disp,
761 RelocationHolder const& rspec,
762 int rip_relative_correction = 0);
763
764 void emit_operand(Register reg,
765 Register base, XMMRegister index, Address::ScaleFactor scale,
766 int disp,
767 RelocationHolder const& rspec);
768
769 void emit_operand(XMMRegister xreg,
770 Register base, XMMRegister xindex, Address::ScaleFactor scale,
771 int disp,
772 RelocationHolder const& rspec);
773
774 void emit_operand(Register reg, Address adr,
775 int rip_relative_correction = 0);
776
777 void emit_operand(XMMRegister reg,
778 Register base, Register index, Address::ScaleFactor scale,
779 int disp,
780 RelocationHolder const& rspec);
781
782 void emit_operand(XMMRegister reg, Address adr);
783
784 // Immediate-to-memory forms
785 void emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32);
786
787 protected:
788 #ifdef ASSERT1
789 void check_relocation(RelocationHolder const& rspec, int format);
790 #endif
791
792 void emit_data(jint data, relocInfo::relocType rtype, int format);
793 void emit_data(jint data, RelocationHolder const& rspec, int format);
794 void emit_data64(jlong data, relocInfo::relocType rtype, int format = 0);
795 void emit_data64(jlong data, RelocationHolder const& rspec, int format = 0);
796
797 bool reachable(AddressLiteral adr) NOT_LP64({ return true;});
798
799 // These are all easily abused and hence protected
800
801 // 32BIT ONLY SECTION
802#ifndef _LP641
803 // Make these disappear in 64bit mode since they would never be correct
804 void cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY
805 void cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY
806
807 void mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY
808 void mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY
809
810 void push_literal32(int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY
811#else
812 // 64BIT ONLY SECTION
813 void mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec); // 64BIT ONLY
814
815 void cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec);
816 void cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec);
817
818 void mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec);
819 void mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec);
820#endif // _LP64
821
822 // These are unique in that we are ensured by the caller that the 32bit
823 // relative in these instructions will always be able to reach the potentially
824 // 64bit address described by entry. Since they can take a 64bit address they
825 // don't have the 32 suffix like the other instructions in this class.
826
827 void call_literal(address entry, RelocationHolder const& rspec);
828 void jmp_literal(address entry, RelocationHolder const& rspec);
829
830 // Avoid using directly section
831 // Instructions in this section are actually usable by anyone without danger
832 // of failure but have performance issues that are addressed my enhanced
833 // instructions which will do the proper thing base on the particular cpu.
834 // We protect them because we don't trust you...
835
836 // Don't use next inc() and dec() methods directly. INC & DEC instructions
837 // could cause a partial flag stall since they don't set CF flag.
838 // Use MacroAssembler::decrement() & MacroAssembler::increment() methods
839 // which call inc() & dec() or add() & sub() in accordance with
840 // the product flag UseIncDec value.
841
842 void decl(Register dst);
843 void decl(Address dst);
844 void decq(Address dst);
845
846 void incl(Register dst);
847 void incl(Address dst);
848 void incq(Register dst);
849 void incq(Address dst);
850
851 // New cpus require use of movsd and movss to avoid partial register stall
852 // when loading from memory. But for old Opteron use movlpd instead of movsd.
853 // The selection is done in MacroAssembler::movdbl() and movflt().
854
855 // Move Scalar Single-Precision Floating-Point Values
856 void movss(XMMRegister dst, Address src);
857 void movss(XMMRegister dst, XMMRegister src);
858 void movss(Address dst, XMMRegister src);
859
860 // Move Scalar Double-Precision Floating-Point Values
861 void movsd(XMMRegister dst, Address src);
862 void movsd(XMMRegister dst, XMMRegister src);
863 void movsd(Address dst, XMMRegister src);
864 void movlpd(XMMRegister dst, Address src);
865
866 // New cpus require use of movaps and movapd to avoid partial register stall
867 // when moving between registers.
868 void movaps(XMMRegister dst, XMMRegister src);
869 void movapd(XMMRegister dst, XMMRegister src);
870
871 // End avoid using directly
872
873
874 // Instruction prefixes
875 void prefix(Prefix p);
876
877 public:
878
879 // Creation
880 Assembler(CodeBuffer* code) : AbstractAssembler(code) {
881 init_attributes();
882 }
883
884 // Decoding
885 static address locate_operand(address inst, WhichOperand which);
886 static address locate_next_instruction(address inst);
887
888 // Utilities
889 static bool query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
890 int cur_tuple_type, int in_size_in_bits, int cur_encoding);
891
892 // Generic instructions
893 // Does 32bit or 64bit as needed for the platform. In some sense these
894 // belong in macro assembler but there is no need for both varieties to exist
895
896 void init_attributes(void);
897
898 void set_attributes(InstructionAttr *attributes) { _attributes = attributes; }
899 void clear_attributes(void) { _attributes = NULL__null; }
900
901 void set_managed(void) { NOT_LP64(_is_managed = true;) }
902 void clear_managed(void) { NOT_LP64(_is_managed = false;) }
903 bool is_managed(void) {
904 NOT_LP64(return _is_managed;)
905 LP64_ONLY(return false;)return false; }
906
907 void lea(Register dst, Address src);
908
909 void mov(Register dst, Register src);
910
911#ifdef _LP641
912 // support caching the result of some routines
913
914 // must be called before pusha(), popa(), vzeroupper() - checked with asserts
915 static void precompute_instructions();
916
917 void pusha_uncached();
918 void popa_uncached();
919#endif
920 void vzeroupper_uncached();
921 void decq(Register dst);
922
923 void pusha();
924 void popa();
925
926 void pushf();
927 void popf();
928
929 void push(int32_t imm32);
930
931 void push(Register src);
932
933 void pop(Register dst);
934
935 // These are dummies to prevent surprise implicit conversions to Register
936 void push(void* v);
937 void pop(void* v);
938
939 // These do register sized moves/scans
940 void rep_mov();
941 void rep_stos();
942 void rep_stosb();
943 void repne_scan();
944#ifdef _LP641
945 void repne_scanl();
946#endif
947
948 // Vanilla instructions in lexical order
949
950 void adcl(Address dst, int32_t imm32);
951 void adcl(Address dst, Register src);
952 void adcl(Register dst, int32_t imm32);
953 void adcl(Register dst, Address src);
954 void adcl(Register dst, Register src);
955
956 void adcq(Register dst, int32_t imm32);
957 void adcq(Register dst, Address src);
958 void adcq(Register dst, Register src);
959
960 void addb(Address dst, int imm8);
961 void addw(Register dst, Register src);
962 void addw(Address dst, int imm16);
963
964 void addl(Address dst, int32_t imm32);
965 void addl(Address dst, Register src);
966 void addl(Register dst, int32_t imm32);
967 void addl(Register dst, Address src);
968 void addl(Register dst, Register src);
969
970 void addq(Address dst, int32_t imm32);
971 void addq(Address dst, Register src);
972 void addq(Register dst, int32_t imm32);
973 void addq(Register dst, Address src);
974 void addq(Register dst, Register src);
975
976#ifdef _LP641
977 //Add Unsigned Integers with Carry Flag
978 void adcxq(Register dst, Register src);
979
980 //Add Unsigned Integers with Overflow Flag
981 void adoxq(Register dst, Register src);
982#endif
983
984 void addr_nop_4();
985 void addr_nop_5();
986 void addr_nop_7();
987 void addr_nop_8();
988
989 // Add Scalar Double-Precision Floating-Point Values
990 void addsd(XMMRegister dst, Address src);
991 void addsd(XMMRegister dst, XMMRegister src);
992
993 // Add Scalar Single-Precision Floating-Point Values
994 void addss(XMMRegister dst, Address src);
995 void addss(XMMRegister dst, XMMRegister src);
996
997 // AES instructions
998 void aesdec(XMMRegister dst, Address src);
999 void aesdec(XMMRegister dst, XMMRegister src);
1000 void aesdeclast(XMMRegister dst, Address src);
1001 void aesdeclast(XMMRegister dst, XMMRegister src);
1002 void aesenc(XMMRegister dst, Address src);
1003 void aesenc(XMMRegister dst, XMMRegister src);
1004 void aesenclast(XMMRegister dst, Address src);
1005 void aesenclast(XMMRegister dst, XMMRegister src);
1006 // Vector AES instructions
1007 void vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1008 void vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1009 void vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1010 void vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1011
1012 void andw(Register dst, Register src);
1013 void andb(Address dst, Register src);
1014
1015 void andl(Address dst, int32_t imm32);
1016 void andl(Register dst, int32_t imm32);
1017 void andl(Register dst, Address src);
1018 void andl(Register dst, Register src);
1019 void andl(Address dst, Register src);
1020
1021 void andq(Address dst, int32_t imm32);
1022 void andq(Register dst, int32_t imm32);
1023 void andq(Register dst, Address src);
1024 void andq(Register dst, Register src);
1025 void andq(Address dst, Register src);
1026
1027 // BMI instructions
1028 void andnl(Register dst, Register src1, Register src2);
1029 void andnl(Register dst, Register src1, Address src2);
1030 void andnq(Register dst, Register src1, Register src2);
1031 void andnq(Register dst, Register src1, Address src2);
1032
1033 void blsil(Register dst, Register src);
1034 void blsil(Register dst, Address src);
1035 void blsiq(Register dst, Register src);
1036 void blsiq(Register dst, Address src);
1037
1038 void blsmskl(Register dst, Register src);
1039 void blsmskl(Register dst, Address src);
1040 void blsmskq(Register dst, Register src);
1041 void blsmskq(Register dst, Address src);
1042
1043 void blsrl(Register dst, Register src);
1044 void blsrl(Register dst, Address src);
1045 void blsrq(Register dst, Register src);
1046 void blsrq(Register dst, Address src);
1047
1048 void bsfl(Register dst, Register src);
1049 void bsrl(Register dst, Register src);
1050
1051#ifdef _LP641
1052 void bsfq(Register dst, Register src);
1053 void bsrq(Register dst, Register src);
1054#endif
1055
1056 void bswapl(Register reg);
1057
1058 void bswapq(Register reg);
1059
1060 void call(Label& L, relocInfo::relocType rtype);
1061 void call(Register reg); // push pc; pc <- reg
1062 void call(Address adr); // push pc; pc <- adr
1063
1064 void cdql();
1065
1066 void cdqq();
1067
1068 void cld();
1069
1070 void clflush(Address adr);
1071 void clflushopt(Address adr);
1072 void clwb(Address adr);
1073
1074 void cmovl(Condition cc, Register dst, Register src);
1075 void cmovl(Condition cc, Register dst, Address src);
1076
1077 void cmovq(Condition cc, Register dst, Register src);
1078 void cmovq(Condition cc, Register dst, Address src);
1079
1080
1081 void cmpb(Address dst, int imm8);
1082
1083 void cmpl(Address dst, int32_t imm32);
1084
1085 void cmp(Register dst, int32_t imm32);
1086 void cmpl(Register dst, int32_t imm32);
1087 void cmpl(Register dst, Register src);
1088 void cmpl(Register dst, Address src);
1089
1090 void cmpq(Address dst, int32_t imm32);
1091 void cmpq(Address dst, Register src);
1092
1093 void cmpq(Register dst, int32_t imm32);
1094 void cmpq(Register dst, Register src);
1095 void cmpq(Register dst, Address src);
1096
1097 // these are dummies used to catch attempting to convert NULL to Register
1098 void cmpl(Register dst, void* junk); // dummy
1099 void cmpq(Register dst, void* junk); // dummy
1100
1101 void cmpw(Address dst, int imm16);
1102
1103 void cmpxchg8 (Address adr);
1104
1105 void cmpxchgb(Register reg, Address adr);
1106 void cmpxchgl(Register reg, Address adr);
1107
1108 void cmpxchgq(Register reg, Address adr);
1109 void cmpxchgw(Register reg, Address adr);
1110
1111 // Ordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS
1112 void comisd(XMMRegister dst, Address src);
1113 void comisd(XMMRegister dst, XMMRegister src);
1114
1115 // Ordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS
1116 void comiss(XMMRegister dst, Address src);
1117 void comiss(XMMRegister dst, XMMRegister src);
1118
1119 // Identify processor type and features
1120 void cpuid();
1121
1122 // CRC32C
1123 void crc32(Register crc, Register v, int8_t sizeInBytes);
1124 void crc32(Register crc, Address adr, int8_t sizeInBytes);
1125
1126 // Convert Scalar Double-Precision Floating-Point Value to Scalar Single-Precision Floating-Point Value
1127 void cvtsd2ss(XMMRegister dst, XMMRegister src);
1128 void cvtsd2ss(XMMRegister dst, Address src);
1129
1130 // Convert Doubleword Integer to Scalar Double-Precision Floating-Point Value
1131 void cvtsi2sdl(XMMRegister dst, Register src);
1132 void cvtsi2sdl(XMMRegister dst, Address src);
1133 void cvtsi2sdq(XMMRegister dst, Register src);
1134 void cvtsi2sdq(XMMRegister dst, Address src);
1135
1136 // Convert Doubleword Integer to Scalar Single-Precision Floating-Point Value
1137 void cvtsi2ssl(XMMRegister dst, Register src);
1138 void cvtsi2ssl(XMMRegister dst, Address src);
1139 void cvtsi2ssq(XMMRegister dst, Register src);
1140 void cvtsi2ssq(XMMRegister dst, Address src);
1141
1142 // Convert Packed Signed Doubleword Integers to Packed Double-Precision Floating-Point Value
1143 void cvtdq2pd(XMMRegister dst, XMMRegister src);
1144 void vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len);
1145
1146 // Convert Packed Signed Doubleword Integers to Packed Single-Precision Floating-Point Value
1147 void cvtdq2ps(XMMRegister dst, XMMRegister src);
1148 void vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len);
1149
1150 // Convert Scalar Single-Precision Floating-Point Value to Scalar Double-Precision Floating-Point Value
1151 void cvtss2sd(XMMRegister dst, XMMRegister src);
1152 void cvtss2sd(XMMRegister dst, Address src);
1153
1154 // Convert with Truncation Scalar Double-Precision Floating-Point Value to Doubleword Integer
1155 void cvttsd2sil(Register dst, Address src);
1156 void cvttsd2sil(Register dst, XMMRegister src);
1157 void cvttsd2siq(Register dst, Address src);
1158 void cvttsd2siq(Register dst, XMMRegister src);
1159
1160 // Convert with Truncation Scalar Single-Precision Floating-Point Value to Doubleword Integer
1161 void cvttss2sil(Register dst, XMMRegister src);
1162 void cvttss2siq(Register dst, XMMRegister src);
1163
1164 // Convert vector double to int
1165 void cvttpd2dq(XMMRegister dst, XMMRegister src);
1166
1167 // Convert vector float and double
1168 void vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len);
1169 void vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len);
1170
1171 // Convert vector float and int
1172 void vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len);
1173
1174 // Convert vector long to vector FP
1175 void evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len);
1176 void evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len);
1177
1178 // Convert vector double to long
1179 void evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len);
1180
1181 // Evex casts with truncation
1182 void evpmovwb(XMMRegister dst, XMMRegister src, int vector_len);
1183 void evpmovdw(XMMRegister dst, XMMRegister src, int vector_len);
1184 void evpmovdb(XMMRegister dst, XMMRegister src, int vector_len);
1185 void evpmovqd(XMMRegister dst, XMMRegister src, int vector_len);
1186 void evpmovqb(XMMRegister dst, XMMRegister src, int vector_len);
1187 void evpmovqw(XMMRegister dst, XMMRegister src, int vector_len);
1188
1189 //Abs of packed Integer values
1190 void pabsb(XMMRegister dst, XMMRegister src);
1191 void pabsw(XMMRegister dst, XMMRegister src);
1192 void pabsd(XMMRegister dst, XMMRegister src);
1193 void vpabsb(XMMRegister dst, XMMRegister src, int vector_len);
1194 void vpabsw(XMMRegister dst, XMMRegister src, int vector_len);
1195 void vpabsd(XMMRegister dst, XMMRegister src, int vector_len);
1196 void evpabsq(XMMRegister dst, XMMRegister src, int vector_len);
1197
1198 // Divide Scalar Double-Precision Floating-Point Values
1199 void divsd(XMMRegister dst, Address src);
1200 void divsd(XMMRegister dst, XMMRegister src);
1201
1202 // Divide Scalar Single-Precision Floating-Point Values
1203 void divss(XMMRegister dst, Address src);
1204 void divss(XMMRegister dst, XMMRegister src);
1205
1206
1207#ifndef _LP641
1208 private:
1209
1210 void emit_farith(int b1, int b2, int i);
1211
1212 public:
1213 void emms();
1214
1215 void fabs();
1216
1217 void fadd(int i);
1218
1219 void fadd_d(Address src);
1220 void fadd_s(Address src);
1221
1222 // "Alternate" versions of x87 instructions place result down in FPU
1223 // stack instead of on TOS
1224
1225 void fadda(int i); // "alternate" fadd
1226 void faddp(int i = 1);
1227
1228 void fchs();
1229
1230 void fcom(int i);
1231
1232 void fcomp(int i = 1);
1233 void fcomp_d(Address src);
1234 void fcomp_s(Address src);
1235
1236 void fcompp();
1237
1238 void fcos();
1239
1240 void fdecstp();
1241
1242 void fdiv(int i);
1243 void fdiv_d(Address src);
1244 void fdivr_s(Address src);
1245 void fdiva(int i); // "alternate" fdiv
1246 void fdivp(int i = 1);
1247
1248 void fdivr(int i);
1249 void fdivr_d(Address src);
1250 void fdiv_s(Address src);
1251
1252 void fdivra(int i); // "alternate" reversed fdiv
1253
1254 void fdivrp(int i = 1);
1255
1256 void ffree(int i = 0);
1257
1258 void fild_d(Address adr);
1259 void fild_s(Address adr);
1260
1261 void fincstp();
1262
1263 void finit();
1264
1265 void fist_s (Address adr);
1266 void fistp_d(Address adr);
1267 void fistp_s(Address adr);
1268
1269 void fld1();
1270
1271 void fld_d(Address adr);
1272 void fld_s(Address adr);
1273 void fld_s(int index);
1274
1275 void fldcw(Address src);
1276
1277 void fldenv(Address src);
1278
1279 void fldlg2();
1280
1281 void fldln2();
1282
1283 void fldz();
1284
1285 void flog();
1286 void flog10();
1287
1288 void fmul(int i);
1289
1290 void fmul_d(Address src);
1291 void fmul_s(Address src);
1292
1293 void fmula(int i); // "alternate" fmul
1294
1295 void fmulp(int i = 1);
1296
1297 void fnsave(Address dst);
1298
1299 void fnstcw(Address src);
1300
1301 void fnstsw_ax();
1302
1303 void fprem();
1304 void fprem1();
1305
1306 void frstor(Address src);
1307
1308 void fsin();
1309
1310 void fsqrt();
1311
1312 void fst_d(Address adr);
1313 void fst_s(Address adr);
1314
1315 void fstp_d(Address adr);
1316 void fstp_d(int index);
1317 void fstp_s(Address adr);
1318
1319 void fsub(int i);
1320 void fsub_d(Address src);
1321 void fsub_s(Address src);
1322
1323 void fsuba(int i); // "alternate" fsub
1324
1325 void fsubp(int i = 1);
1326
1327 void fsubr(int i);
1328 void fsubr_d(Address src);
1329 void fsubr_s(Address src);
1330
1331 void fsubra(int i); // "alternate" reversed fsub
1332
1333 void fsubrp(int i = 1);
1334
1335 void ftan();
1336
1337 void ftst();
1338
1339 void fucomi(int i = 1);
1340 void fucomip(int i = 1);
1341
1342 void fwait();
1343
1344 void fxch(int i = 1);
1345
1346 void fyl2x();
1347 void frndint();
1348 void f2xm1();
1349 void fldl2e();
1350#endif // !_LP64
1351
1352 // operands that only take the original 32bit registers
1353 void emit_operand32(Register reg, Address adr);
1354
1355 void fld_x(Address adr); // extended-precision (80-bit) format
1356 void fstp_x(Address adr); // extended-precision (80-bit) format
1357 void fxrstor(Address src);
1358 void xrstor(Address src);
1359
1360 void fxsave(Address dst);
1361 void xsave(Address dst);
1362
1363 void hlt();
1364
1365 void idivl(Register src);
1366 void divl(Register src); // Unsigned division
1367
1368#ifdef _LP641
1369 void idivq(Register src);
1370#endif
1371
1372 void imull(Register src);
1373 void imull(Register dst, Register src);
1374 void imull(Register dst, Register src, int value);
1375 void imull(Register dst, Address src, int value);
1376 void imull(Register dst, Address src);
1377
1378#ifdef _LP641
1379 void imulq(Register dst, Register src);
1380 void imulq(Register dst, Register src, int value);
1381 void imulq(Register dst, Address src, int value);
1382 void imulq(Register dst, Address src);
1383 void imulq(Register dst);
1384#endif
1385
1386 // jcc is the generic conditional branch generator to run-
1387 // time routines, jcc is used for branches to labels. jcc
1388 // takes a branch opcode (cc) and a label (L) and generates
1389 // either a backward branch or a forward branch and links it
1390 // to the label fixup chain. Usage:
1391 //
1392 // Label L; // unbound label
1393 // jcc(cc, L); // forward branch to unbound label
1394 // bind(L); // bind label to the current pc
1395 // jcc(cc, L); // backward branch to bound label
1396 // bind(L); // illegal: a label may be bound only once
1397 //
1398 // Note: The same Label can be used for forward and backward branches
1399 // but it may be bound only once.
1400
1401 void jcc(Condition cc, Label& L, bool maybe_short = true);
1402
1403 // Conditional jump to a 8-bit offset to L.
1404 // WARNING: be very careful using this for forward jumps. If the label is
1405 // not bound within an 8-bit offset of this instruction, a run-time error
1406 // will occur.
1407
1408 // Use macro to record file and line number.
1409 #define jccb(cc, L)jccb_0(cc, L, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 1409) jccb_0(cc, L, __FILE__"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp", __LINE__1409)
1410
1411 void jccb_0(Condition cc, Label& L, const char* file, int line);
1412
1413 void jmp(Address entry); // pc <- entry
1414
1415 // Label operations & relative jumps (PPUM Appendix D)
1416 void jmp(Label& L, bool maybe_short = true); // unconditional jump to L
1417
1418 void jmp(Register entry); // pc <- entry
1419
1420 // Unconditional 8-bit offset jump to L.
1421 // WARNING: be very careful using this for forward jumps. If the label is
1422 // not bound within an 8-bit offset of this instruction, a run-time error
1423 // will occur.
1424
1425 // Use macro to record file and line number.
1426 #define jmpb(L)jmpb_0(L, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp"
, 1426) jmpb_0(L, __FILE__"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp", __LINE__1426)
1427
1428 void jmpb_0(Label& L, const char* file, int line);
1429
1430 void ldmxcsr( Address src );
1431
1432 void leal(Register dst, Address src);
1433
1434 void leaq(Register dst, Address src);
1435
1436 void lfence();
1437
1438 void lock();
1439 void size_prefix();
1440
1441 void lzcntl(Register dst, Register src);
1442
1443#ifdef _LP641
1444 void lzcntq(Register dst, Register src);
1445#endif
1446
1447 enum Membar_mask_bits {
1448 StoreStore = 1 << 3,
1449 LoadStore = 1 << 2,
1450 StoreLoad = 1 << 1,
1451 LoadLoad = 1 << 0
1452 };
1453
1454 // Serializes memory and blows flags
1455 void membar(Membar_mask_bits order_constraint);
1456
1457 void mfence();
1458 void sfence();
1459
1460 // Moves
1461
1462 void mov64(Register dst, int64_t imm64);
1463 void mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format);
1464
1465 void movb(Address dst, Register src);
1466 void movb(Address dst, int imm8);
1467 void movb(Register dst, Address src);
1468
1469 void movddup(XMMRegister dst, XMMRegister src);
1470
1471 void kandbl(KRegister dst, KRegister src1, KRegister src2);
1472 void kandwl(KRegister dst, KRegister src1, KRegister src2);
1473 void kanddl(KRegister dst, KRegister src1, KRegister src2);
1474 void kandql(KRegister dst, KRegister src1, KRegister src2);
1475
1476 void korbl(KRegister dst, KRegister src1, KRegister src2);
1477 void korwl(KRegister dst, KRegister src1, KRegister src2);
1478 void kordl(KRegister dst, KRegister src1, KRegister src2);
1479 void korql(KRegister dst, KRegister src1, KRegister src2);
1480
1481 void kxorbl(KRegister dst, KRegister src1, KRegister src2);
1482 void kxorwl(KRegister dst, KRegister src1, KRegister src2);
1483 void kxordl(KRegister dst, KRegister src1, KRegister src2);
1484 void kxorql(KRegister dst, KRegister src1, KRegister src2);
1485 void kmovbl(KRegister dst, Register src);
1486 void kmovbl(Register dst, KRegister src);
1487 void kmovbl(KRegister dst, KRegister src);
1488 void kmovwl(KRegister dst, Register src);
1489 void kmovwl(KRegister dst, Address src);
1490 void kmovwl(Register dst, KRegister src);
1491 void kmovwl(Address dst, KRegister src);
1492 void kmovwl(KRegister dst, KRegister src);
1493 void kmovdl(KRegister dst, Register src);
1494 void kmovdl(Register dst, KRegister src);
1495 void kmovql(KRegister dst, KRegister src);
1496 void kmovql(Address dst, KRegister src);
1497 void kmovql(KRegister dst, Address src);
1498 void kmovql(KRegister dst, Register src);
1499 void kmovql(Register dst, KRegister src);
1500
1501 void knotbl(KRegister dst, KRegister src);
1502 void knotwl(KRegister dst, KRegister src);
1503 void knotdl(KRegister dst, KRegister src);
1504 void knotql(KRegister dst, KRegister src);
1505
1506 void kortestbl(KRegister dst, KRegister src);
1507 void kortestwl(KRegister dst, KRegister src);
1508 void kortestdl(KRegister dst, KRegister src);
1509 void kortestql(KRegister dst, KRegister src);
1510
1511 void kxnorbl(KRegister dst, KRegister src1, KRegister src2);
1512 void kshiftlbl(KRegister dst, KRegister src, int imm8);
1513 void kshiftrbl(KRegister dst, KRegister src, int imm8);
1514 void kshiftrwl(KRegister dst, KRegister src, int imm8);
1515 void kshiftrdl(KRegister dst, KRegister src, int imm8);
1516 void kshiftrql(KRegister dst, KRegister src, int imm8);
1517 void ktestq(KRegister src1, KRegister src2);
1518 void ktestd(KRegister src1, KRegister src2);
1519
1520 void ktestql(KRegister dst, KRegister src);
1521 void ktestdl(KRegister dst, KRegister src);
1522 void ktestwl(KRegister dst, KRegister src);
1523 void ktestbl(KRegister dst, KRegister src);
1524
1525 void movdl(XMMRegister dst, Register src);
1526 void movdl(Register dst, XMMRegister src);
1527 void movdl(XMMRegister dst, Address src);
1528 void movdl(Address dst, XMMRegister src);
1529
1530 // Move Double Quadword
1531 void movdq(XMMRegister dst, Register src);
1532 void movdq(Register dst, XMMRegister src);
1533
1534 // Move Aligned Double Quadword
1535 void movdqa(XMMRegister dst, XMMRegister src);
1536 void movdqa(XMMRegister dst, Address src);
1537
1538 // Move Unaligned Double Quadword
1539 void movdqu(Address dst, XMMRegister src);
1540 void movdqu(XMMRegister dst, Address src);
1541 void movdqu(XMMRegister dst, XMMRegister src);
1542
1543 // Move Unaligned 256bit Vector
1544 void vmovdqu(Address dst, XMMRegister src);
1545 void vmovdqu(XMMRegister dst, Address src);
1546 void vmovdqu(XMMRegister dst, XMMRegister src);
1547
1548 // Move Unaligned 512bit Vector
1549 void evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len);
1550 void evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len);
1551 void evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len);
1552 void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
1553 void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
1554 void evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len);
1555 void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
1556 void evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len);
1557 void evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
1558 void evmovdqul(Address dst, XMMRegister src, int vector_len);
1559 void evmovdqul(XMMRegister dst, Address src, int vector_len);
1560 void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len);
1561 void evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
1562 void evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
1563 void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
1564 void evmovdquq(Address dst, XMMRegister src, int vector_len);
1565 void evmovdquq(XMMRegister dst, Address src, int vector_len);
1566 void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len);
1567 void evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
1568 void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
1569 void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
1570
1571 // Move lower 64bit to high 64bit in 128bit register
1572 void movlhps(XMMRegister dst, XMMRegister src);
1573
1574 void movl(Register dst, int32_t imm32);
1575 void movl(Address dst, int32_t imm32);
1576 void movl(Register dst, Register src);
1577 void movl(Register dst, Address src);
1578 void movl(Address dst, Register src);
1579
1580 // These dummies prevent using movl from converting a zero (like NULL) into Register
1581 // by giving the compiler two choices it can't resolve
1582
1583 void movl(Address dst, void* junk);
1584 void movl(Register dst, void* junk);
1585
1586#ifdef _LP641
1587 void movq(Register dst, Register src);
1588 void movq(Register dst, Address src);
1589 void movq(Address dst, Register src);
1590 void movq(Address dst, int32_t imm32);
1591 void movq(Register dst, int32_t imm32);
1592
1593 // These dummies prevent using movq from converting a zero (like NULL) into Register
1594 // by giving the compiler two choices it can't resolve
1595
1596 void movq(Address dst, void* dummy);
1597 void movq(Register dst, void* dummy);
1598#endif
1599
1600 // Move Quadword
1601 void movq(Address dst, XMMRegister src);
1602 void movq(XMMRegister dst, Address src);
1603 void movq(XMMRegister dst, XMMRegister src);
1604 void movq(Register dst, XMMRegister src);
1605 void movq(XMMRegister dst, Register src);
1606
1607 void movsbl(Register dst, Address src);
1608 void movsbl(Register dst, Register src);
1609
1610#ifdef _LP641
1611 void movsbq(Register dst, Address src);
1612 void movsbq(Register dst, Register src);
1613
1614 // Move signed 32bit immediate to 64bit extending sign
1615 void movslq(Address dst, int32_t imm64);
1616 void movslq(Register dst, int32_t imm64);
1617
1618 void movslq(Register dst, Address src);
1619 void movslq(Register dst, Register src);
1620 void movslq(Register dst, void* src); // Dummy declaration to cause NULL to be ambiguous
1621#endif
1622
1623 void movswl(Register dst, Address src);
1624 void movswl(Register dst, Register src);
1625
1626#ifdef _LP641
1627 void movswq(Register dst, Address src);
1628 void movswq(Register dst, Register src);
1629#endif
1630
1631 void movw(Address dst, int imm16);
1632 void movw(Register dst, Address src);
1633 void movw(Address dst, Register src);
1634
1635 void movzbl(Register dst, Address src);
1636 void movzbl(Register dst, Register src);
1637
1638#ifdef _LP641
1639 void movzbq(Register dst, Address src);
1640 void movzbq(Register dst, Register src);
1641#endif
1642
1643 void movzwl(Register dst, Address src);
1644 void movzwl(Register dst, Register src);
1645
1646#ifdef _LP641
1647 void movzwq(Register dst, Address src);
1648 void movzwq(Register dst, Register src);
1649#endif
1650
1651 // Unsigned multiply with RAX destination register
1652 void mull(Address src);
1653 void mull(Register src);
1654
1655#ifdef _LP641
1656 void mulq(Address src);
1657 void mulq(Register src);
1658 void mulxq(Register dst1, Register dst2, Register src);
1659#endif
1660
1661 // Multiply Scalar Double-Precision Floating-Point Values
1662 void mulsd(XMMRegister dst, Address src);
1663 void mulsd(XMMRegister dst, XMMRegister src);
1664
1665 // Multiply Scalar Single-Precision Floating-Point Values
1666 void mulss(XMMRegister dst, Address src);
1667 void mulss(XMMRegister dst, XMMRegister src);
1668
1669 void negl(Register dst);
1670 void negl(Address dst);
1671
1672#ifdef _LP641
1673 void negq(Register dst);
1674 void negq(Address dst);
1675#endif
1676
1677 void nop(int i = 1);
1678
1679 void notl(Register dst);
1680
1681#ifdef _LP641
1682 void notq(Register dst);
1683
1684 void btsq(Address dst, int imm8);
1685 void btrq(Address dst, int imm8);
1686#endif
1687
1688 void orw(Register dst, Register src);
1689
1690 void orl(Address dst, int32_t imm32);
1691 void orl(Register dst, int32_t imm32);
1692 void orl(Register dst, Address src);
1693 void orl(Register dst, Register src);
1694 void orl(Address dst, Register src);
1695
1696 void orb(Address dst, int imm8);
1697 void orb(Address dst, Register src);
1698
1699 void orq(Address dst, int32_t imm32);
1700 void orq(Address dst, Register src);
1701 void orq(Register dst, int32_t imm32);
1702 void orq(Register dst, Address src);
1703 void orq(Register dst, Register src);
1704
1705 // Pack with signed saturation
1706 void packsswb(XMMRegister dst, XMMRegister src);
1707 void vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1708 void packssdw(XMMRegister dst, XMMRegister src);
1709 void vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1710
1711 // Pack with unsigned saturation
1712 void packuswb(XMMRegister dst, XMMRegister src);
1713 void packuswb(XMMRegister dst, Address src);
1714 void packusdw(XMMRegister dst, XMMRegister src);
1715 void vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1716 void vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1717
1718 // Permutations
1719 void vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
1720 void vpermq(XMMRegister dst, XMMRegister src, int imm8);
1721 void vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1722 void vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1723 void vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1724 void vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1725 void vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1726 void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1727 void vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
1728 void vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
1729 void vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
1730 void vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
1731 void vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len);
1732 void evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1733 void evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1734 void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len);
1735
1736 void pause();
1737
1738 // Undefined Instruction
1739 void ud2();
1740
1741 // SSE4.2 string instructions
1742 void pcmpestri(XMMRegister xmm1, XMMRegister xmm2, int imm8);
1743 void pcmpestri(XMMRegister xmm1, Address src, int imm8);
1744
1745 void pcmpeqb(XMMRegister dst, XMMRegister src);
1746 void vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len);
1747
1748 void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1749 void evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len);
1750 void evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len);
1751 void evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len);
1752
1753 void vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1754 void evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len);
1755 void evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len);
1756
1757 void evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len);
1758 void evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len);
1759
1760 void pcmpeqw(XMMRegister dst, XMMRegister src);
1761 void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1762 void evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len);
1763 void evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len);
1764
1765 void vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1766
1767 void pcmpeqd(XMMRegister dst, XMMRegister src);
1768 void vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1769 void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len);
1770 void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len);
1771
1772 void pcmpeqq(XMMRegister dst, XMMRegister src);
1773 void vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len);
1774 void vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1775 void evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len);
1776 void evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len);
1777
1778 void pcmpgtq(XMMRegister dst, XMMRegister src);
1779 void vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1780
1781 void pmovmskb(Register dst, XMMRegister src);
1782 void vpmovmskb(Register dst, XMMRegister src, int vec_enc);
1783 void vmovmskps(Register dst, XMMRegister src, int vec_enc);
1784 void vmovmskpd(Register dst, XMMRegister src, int vec_enc);
1785 void vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1786
1787 // SSE 4.1 extract
1788 void pextrd(Register dst, XMMRegister src, int imm8);
1789 void pextrq(Register dst, XMMRegister src, int imm8);
1790 void pextrd(Address dst, XMMRegister src, int imm8);
1791 void pextrq(Address dst, XMMRegister src, int imm8);
1792 void pextrb(Register dst, XMMRegister src, int imm8);
1793 void pextrb(Address dst, XMMRegister src, int imm8);
1794 // SSE 2 extract
1795 void pextrw(Register dst, XMMRegister src, int imm8);
1796 void pextrw(Address dst, XMMRegister src, int imm8);
1797
1798 // SSE 4.1 insert
1799 void pinsrd(XMMRegister dst, Register src, int imm8);
1800 void pinsrq(XMMRegister dst, Register src, int imm8);
1801 void pinsrb(XMMRegister dst, Register src, int imm8);
1802 void pinsrd(XMMRegister dst, Address src, int imm8);
1803 void pinsrq(XMMRegister dst, Address src, int imm8);
1804 void pinsrb(XMMRegister dst, Address src, int imm8);
1805 void insertps(XMMRegister dst, XMMRegister src, int imm8);
1806 // SSE 2 insert
1807 void pinsrw(XMMRegister dst, Register src, int imm8);
1808 void pinsrw(XMMRegister dst, Address src, int imm8);
1809
1810 // AVX insert
1811 void vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8);
1812 void vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8);
1813 void vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8);
1814 void vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8);
1815 void vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8);
1816
1817 // Zero extend moves
1818 void pmovzxbw(XMMRegister dst, XMMRegister src);
1819 void pmovzxbw(XMMRegister dst, Address src);
1820 void pmovzxbd(XMMRegister dst, XMMRegister src);
1821 void vpmovzxbw( XMMRegister dst, Address src, int vector_len);
1822 void pmovzxdq(XMMRegister dst, XMMRegister src);
1823 void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len);
1824 void vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len);
1825 void vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len);
1826 void vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len);
1827 void evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len);
1828
1829 // Sign extend moves
1830 void pmovsxbd(XMMRegister dst, XMMRegister src);
1831 void pmovsxbq(XMMRegister dst, XMMRegister src);
1832 void pmovsxbw(XMMRegister dst, XMMRegister src);
1833 void pmovsxwd(XMMRegister dst, XMMRegister src);
1834 void vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len);
1835 void vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len);
1836 void vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len);
1837 void vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len);
1838 void vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len);
1839 void vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len);
1840
1841 void evpmovwb(Address dst, XMMRegister src, int vector_len);
1842 void evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len);
1843
1844 void vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len);
1845
1846 void evpmovdb(Address dst, XMMRegister src, int vector_len);
1847
1848 // Multiply add
1849 void pmaddwd(XMMRegister dst, XMMRegister src);
1850 void vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1851 void vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
1852
1853 // Multiply add accumulate
1854 void evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1855
1856#ifndef _LP641 // no 32bit push/pop on amd64
1857 void popl(Address dst);
1858#endif
1859
1860#ifdef _LP641
1861 void popq(Address dst);
1862 void popq(Register dst);
1863#endif
1864
1865 void popcntl(Register dst, Address src);
1866 void popcntl(Register dst, Register src);
1867
1868 void vpopcntd(XMMRegister dst, XMMRegister src, int vector_len);
1869
1870#ifdef _LP641
1871 void popcntq(Register dst, Address src);
1872 void popcntq(Register dst, Register src);
1873#endif
1874
1875 // Prefetches (SSE, SSE2, 3DNOW only)
1876
1877 void prefetchnta(Address src);
1878 void prefetchr(Address src);
1879 void prefetcht0(Address src);
1880 void prefetcht1(Address src);
1881 void prefetcht2(Address src);
1882 void prefetchw(Address src);
1883
1884 // Shuffle Bytes
1885 void pshufb(XMMRegister dst, XMMRegister src);
1886 void pshufb(XMMRegister dst, Address src);
1887 void vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1888
1889 // Shuffle Packed Doublewords
1890 void pshufd(XMMRegister dst, XMMRegister src, int mode);
1891 void pshufd(XMMRegister dst, Address src, int mode);
1892 void vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len);
1893
1894 // Shuffle Packed High/Low Words
1895 void pshufhw(XMMRegister dst, XMMRegister src, int mode);
1896 void pshuflw(XMMRegister dst, XMMRegister src, int mode);
1897 void pshuflw(XMMRegister dst, Address src, int mode);
1898
1899 //shuffle floats and doubles
1900 void pshufps(XMMRegister, XMMRegister, int);
1901 void pshufpd(XMMRegister, XMMRegister, int);
1902 void vpshufps(XMMRegister, XMMRegister, XMMRegister, int, int);
1903 void vpshufpd(XMMRegister, XMMRegister, XMMRegister, int, int);
1904
1905 // Shuffle packed values at 128 bit granularity
1906 void evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len);
1907
1908 // Shift Right by bytes Logical DoubleQuadword Immediate
1909 void psrldq(XMMRegister dst, int shift);
1910 // Shift Left by bytes Logical DoubleQuadword Immediate
1911 void pslldq(XMMRegister dst, int shift);
1912
1913 // Logical Compare 128bit
1914 void ptest(XMMRegister dst, XMMRegister src);
1915 void ptest(XMMRegister dst, Address src);
1916 // Logical Compare 256bit
1917 void vptest(XMMRegister dst, XMMRegister src);
1918 void vptest(XMMRegister dst, Address src);
1919
1920 void evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1921
1922 // Vector compare
1923 void vptest(XMMRegister dst, XMMRegister src, int vector_len);
1924
1925 // Interleave Low Bytes
1926 void punpcklbw(XMMRegister dst, XMMRegister src);
1927 void punpcklbw(XMMRegister dst, Address src);
1928
1929 // Interleave Low Doublewords
1930 void punpckldq(XMMRegister dst, XMMRegister src);
1931 void punpckldq(XMMRegister dst, Address src);
1932
1933 // Interleave Low Quadwords
1934 void punpcklqdq(XMMRegister dst, XMMRegister src);
1935
1936#ifndef _LP641 // no 32bit push/pop on amd64
1937 void pushl(Address src);
1938#endif
1939
1940 void pushq(Address src);
1941
1942 void rcll(Register dst, int imm8);
1943
1944 void rclq(Register dst, int imm8);
1945
1946 void rcrq(Register dst, int imm8);
1947
1948 void rcpps(XMMRegister dst, XMMRegister src);
1949
1950 void rcpss(XMMRegister dst, XMMRegister src);
1951
1952 void rdtsc();
1953
1954 void ret(int imm16);
1955
1956 void roll(Register dst);
1957
1958 void roll(Register dst, int imm8);
1959
1960 void rorl(Register dst);
1961
1962 void rorl(Register dst, int imm8);
1963
1964#ifdef _LP641
1965 void rolq(Register dst);
1966 void rolq(Register dst, int imm8);
1967 void rorq(Register dst);
1968 void rorq(Register dst, int imm8);
1969 void rorxq(Register dst, Register src, int imm8);
1970 void rorxd(Register dst, Register src, int imm8);
1971#endif
1972
1973 void sahf();
1974
1975 void sall(Register dst, int imm8);
1976 void sall(Register dst);
1977 void sall(Address dst, int imm8);
1978 void sall(Address dst);
1979
1980 void sarl(Address dst, int imm8);
1981 void sarl(Address dst);
1982 void sarl(Register dst, int imm8);
1983 void sarl(Register dst);
1984
1985#ifdef _LP641
1986 void salq(Register dst, int imm8);
1987 void salq(Register dst);
1988 void salq(Address dst, int imm8);
1989 void salq(Address dst);
1990
1991 void sarq(Address dst, int imm8);
1992 void sarq(Address dst);
1993 void sarq(Register dst, int imm8);
1994 void sarq(Register dst);
1995#endif
1996
1997 void sbbl(Address dst, int32_t imm32);
1998 void sbbl(Register dst, int32_t imm32);
1999 void sbbl(Register dst, Address src);
2000 void sbbl(Register dst, Register src);
2001
2002 void sbbq(Address dst, int32_t imm32);
2003 void sbbq(Register dst, int32_t imm32);
2004 void sbbq(Register dst, Address src);
2005 void sbbq(Register dst, Register src);
2006
2007 void setb(Condition cc, Register dst);
2008
2009 void sete(Register dst);
2010 void setl(Register dst);
2011 void setne(Register dst);
2012
2013 void palignr(XMMRegister dst, XMMRegister src, int imm8);
2014 void vpalignr(XMMRegister dst, XMMRegister src1, XMMRegister src2, int imm8, int vector_len);
2015 void evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2016
2017 void pblendw(XMMRegister dst, XMMRegister src, int imm8);
2018 void vblendps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int imm8, int vector_len);
2019
2020 void sha1rnds4(XMMRegister dst, XMMRegister src, int imm8);
2021 void sha1nexte(XMMRegister dst, XMMRegister src);
2022 void sha1msg1(XMMRegister dst, XMMRegister src);
2023 void sha1msg2(XMMRegister dst, XMMRegister src);
2024 // xmm0 is implicit additional source to the following instruction.
2025 void sha256rnds2(XMMRegister dst, XMMRegister src);
2026 void sha256msg1(XMMRegister dst, XMMRegister src);
2027 void sha256msg2(XMMRegister dst, XMMRegister src);
2028
2029 void shldl(Register dst, Register src);
2030 void shldl(Register dst, Register src, int8_t imm8);
2031 void shrdl(Register dst, Register src);
2032 void shrdl(Register dst, Register src, int8_t imm8);
2033
2034 void shll(Register dst, int imm8);
2035 void shll(Register dst);
2036
2037 void shlq(Register dst, int imm8);
2038 void shlq(Register dst);
2039
2040 void shrl(Register dst, int imm8);
2041 void shrl(Register dst);
2042 void shrl(Address dst);
2043 void shrl(Address dst, int imm8);
2044
2045 void shrq(Register dst, int imm8);
2046 void shrq(Register dst);
2047 void shrq(Address dst);
2048 void shrq(Address dst, int imm8);
2049
2050 void smovl(); // QQQ generic?
2051
2052 // Compute Square Root of Scalar Double-Precision Floating-Point Value
2053 void sqrtsd(XMMRegister dst, Address src);
2054 void sqrtsd(XMMRegister dst, XMMRegister src);
2055
2056 void roundsd(XMMRegister dst, Address src, int32_t rmode);
2057 void roundsd(XMMRegister dst, XMMRegister src, int32_t rmode);
2058
2059 // Compute Square Root of Scalar Single-Precision Floating-Point Value
2060 void sqrtss(XMMRegister dst, Address src);
2061 void sqrtss(XMMRegister dst, XMMRegister src);
2062
2063 void std();
2064
2065 void stmxcsr( Address dst );
2066
2067 void subl(Address dst, int32_t imm32);
2068 void subl(Address dst, Register src);
2069 void subl(Register dst, int32_t imm32);
2070 void subl(Register dst, Address src);
2071 void subl(Register dst, Register src);
2072
2073 void subq(Address dst, int32_t imm32);
2074 void subq(Address dst, Register src);
2075 void subq(Register dst, int32_t imm32);
2076 void subq(Register dst, Address src);
2077 void subq(Register dst, Register src);
2078
2079 // Force generation of a 4 byte immediate value even if it fits into 8bit
2080 void subl_imm32(Register dst, int32_t imm32);
2081 void subq_imm32(Register dst, int32_t imm32);
2082
2083 // Subtract Scalar Double-Precision Floating-Point Values
2084 void subsd(XMMRegister dst, Address src);
2085 void subsd(XMMRegister dst, XMMRegister src);
2086
2087 // Subtract Scalar Single-Precision Floating-Point Values
2088 void subss(XMMRegister dst, Address src);
2089 void subss(XMMRegister dst, XMMRegister src);
2090
2091 void testb(Register dst, int imm8);
2092 void testb(Address dst, int imm8);
2093
2094 void testl(Register dst, int32_t imm32);
2095 void testl(Register dst, Register src);
2096 void testl(Register dst, Address src);
2097
2098 void testq(Address dst, int32_t imm32);
2099 void testq(Register dst, int32_t imm32);
2100 void testq(Register dst, Register src);
2101 void testq(Register dst, Address src);
2102
2103 // BMI - count trailing zeros
2104 void tzcntl(Register dst, Register src);
2105 void tzcntq(Register dst, Register src);
2106
2107 // Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS
2108 void ucomisd(XMMRegister dst, Address src);
2109 void ucomisd(XMMRegister dst, XMMRegister src);
2110
2111 // Unordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS
2112 void ucomiss(XMMRegister dst, Address src);
2113 void ucomiss(XMMRegister dst, XMMRegister src);
2114
2115 void xabort(int8_t imm8);
2116
2117 void xaddb(Address dst, Register src);
2118 void xaddw(Address dst, Register src);
2119 void xaddl(Address dst, Register src);
2120 void xaddq(Address dst, Register src);
2121
2122 void xbegin(Label& abort, relocInfo::relocType rtype = relocInfo::none);
2123
2124 void xchgb(Register reg, Address adr);
2125 void xchgw(Register reg, Address adr);
2126 void xchgl(Register reg, Address adr);
2127 void xchgl(Register dst, Register src);
2128
2129 void xchgq(Register reg, Address adr);
2130 void xchgq(Register dst, Register src);
2131
2132 void xend();
2133
2134 // Get Value of Extended Control Register
2135 void xgetbv();
2136
2137 void xorl(Register dst, int32_t imm32);
2138 void xorl(Address dst, int32_t imm32);
2139 void xorl(Register dst, Address src);
2140 void xorl(Register dst, Register src);
2141 void xorl(Address dst, Register src);
2142
2143 void xorb(Address dst, Register src);
2144 void xorb(Register dst, Address src);
2145 void xorw(Register dst, Register src);
2146
2147 void xorq(Register dst, Address src);
2148 void xorq(Address dst, int32_t imm32);
2149 void xorq(Register dst, Register src);
2150 void xorq(Register dst, int32_t imm32);
2151 void xorq(Address dst, Register src);
2152
2153 void set_byte_if_not_zero(Register dst); // sets reg to 1 if not zero, otherwise 0
2154
2155 // AVX 3-operands scalar instructions (encoded with VEX prefix)
2156
2157 void vaddsd(XMMRegister dst, XMMRegister nds, Address src);
2158 void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2159 void vaddss(XMMRegister dst, XMMRegister nds, Address src);
2160 void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2161 void vdivsd(XMMRegister dst, XMMRegister nds, Address src);
2162 void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2163 void vdivss(XMMRegister dst, XMMRegister nds, Address src);
2164 void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2165 void vfmadd231sd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2166 void vfmadd231ss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2167 void vmulsd(XMMRegister dst, XMMRegister nds, Address src);
2168 void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2169 void vmulss(XMMRegister dst, XMMRegister nds, Address src);
2170 void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2171 void vsubsd(XMMRegister dst, XMMRegister nds, Address src);
2172 void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2173 void vsubss(XMMRegister dst, XMMRegister nds, Address src);
2174 void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2175
2176 void vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2177 void vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2178 void vminss(XMMRegister dst, XMMRegister nds, XMMRegister src);
2179 void vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src);
2180
2181 void shlxl(Register dst, Register src1, Register src2);
2182 void shlxq(Register dst, Register src1, Register src2);
2183 void shrxl(Register dst, Register src1, Register src2);
2184 void shrxq(Register dst, Register src1, Register src2);
2185
2186 void bzhiq(Register dst, Register src1, Register src2);
2187 void pdep(Register dst, Register src1, Register src2);
2188 void pext(Register dst, Register src1, Register src2);
2189
2190
2191 //====================VECTOR ARITHMETIC=====================================
2192 // Add Packed Floating-Point Values
2193 void addpd(XMMRegister dst, XMMRegister src);
2194 void addpd(XMMRegister dst, Address src);
2195 void addps(XMMRegister dst, XMMRegister src);
2196 void vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2197 void vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2198 void vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2199 void vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2200
2201 // Subtract Packed Floating-Point Values
2202 void subpd(XMMRegister dst, XMMRegister src);
2203 void subps(XMMRegister dst, XMMRegister src);
2204 void vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2205 void vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2206 void vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2207 void vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2208
2209 // Multiply Packed Floating-Point Values
2210 void mulpd(XMMRegister dst, XMMRegister src);
2211 void mulpd(XMMRegister dst, Address src);
2212 void mulps(XMMRegister dst, XMMRegister src);
2213 void vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2214 void vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2215 void vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2216 void vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2217
2218 void vfmadd231pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2219 void vfmadd231ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2220 void vfmadd231pd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2221 void vfmadd231ps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2222
2223 // Divide Packed Floating-Point Values
2224 void divpd(XMMRegister dst, XMMRegister src);
2225 void divps(XMMRegister dst, XMMRegister src);
2226 void vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2227 void vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2228 void vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2229 void vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2230
2231 // Sqrt Packed Floating-Point Values
2232 void vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len);
2233 void vsqrtpd(XMMRegister dst, Address src, int vector_len);
2234 void vsqrtps(XMMRegister dst, XMMRegister src, int vector_len);
2235 void vsqrtps(XMMRegister dst, Address src, int vector_len);
2236
2237 // Round Packed Double precision value.
2238 void vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len);
2239 void vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len);
2240 void vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len);
2241 void vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len);
2242
2243 // Bitwise Logical AND of Packed Floating-Point Values
2244 void andpd(XMMRegister dst, XMMRegister src);
2245 void andps(XMMRegister dst, XMMRegister src);
2246 void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2247 void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2248 void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2249 void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2250
2251 void unpckhpd(XMMRegister dst, XMMRegister src);
2252 void unpcklpd(XMMRegister dst, XMMRegister src);
2253
2254 // Bitwise Logical XOR of Packed Floating-Point Values
2255 void xorpd(XMMRegister dst, XMMRegister src);
2256 void xorps(XMMRegister dst, XMMRegister src);
2257 void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2258 void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2259 void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2260 void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2261
2262 // Add horizontal packed integers
2263 void vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2264 void vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2265 void phaddw(XMMRegister dst, XMMRegister src);
2266 void phaddd(XMMRegister dst, XMMRegister src);
2267
2268 // Add packed integers
2269 void paddb(XMMRegister dst, XMMRegister src);
2270 void paddw(XMMRegister dst, XMMRegister src);
2271 void paddd(XMMRegister dst, XMMRegister src);
2272 void paddd(XMMRegister dst, Address src);
2273 void paddq(XMMRegister dst, XMMRegister src);
2274 void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2275 void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2276 void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2277 void vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2278 void vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2279 void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2280 void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2281 void vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2282
2283 // Leaf level assembler routines for masked operations.
2284 void evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2285 void evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2286 void evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2287 void evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2288 void evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2289 void evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2290 void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2291 void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2292 void evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2293 void evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2294 void evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2295 void evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2296 void evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2297 void evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2298 void evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2299 void evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2300 void evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2301 void evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2302 void evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2303 void evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2304 void evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2305 void evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2306 void evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2307 void evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2308 void evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2309 void evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2310 void evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2311 void evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2312 void evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2313 void evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2314 void evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2315 void evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2316 void evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2317 void evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2318 void evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2319 void evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2320 void evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2321 void evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2322 void evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
2323 void evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
2324 void evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
2325 void evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
2326 void evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
2327 void evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
2328 void evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len);
2329 void evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
2330 void evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2331 void evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2332 void evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2333 void evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2334 void evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2335 void evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2336 void evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2337 void evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2338 void evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2339 void evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2340 void evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2341 void evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2342 void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2343 void evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2344 void evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2345 void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2346 void evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2347 void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2348 void evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2349 void evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2350 void evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2351 void evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2352 void evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2353 void evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2354 void evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2355
2356 void evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2357 void evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2358 void evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2359 void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2360 void evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2361 void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2362 void evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2363 void evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2364 void evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2365
2366 void evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2367 void evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2368 void evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2369 void evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2370 void evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2371 void evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2372 void evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2373 void evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2374 void evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2375 void evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2376 void evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2377 void evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2378 void evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2379 void evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2380 void evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2381 void evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2382 void evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2383 void evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2384 void evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2385 void evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2386 void evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2387 void evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2388 void evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2389 void evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2390 void evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2391 void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2392 void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2393 void evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2394 void evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2395 void evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2396 void evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2397 void evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2398 void evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2399 void evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2400 void evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2401 void evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2402 void evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len);
2403
2404 void evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2405 void evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2406 void evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2407 void evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2408 void evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2409 void evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len);
2410 void evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2411 void evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2412
2413 // Sub packed integers
2414 void psubb(XMMRegister dst, XMMRegister src);
2415 void psubw(XMMRegister dst, XMMRegister src);
2416 void psubd(XMMRegister dst, XMMRegister src);
2417 void psubq(XMMRegister dst, XMMRegister src);
2418 void vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2419 void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2420 void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2421 void vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2422 void vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2423 void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2424 void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2425 void vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2426 void vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2427
2428 // Multiply packed integers (only shorts and ints)
2429 void pmullw(XMMRegister dst, XMMRegister src);
2430 void pmulld(XMMRegister dst, XMMRegister src);
2431 void pmuludq(XMMRegister dst, XMMRegister src);
2432 void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2433 void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2434 void vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2435 void vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2436 void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2437 void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2438 void vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2439 void vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2440
2441 // Minimum of packed integers
2442 void pminsb(XMMRegister dst, XMMRegister src);
2443 void vpminsb(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2444 void pminsw(XMMRegister dst, XMMRegister src);
2445 void vpminsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2446 void pminsd(XMMRegister dst, XMMRegister src);
2447 void vpminsd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2448 void vpminsq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2449 void minps(XMMRegister dst, XMMRegister src);
2450 void vminps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2451 void minpd(XMMRegister dst, XMMRegister src);
2452 void vminpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2453
2454 // Maximum of packed integers
2455 void pmaxsb(XMMRegister dst, XMMRegister src);
2456 void vpmaxsb(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2457 void pmaxsw(XMMRegister dst, XMMRegister src);
2458 void vpmaxsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2459 void pmaxsd(XMMRegister dst, XMMRegister src);
2460 void vpmaxsd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2461 void vpmaxsq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2462 void maxps(XMMRegister dst, XMMRegister src);
2463 void vmaxps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2464 void maxpd(XMMRegister dst, XMMRegister src);
2465 void vmaxpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len);
2466
2467 // Shift left packed integers
2468 void psllw(XMMRegister dst, int shift);
2469 void pslld(XMMRegister dst, int shift);
2470 void psllq(XMMRegister dst, int shift);
2471 void psllw(XMMRegister dst, XMMRegister shift);
2472 void pslld(XMMRegister dst, XMMRegister shift);
2473 void psllq(XMMRegister dst, XMMRegister shift);
2474 void vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2475 void vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2476 void vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2477 void vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2478 void vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2479 void vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2480 void vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2481
2482 // Logical shift right packed integers
2483 void psrlw(XMMRegister dst, int shift);
2484 void psrld(XMMRegister dst, int shift);
2485 void psrlq(XMMRegister dst, int shift);
2486 void psrlw(XMMRegister dst, XMMRegister shift);
2487 void psrld(XMMRegister dst, XMMRegister shift);
2488 void psrlq(XMMRegister dst, XMMRegister shift);
2489 void vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2490 void vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2491 void vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2492 void vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2493 void vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2494 void vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2495 void vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2496 void evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2497 void evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2498
2499 // Arithmetic shift right packed integers (only shorts and ints, no instructions for longs)
2500 void psraw(XMMRegister dst, int shift);
2501 void psrad(XMMRegister dst, int shift);
2502 void psraw(XMMRegister dst, XMMRegister shift);
2503 void psrad(XMMRegister dst, XMMRegister shift);
2504 void vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2505 void vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2506 void vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2507 void vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2508 void evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2509 void evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2510 void evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2511
2512 // Variable shift left packed integers
2513 void vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2514 void vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2515
2516 // Variable shift right packed integers
2517 void vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2518 void vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2519
2520 // Variable shift right arithmetic packed integers
2521 void vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2522 void evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2523
2524 void vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2525 void vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2526
2527 // And packed integers
2528 void pand(XMMRegister dst, XMMRegister src);
2529 void vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2530 void vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2531 void vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2532
2533 // Andn packed integers
2534 void pandn(XMMRegister dst, XMMRegister src);
2535 void vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2536
2537 // Or packed integers
2538 void por(XMMRegister dst, XMMRegister src);
2539 void vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2540 void vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2541 void vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2542
2543 // Xor packed integers
2544 void pxor(XMMRegister dst, XMMRegister src);
2545 void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2546 void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2547 void vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2548 void evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2549 void evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
2550
2551 // Ternary logic instruction.
2552 void vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len);
2553 void vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len);
2554 void vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len);
2555
2556 // Vector Rotate Left/Right instruction.
2557 void evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2558 void evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2559 void evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2560 void evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len);
2561 void evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2562 void evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2563 void evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2564 void evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len);
2565
2566 // vinserti forms
2567 void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2568 void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8);
2569 void vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2570 void vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8);
2571 void vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2572
2573 // vinsertf forms
2574 void vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2575 void vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8);
2576 void vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2577 void vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8);
2578 void vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8);
2579 void vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8);
2580
2581 // vextracti forms
2582 void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8);
2583 void vextracti128(Address dst, XMMRegister src, uint8_t imm8);
2584 void vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8);
2585 void vextracti32x4(Address dst, XMMRegister src, uint8_t imm8);
2586 void vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8);
2587 void vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8);
2588 void vextracti64x4(Address dst, XMMRegister src, uint8_t imm8);
2589
2590 // vextractf forms
2591 void vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8);
2592 void vextractf128(Address dst, XMMRegister src, uint8_t imm8);
2593 void vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8);
2594 void vextractf32x4(Address dst, XMMRegister src, uint8_t imm8);
2595 void vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8);
2596 void vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8);
2597 void vextractf64x4(Address dst, XMMRegister src, uint8_t imm8);
2598
2599 // xmm/mem sourced byte/word/dword/qword replicate
2600 void vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len);
2601 void vpbroadcastb(XMMRegister dst, Address src, int vector_len);
2602 void vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len);
2603 void vpbroadcastw(XMMRegister dst, Address src, int vector_len);
2604 void vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len);
2605 void vpbroadcastd(XMMRegister dst, Address src, int vector_len);
2606 void vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len);
2607 void vpbroadcastq(XMMRegister dst, Address src, int vector_len);
2608
2609 void evbroadcasti32x4(XMMRegister dst, Address src, int vector_len);
2610 void evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len);
2611 void evbroadcasti64x2(XMMRegister dst, Address src, int vector_len);
2612
2613 // scalar single/double/128bit precision replicate
2614 void vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len);
2615 void vbroadcastss(XMMRegister dst, Address src, int vector_len);
2616 void vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len);
2617 void vbroadcastsd(XMMRegister dst, Address src, int vector_len);
2618 void vbroadcastf128(XMMRegister dst, Address src, int vector_len);
2619
2620 // gpr sourced byte/word/dword/qword replicate
2621 void evpbroadcastb(XMMRegister dst, Register src, int vector_len);
2622 void evpbroadcastw(XMMRegister dst, Register src, int vector_len);
2623 void evpbroadcastd(XMMRegister dst, Register src, int vector_len);
2624 void evpbroadcastq(XMMRegister dst, Register src, int vector_len);
2625
2626 // Gather AVX2 and AVX3
2627 void vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len);
2628 void vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len);
2629 void vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len);
2630 void vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len);
2631 void evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len);
2632 void evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len);
2633 void evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len);
2634 void evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len);
2635
2636 //Scatter AVX3 only
2637 void evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len);
2638 void evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len);
2639 void evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len);
2640 void evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len);
2641
2642 // Carry-Less Multiplication Quadword
2643 void pclmulqdq(XMMRegister dst, XMMRegister src, int mask);
2644 void vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask);
2645 void evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len);
2646 // AVX instruction which is used to clear upper 128 bits of YMM registers and
2647 // to avoid transaction penalty between AVX and SSE states. There is no
2648 // penalty if legacy SSE instructions are encoded using VEX prefix because
2649 // they always clear upper 128 bits. It should be used before calling
2650 // runtime code and native libraries.
2651 void vzeroupper();
2652
2653 // Vector double compares
2654 void vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len);
2655 void evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
2656 ComparisonPredicateFP comparison, int vector_len);
2657
2658 // Vector float compares
2659 void vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len);
2660 void evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
2661 ComparisonPredicateFP comparison, int vector_len);
2662
2663 // Vector integer compares
2664 void vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
2665 void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
2666 int comparison, bool is_signed, int vector_len);
2667 void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
2668 int comparison, bool is_signed, int vector_len);
2669
2670 // Vector long compares
2671 void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
2672 int comparison, bool is_signed, int vector_len);
2673 void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
2674 int comparison, bool is_signed, int vector_len);
2675
2676 // Vector byte compares
2677 void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
2678 int comparison, bool is_signed, int vector_len);
2679 void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
2680 int comparison, bool is_signed, int vector_len);
2681
2682 // Vector short compares
2683 void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
2684 int comparison, bool is_signed, int vector_len);
2685 void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src,
2686 int comparison, bool is_signed, int vector_len);
2687
2688 void evpmovb2m(KRegister dst, XMMRegister src, int vector_len);
2689 void evpmovw2m(KRegister dst, XMMRegister src, int vector_len);
2690 void evpmovd2m(KRegister dst, XMMRegister src, int vector_len);
2691 void evpmovq2m(KRegister dst, XMMRegister src, int vector_len);
2692 void evpmovm2b(XMMRegister dst, KRegister src, int vector_len);
2693 void evpmovm2w(XMMRegister dst, KRegister src, int vector_len);
2694 void evpmovm2d(XMMRegister dst, KRegister src, int vector_len);
2695 void evpmovm2q(XMMRegister dst, KRegister src, int vector_len);
2696
2697 // Vector blends
2698 void blendvps(XMMRegister dst, XMMRegister src);
2699 void blendvpd(XMMRegister dst, XMMRegister src);
2700 void pblendvb(XMMRegister dst, XMMRegister src);
2701 void blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len);
2702 void vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len);
2703 void vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len);
2704 void vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len);
2705 void vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len);
2706 void evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2707 void evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2708 void evpblendmb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2709 void evpblendmw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2710 void evpblendmd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2711 void evpblendmq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len);
2712 protected:
2713 // Next instructions require address alignment 16 bytes SSE mode.
2714 // They should be called only from corresponding MacroAssembler instructions.
2715 void andpd(XMMRegister dst, Address src);
2716 void andps(XMMRegister dst, Address src);
2717 void xorpd(XMMRegister dst, Address src);
2718 void xorps(XMMRegister dst, Address src);
2719
2720};
2721
2722// The Intel x86/Amd64 Assembler attributes: All fields enclosed here are to guide encoding level decisions.
2723// Specific set functions are for specialized use, else defaults or whatever was supplied to object construction
2724// are applied.
2725class InstructionAttr {
2726public:
2727 InstructionAttr(
2728 int vector_len, // The length of vector to be applied in encoding - for both AVX and EVEX
2729 bool rex_vex_w, // Width of data: if 32-bits or less, false, else if 64-bit or specially defined, true
2730 bool legacy_mode, // Details if either this instruction is conditionally encoded to AVX or earlier if true else possibly EVEX
2731 bool no_reg_mask, // when true, k0 is used when EVEX encoding is chosen, else embedded_opmask_register_specifier is used
2732 bool uses_vl) // This instruction may have legacy constraints based on vector length for EVEX
2733 :
2734 _rex_vex_w(rex_vex_w),
2735 _legacy_mode(legacy_mode || UseAVX < 3),
2736 _no_reg_mask(no_reg_mask),
2737 _uses_vl(uses_vl),
2738 _rex_vex_w_reverted(false),
2739 _is_evex_instruction(false),
2740 _is_clear_context(true),
2741 _is_extended_context(false),
2742 _avx_vector_len(vector_len),
2743 _tuple_type(Assembler::EVEX_ETUP),
2744 _input_size_in_bits(Assembler::EVEX_NObit),
2745 _evex_encoding(0),
2746 _embedded_opmask_register_specifier(0), // hard code k0
2747 _current_assembler(NULL__null) { }
2748
2749 ~InstructionAttr() {
2750 if (_current_assembler != NULL__null) {
2751 _current_assembler->clear_attributes();
2752 }
2753 _current_assembler = NULL__null;
2754 }
2755
2756private:
2757 bool _rex_vex_w;
2758 bool _legacy_mode;
2759 bool _no_reg_mask;
2760 bool _uses_vl;
2761 bool _rex_vex_w_reverted;
2762 bool _is_evex_instruction;
2763 bool _is_clear_context;
2764 bool _is_extended_context;
2765 int _avx_vector_len;
2766 int _tuple_type;
2767 int _input_size_in_bits;
2768 int _evex_encoding;
2769 int _embedded_opmask_register_specifier;
2770
2771 Assembler *_current_assembler;
2772
2773public:
2774 // query functions for field accessors
2775 bool is_rex_vex_w(void) const { return _rex_vex_w; }
2776 bool is_legacy_mode(void) const { return _legacy_mode; }
2777 bool is_no_reg_mask(void) const { return _no_reg_mask; }
2778 bool uses_vl(void) const { return _uses_vl; }
2779 bool is_rex_vex_w_reverted(void) { return _rex_vex_w_reverted; }
2780 bool is_evex_instruction(void) const { return _is_evex_instruction; }
2781 bool is_clear_context(void) const { return _is_clear_context; }
2782 bool is_extended_context(void) const { return _is_extended_context; }
2783 int get_vector_len(void) const { return _avx_vector_len; }
2784 int get_tuple_type(void) const { return _tuple_type; }
2785 int get_input_size(void) const { return _input_size_in_bits; }
2786 int get_evex_encoding(void) const { return _evex_encoding; }
2787 int get_embedded_opmask_register_specifier(void) const { return _embedded_opmask_register_specifier; }
2788
2789 // Set the vector len manually
2790 void set_vector_len(int vector_len) { _avx_vector_len = vector_len; }
2791
2792 // Set revert rex_vex_w for avx encoding
2793 void set_rex_vex_w_reverted(void) { _rex_vex_w_reverted = true; }
2794
2795 // Set rex_vex_w based on state
2796 void set_rex_vex_w(bool state) { _rex_vex_w = state; }
2797
2798 // Set the instruction to be encoded in AVX mode
2799 void set_is_legacy_mode(void) { _legacy_mode = true; }
2800
2801 // Set the current instuction to be encoded as an EVEX instuction
2802 void set_is_evex_instruction(void) { _is_evex_instruction = true; }
2803
2804 // Internal encoding data used in compressed immediate offset programming
2805 void set_evex_encoding(int value) { _evex_encoding = value; }
2806
2807 // When the Evex.Z field is set (true), it is used to clear all non directed XMM/YMM/ZMM components.
2808 // This method unsets it so that merge semantics are used instead.
2809 void reset_is_clear_context(void) { _is_clear_context = false; }
2810
2811 // Map back to current asembler so that we can manage object level assocation
2812 void set_current_assembler(Assembler *current_assembler) { _current_assembler = current_assembler; }
2813
2814 // Address modifiers used for compressed displacement calculation
2815 void set_address_attributes(int tuple_type, int input_size_in_bits);
2816
2817 // Set embedded opmask register specifier.
2818 void set_embedded_opmask_register_specifier(KRegister mask) {
2819 _embedded_opmask_register_specifier = (*mask).encoding() & 0x7;
2820 }
2821
2822};
2823
2824#endif // CPU_X86_ASSEMBLER_X86_HPP