| File: | jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp |
| Warning: | line 1673, column 9 2nd function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | ||||
| 2 | * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. | ||||
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | ||||
| 4 | * | ||||
| 5 | * This code is free software; you can redistribute it and/or modify it | ||||
| 6 | * under the terms of the GNU General Public License version 2 only, as | ||||
| 7 | * published by the Free Software Foundation. | ||||
| 8 | * | ||||
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT | ||||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||||
| 12 | * version 2 for more details (a copy is included in the LICENSE file that | ||||
| 13 | * accompanied this code). | ||||
| 14 | * | ||||
| 15 | * You should have received a copy of the GNU General Public License version | ||||
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, | ||||
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | ||||
| 18 | * | ||||
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | ||||
| 20 | * or visit www.oracle.com if you need additional information or have any | ||||
| 21 | * questions. | ||||
| 22 | * | ||||
| 23 | */ | ||||
| 24 | |||||
| 25 | #include "precompiled.hpp" | ||||
| 26 | #include "asm/macroAssembler.hpp" | ||||
| 27 | #include "asm/macroAssembler.inline.hpp" | ||||
| 28 | #include "ci/ciUtilities.hpp" | ||||
| 29 | #include "compiler/oopMap.hpp" | ||||
| 30 | #include "gc/shared/barrierSet.hpp" | ||||
| 31 | #include "gc/shared/barrierSetAssembler.hpp" | ||||
| 32 | #include "gc/shared/barrierSetNMethod.hpp" | ||||
| 33 | #include "gc/shared/gc_globals.hpp" | ||||
| 34 | #include "interpreter/interpreter.hpp" | ||||
| 35 | #include "memory/universe.hpp" | ||||
| 36 | #include "nativeInst_x86.hpp" | ||||
| 37 | #include "oops/instanceOop.hpp" | ||||
| 38 | #include "oops/method.hpp" | ||||
| 39 | #include "oops/objArrayKlass.hpp" | ||||
| 40 | #include "oops/oop.inline.hpp" | ||||
| 41 | #include "prims/methodHandles.hpp" | ||||
| 42 | #include "runtime/arguments.hpp" | ||||
| 43 | #include "runtime/frame.inline.hpp" | ||||
| 44 | #include "runtime/handles.inline.hpp" | ||||
| 45 | #include "runtime/sharedRuntime.hpp" | ||||
| 46 | #include "runtime/stubCodeGenerator.hpp" | ||||
| 47 | #include "runtime/stubRoutines.hpp" | ||||
| 48 | #include "runtime/thread.inline.hpp" | ||||
| 49 | #ifdef COMPILER21 | ||||
| 50 | #include "opto/runtime.hpp" | ||||
| 51 | #endif | ||||
| 52 | #if INCLUDE_JVMCI1 | ||||
| 53 | #include "jvmci/jvmci_globals.hpp" | ||||
| 54 | #endif | ||||
| 55 | #if INCLUDE_ZGC1 | ||||
| 56 | #include "gc/z/zThreadLocalData.hpp" | ||||
| 57 | #endif | ||||
| 58 | |||||
| 59 | // Declaration and definition of StubGenerator (no .hpp file). | ||||
| 60 | // For a more detailed description of the stub routine structure | ||||
| 61 | // see the comment in stubRoutines.hpp | ||||
| 62 | |||||
| 63 | #define __masm-> _masm-> | ||||
| 64 | #define TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8) (UseCompressedOops ? Address::times_4 : Address::times_8) | ||||
| 65 | #define a__((Assembler*)_masm)-> ((Assembler*)_masm)-> | ||||
| 66 | |||||
| 67 | #ifdef PRODUCT | ||||
| 68 | #define BLOCK_COMMENT(str)masm-> block_comment(str) /* nothing */ | ||||
| 69 | #else | ||||
| 70 | #define BLOCK_COMMENT(str)masm-> block_comment(str) __masm-> block_comment(str) | ||||
| 71 | #endif | ||||
| 72 | |||||
| 73 | #define BIND(label)bind(label); masm-> block_comment("label" ":") bind(label); BLOCK_COMMENT(#label ":")masm-> block_comment(#label ":") | ||||
| 74 | const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions | ||||
| 75 | |||||
| 76 | // Stub Code definitions | ||||
| 77 | |||||
| 78 | class StubGenerator: public StubCodeGenerator { | ||||
| 79 | private: | ||||
| 80 | |||||
| 81 | #ifdef PRODUCT | ||||
| 82 | #define inc_counter_np(counter)masm-> block_comment("inc_counter " "counter"); inc_counter_np_ (counter); ((void)0) | ||||
| 83 | #else | ||||
| 84 | void inc_counter_np_(int& counter) { | ||||
| 85 | // This can destroy rscratch1 if counter is far from the code cache | ||||
| 86 | __masm-> incrementl(ExternalAddress((address)&counter)); | ||||
| 87 | } | ||||
| 88 | #define inc_counter_np(counter)masm-> block_comment("inc_counter " "counter"); inc_counter_np_ (counter); \ | ||||
| 89 | BLOCK_COMMENT("inc_counter " #counter)masm-> block_comment("inc_counter " #counter); \ | ||||
| 90 | inc_counter_np_(counter); | ||||
| 91 | #endif | ||||
| 92 | |||||
| 93 | // Call stubs are used to call Java from C | ||||
| 94 | // | ||||
| 95 | // Linux Arguments: | ||||
| 96 | // c_rarg0: call wrapper address address | ||||
| 97 | // c_rarg1: result address | ||||
| 98 | // c_rarg2: result type BasicType | ||||
| 99 | // c_rarg3: method Method* | ||||
| 100 | // c_rarg4: (interpreter) entry point address | ||||
| 101 | // c_rarg5: parameters intptr_t* | ||||
| 102 | // 16(rbp): parameter size (in words) int | ||||
| 103 | // 24(rbp): thread Thread* | ||||
| 104 | // | ||||
| 105 | // [ return_from_Java ] <--- rsp | ||||
| 106 | // [ argument word n ] | ||||
| 107 | // ... | ||||
| 108 | // -12 [ argument word 1 ] | ||||
| 109 | // -11 [ saved r15 ] <--- rsp_after_call | ||||
| 110 | // -10 [ saved r14 ] | ||||
| 111 | // -9 [ saved r13 ] | ||||
| 112 | // -8 [ saved r12 ] | ||||
| 113 | // -7 [ saved rbx ] | ||||
| 114 | // -6 [ call wrapper ] | ||||
| 115 | // -5 [ result ] | ||||
| 116 | // -4 [ result type ] | ||||
| 117 | // -3 [ method ] | ||||
| 118 | // -2 [ entry point ] | ||||
| 119 | // -1 [ parameters ] | ||||
| 120 | // 0 [ saved rbp ] <--- rbp | ||||
| 121 | // 1 [ return address ] | ||||
| 122 | // 2 [ parameter size ] | ||||
| 123 | // 3 [ thread ] | ||||
| 124 | // | ||||
| 125 | // Windows Arguments: | ||||
| 126 | // c_rarg0: call wrapper address address | ||||
| 127 | // c_rarg1: result address | ||||
| 128 | // c_rarg2: result type BasicType | ||||
| 129 | // c_rarg3: method Method* | ||||
| 130 | // 48(rbp): (interpreter) entry point address | ||||
| 131 | // 56(rbp): parameters intptr_t* | ||||
| 132 | // 64(rbp): parameter size (in words) int | ||||
| 133 | // 72(rbp): thread Thread* | ||||
| 134 | // | ||||
| 135 | // [ return_from_Java ] <--- rsp | ||||
| 136 | // [ argument word n ] | ||||
| 137 | // ... | ||||
| 138 | // -60 [ argument word 1 ] | ||||
| 139 | // -59 [ saved xmm31 ] <--- rsp after_call | ||||
| 140 | // [ saved xmm16-xmm30 ] (EVEX enabled, else the space is blank) | ||||
| 141 | // -27 [ saved xmm15 ] | ||||
| 142 | // [ saved xmm7-xmm14 ] | ||||
| 143 | // -9 [ saved xmm6 ] (each xmm register takes 2 slots) | ||||
| 144 | // -7 [ saved r15 ] | ||||
| 145 | // -6 [ saved r14 ] | ||||
| 146 | // -5 [ saved r13 ] | ||||
| 147 | // -4 [ saved r12 ] | ||||
| 148 | // -3 [ saved rdi ] | ||||
| 149 | // -2 [ saved rsi ] | ||||
| 150 | // -1 [ saved rbx ] | ||||
| 151 | // 0 [ saved rbp ] <--- rbp | ||||
| 152 | // 1 [ return address ] | ||||
| 153 | // 2 [ call wrapper ] | ||||
| 154 | // 3 [ result ] | ||||
| 155 | // 4 [ result type ] | ||||
| 156 | // 5 [ method ] | ||||
| 157 | // 6 [ entry point ] | ||||
| 158 | // 7 [ parameters ] | ||||
| 159 | // 8 [ parameter size ] | ||||
| 160 | // 9 [ thread ] | ||||
| 161 | // | ||||
| 162 | // Windows reserves the callers stack space for arguments 1-4. | ||||
| 163 | // We spill c_rarg0-c_rarg3 to this space. | ||||
| 164 | |||||
| 165 | // Call stub stack layout word offsets from rbp | ||||
| 166 | enum call_stub_layout { | ||||
| 167 | #ifdef _WIN64 | ||||
| 168 | xmm_save_first = 6, // save from xmm6 | ||||
| 169 | xmm_save_last = 31, // to xmm31 | ||||
| 170 | xmm_save_base = -9, | ||||
| 171 | rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27 | ||||
| 172 | r15_off = -7, | ||||
| 173 | r14_off = -6, | ||||
| 174 | r13_off = -5, | ||||
| 175 | r12_off = -4, | ||||
| 176 | rdi_off = -3, | ||||
| 177 | rsi_off = -2, | ||||
| 178 | rbx_off = -1, | ||||
| 179 | rbp_off = 0, | ||||
| 180 | retaddr_off = 1, | ||||
| 181 | call_wrapper_off = 2, | ||||
| 182 | result_off = 3, | ||||
| 183 | result_type_off = 4, | ||||
| 184 | method_off = 5, | ||||
| 185 | entry_point_off = 6, | ||||
| 186 | parameters_off = 7, | ||||
| 187 | parameter_size_off = 8, | ||||
| 188 | thread_off = 9 | ||||
| 189 | #else | ||||
| 190 | rsp_after_call_off = -12, | ||||
| 191 | mxcsr_off = rsp_after_call_off, | ||||
| 192 | r15_off = -11, | ||||
| 193 | r14_off = -10, | ||||
| 194 | r13_off = -9, | ||||
| 195 | r12_off = -8, | ||||
| 196 | rbx_off = -7, | ||||
| 197 | call_wrapper_off = -6, | ||||
| 198 | result_off = -5, | ||||
| 199 | result_type_off = -4, | ||||
| 200 | method_off = -3, | ||||
| 201 | entry_point_off = -2, | ||||
| 202 | parameters_off = -1, | ||||
| 203 | rbp_off = 0, | ||||
| 204 | retaddr_off = 1, | ||||
| 205 | parameter_size_off = 2, | ||||
| 206 | thread_off = 3 | ||||
| 207 | #endif | ||||
| 208 | }; | ||||
| 209 | |||||
| 210 | #ifdef _WIN64 | ||||
| 211 | Address xmm_save(int reg) { | ||||
| 212 | assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range")do { if (!(reg >= xmm_save_first && reg <= xmm_save_last )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 212, "assert(" "reg >= xmm_save_first && reg <= xmm_save_last" ") failed", "XMM register number out of range"); ::breakpoint (); } } while (0); | ||||
| 213 | return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize); | ||||
| 214 | } | ||||
| 215 | #endif | ||||
| 216 | |||||
| 217 | address generate_call_stub(address& return_address) { | ||||
| 218 | assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&do { if (!((int)frame::entry_frame_after_call_words == -(int) rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 220, "assert(" "(int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off" ") failed", "adjust this code"); ::breakpoint(); } } while ( 0) | ||||
| 219 | (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,do { if (!((int)frame::entry_frame_after_call_words == -(int) rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 220, "assert(" "(int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off" ") failed", "adjust this code"); ::breakpoint(); } } while ( 0) | ||||
| 220 | "adjust this code")do { if (!((int)frame::entry_frame_after_call_words == -(int) rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 220, "assert(" "(int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off" ") failed", "adjust this code"); ::breakpoint(); } } while ( 0); | ||||
| 221 | StubCodeMark mark(this, "StubRoutines", "call_stub"); | ||||
| 222 | address start = __masm-> pc(); | ||||
| 223 | |||||
| 224 | // same as in generate_catch_exception()! | ||||
| 225 | const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); | ||||
| 226 | |||||
| 227 | const Address call_wrapper (rbp, call_wrapper_off * wordSize); | ||||
| 228 | const Address result (rbp, result_off * wordSize); | ||||
| 229 | const Address result_type (rbp, result_type_off * wordSize); | ||||
| 230 | const Address method (rbp, method_off * wordSize); | ||||
| 231 | const Address entry_point (rbp, entry_point_off * wordSize); | ||||
| 232 | const Address parameters (rbp, parameters_off * wordSize); | ||||
| 233 | const Address parameter_size(rbp, parameter_size_off * wordSize); | ||||
| 234 | |||||
| 235 | // same as in generate_catch_exception()! | ||||
| 236 | const Address thread (rbp, thread_off * wordSize); | ||||
| 237 | |||||
| 238 | const Address r15_save(rbp, r15_off * wordSize); | ||||
| 239 | const Address r14_save(rbp, r14_off * wordSize); | ||||
| 240 | const Address r13_save(rbp, r13_off * wordSize); | ||||
| 241 | const Address r12_save(rbp, r12_off * wordSize); | ||||
| 242 | const Address rbx_save(rbp, rbx_off * wordSize); | ||||
| 243 | |||||
| 244 | // stub code | ||||
| 245 | __masm-> enter(); | ||||
| 246 | __masm-> subptr(rsp, -rsp_after_call_off * wordSize); | ||||
| 247 | |||||
| 248 | // save register parameters | ||||
| 249 | #ifndef _WIN64 | ||||
| 250 | __masm-> movptr(parameters, c_rarg5); // parameters | ||||
| 251 | __masm-> movptr(entry_point, c_rarg4); // entry_point | ||||
| 252 | #endif | ||||
| 253 | |||||
| 254 | __masm-> movptr(method, c_rarg3); // method | ||||
| 255 | __masm-> movl(result_type, c_rarg2); // result type | ||||
| 256 | __masm-> movptr(result, c_rarg1); // result | ||||
| 257 | __masm-> movptr(call_wrapper, c_rarg0); // call wrapper | ||||
| 258 | |||||
| 259 | // save regs belonging to calling function | ||||
| 260 | __masm-> movptr(rbx_save, rbx); | ||||
| 261 | __masm-> movptr(r12_save, r12); | ||||
| 262 | __masm-> movptr(r13_save, r13); | ||||
| 263 | __masm-> movptr(r14_save, r14); | ||||
| 264 | __masm-> movptr(r15_save, r15); | ||||
| 265 | |||||
| 266 | #ifdef _WIN64 | ||||
| 267 | int last_reg = 15; | ||||
| 268 | if (UseAVX > 2) { | ||||
| 269 | last_reg = 31; | ||||
| 270 | } | ||||
| 271 | if (VM_Version::supports_evex()) { | ||||
| 272 | for (int i = xmm_save_first; i <= last_reg; i++) { | ||||
| 273 | __masm-> vextractf32x4(xmm_save(i), as_XMMRegister(i), 0); | ||||
| 274 | } | ||||
| 275 | } else { | ||||
| 276 | for (int i = xmm_save_first; i <= last_reg; i++) { | ||||
| 277 | __masm-> movdqu(xmm_save(i), as_XMMRegister(i)); | ||||
| 278 | } | ||||
| 279 | } | ||||
| 280 | |||||
| 281 | const Address rdi_save(rbp, rdi_off * wordSize); | ||||
| 282 | const Address rsi_save(rbp, rsi_off * wordSize); | ||||
| 283 | |||||
| 284 | __masm-> movptr(rsi_save, rsi); | ||||
| 285 | __masm-> movptr(rdi_save, rdi); | ||||
| 286 | #else | ||||
| 287 | const Address mxcsr_save(rbp, mxcsr_off * wordSize); | ||||
| 288 | { | ||||
| 289 | Label skip_ldmx; | ||||
| 290 | __masm-> stmxcsr(mxcsr_save); | ||||
| 291 | __masm-> movl(rax, mxcsr_save); | ||||
| 292 | __masm-> andl(rax, MXCSR_MASK); // Only check control and mask bits | ||||
| 293 | ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); | ||||
| 294 | __masm-> cmp32(rax, mxcsr_std); | ||||
| 295 | __masm-> jcc(Assembler::equal, skip_ldmx); | ||||
| 296 | __masm-> ldmxcsr(mxcsr_std); | ||||
| 297 | __masm-> bind(skip_ldmx); | ||||
| 298 | } | ||||
| 299 | #endif | ||||
| 300 | |||||
| 301 | // Load up thread register | ||||
| 302 | __masm-> movptr(r15_thread, thread); | ||||
| 303 | __masm-> reinit_heapbase(); | ||||
| 304 | |||||
| 305 | #ifdef ASSERT1 | ||||
| 306 | // make sure we have no pending exceptions | ||||
| 307 | { | ||||
| 308 | Label L; | ||||
| 309 | __masm-> cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD0L); | ||||
| 310 | __masm-> jcc(Assembler::equal, L); | ||||
| 311 | __masm-> stop("StubRoutines::call_stub: entered with pending exception"); | ||||
| 312 | __masm-> bind(L); | ||||
| 313 | } | ||||
| 314 | #endif | ||||
| 315 | |||||
| 316 | // pass parameters if any | ||||
| 317 | BLOCK_COMMENT("pass parameters if any")masm-> block_comment("pass parameters if any"); | ||||
| 318 | Label parameters_done; | ||||
| 319 | __masm-> movl(c_rarg3, parameter_size); | ||||
| 320 | __masm-> testl(c_rarg3, c_rarg3); | ||||
| 321 | __masm-> jcc(Assembler::zero, parameters_done); | ||||
| 322 | |||||
| 323 | Label loop; | ||||
| 324 | __masm-> movptr(c_rarg2, parameters); // parameter pointer | ||||
| 325 | __masm-> movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 | ||||
| 326 | __masm-> BIND(loop)bind(loop); masm-> block_comment("loop" ":"); | ||||
| 327 | __masm-> movptr(rax, Address(c_rarg2, 0));// get parameter | ||||
| 328 | __masm-> addptr(c_rarg2, wordSize); // advance to next parameter | ||||
| 329 | __masm-> decrementl(c_rarg1); // decrement counter | ||||
| 330 | __masm-> push(rax); // pass parameter | ||||
| 331 | __masm-> jcc(Assembler::notZero, loop); | ||||
| 332 | |||||
| 333 | // call Java function | ||||
| 334 | __masm-> BIND(parameters_done)bind(parameters_done); masm-> block_comment("parameters_done" ":"); | ||||
| 335 | __masm-> movptr(rbx, method); // get Method* | ||||
| 336 | __masm-> movptr(c_rarg1, entry_point); // get entry_point | ||||
| 337 | __masm-> mov(r13, rsp); // set sender sp | ||||
| 338 | BLOCK_COMMENT("call Java function")masm-> block_comment("call Java function"); | ||||
| 339 | __masm-> call(c_rarg1); | ||||
| 340 | |||||
| 341 | BLOCK_COMMENT("call_stub_return_address:")masm-> block_comment("call_stub_return_address:"); | ||||
| 342 | return_address = __masm-> pc(); | ||||
| 343 | |||||
| 344 | // store result depending on type (everything that is not | ||||
| 345 | // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) | ||||
| 346 | __masm-> movptr(c_rarg0, result); | ||||
| 347 | Label is_long, is_float, is_double, exit; | ||||
| 348 | __masm-> movl(c_rarg1, result_type); | ||||
| 349 | __masm-> cmpl(c_rarg1, T_OBJECT); | ||||
| 350 | __masm-> jcc(Assembler::equal, is_long); | ||||
| 351 | __masm-> cmpl(c_rarg1, T_LONG); | ||||
| 352 | __masm-> jcc(Assembler::equal, is_long); | ||||
| 353 | __masm-> cmpl(c_rarg1, T_FLOAT); | ||||
| 354 | __masm-> jcc(Assembler::equal, is_float); | ||||
| 355 | __masm-> cmpl(c_rarg1, T_DOUBLE); | ||||
| 356 | __masm-> jcc(Assembler::equal, is_double); | ||||
| 357 | |||||
| 358 | // handle T_INT case | ||||
| 359 | __masm-> movl(Address(c_rarg0, 0), rax); | ||||
| 360 | |||||
| 361 | __masm-> BIND(exit)bind(exit); masm-> block_comment("exit" ":"); | ||||
| 362 | |||||
| 363 | // pop parameters | ||||
| 364 | __masm-> lea(rsp, rsp_after_call); | ||||
| 365 | |||||
| 366 | #ifdef ASSERT1 | ||||
| 367 | // verify that threads correspond | ||||
| 368 | { | ||||
| 369 | Label L1, L2, L3; | ||||
| 370 | __masm-> cmpptr(r15_thread, thread); | ||||
| 371 | __masm-> jcc(Assembler::equal, L1); | ||||
| 372 | __masm-> stop("StubRoutines::call_stub: r15_thread is corrupted"); | ||||
| 373 | __masm-> bind(L1); | ||||
| 374 | __masm-> get_thread(rbx); | ||||
| 375 | __masm-> cmpptr(r15_thread, thread); | ||||
| 376 | __masm-> jcc(Assembler::equal, L2); | ||||
| 377 | __masm-> stop("StubRoutines::call_stub: r15_thread is modified by call"); | ||||
| 378 | __masm-> bind(L2); | ||||
| 379 | __masm-> cmpptr(r15_thread, rbx); | ||||
| 380 | __masm-> jcc(Assembler::equal, L3); | ||||
| 381 | __masm-> stop("StubRoutines::call_stub: threads must correspond"); | ||||
| 382 | __masm-> bind(L3); | ||||
| 383 | } | ||||
| 384 | #endif | ||||
| 385 | |||||
| 386 | // restore regs belonging to calling function | ||||
| 387 | #ifdef _WIN64 | ||||
| 388 | // emit the restores for xmm regs | ||||
| 389 | if (VM_Version::supports_evex()) { | ||||
| 390 | for (int i = xmm_save_first; i <= last_reg; i++) { | ||||
| 391 | __masm-> vinsertf32x4(as_XMMRegister(i), as_XMMRegister(i), xmm_save(i), 0); | ||||
| 392 | } | ||||
| 393 | } else { | ||||
| 394 | for (int i = xmm_save_first; i <= last_reg; i++) { | ||||
| 395 | __masm-> movdqu(as_XMMRegister(i), xmm_save(i)); | ||||
| 396 | } | ||||
| 397 | } | ||||
| 398 | #endif | ||||
| 399 | __masm-> movptr(r15, r15_save); | ||||
| 400 | __masm-> movptr(r14, r14_save); | ||||
| 401 | __masm-> movptr(r13, r13_save); | ||||
| 402 | __masm-> movptr(r12, r12_save); | ||||
| 403 | __masm-> movptr(rbx, rbx_save); | ||||
| 404 | |||||
| 405 | #ifdef _WIN64 | ||||
| 406 | __masm-> movptr(rdi, rdi_save); | ||||
| 407 | __masm-> movptr(rsi, rsi_save); | ||||
| 408 | #else | ||||
| 409 | __masm-> ldmxcsr(mxcsr_save); | ||||
| 410 | #endif | ||||
| 411 | |||||
| 412 | // restore rsp | ||||
| 413 | __masm-> addptr(rsp, -rsp_after_call_off * wordSize); | ||||
| 414 | |||||
| 415 | // return | ||||
| 416 | __masm-> vzeroupper(); | ||||
| 417 | __masm-> pop(rbp); | ||||
| 418 | __masm-> ret(0); | ||||
| 419 | |||||
| 420 | // handle return types different from T_INT | ||||
| 421 | __masm-> BIND(is_long)bind(is_long); masm-> block_comment("is_long" ":"); | ||||
| 422 | __masm-> movq(Address(c_rarg0, 0), rax); | ||||
| 423 | __masm-> jmp(exit); | ||||
| 424 | |||||
| 425 | __masm-> BIND(is_float)bind(is_float); masm-> block_comment("is_float" ":"); | ||||
| 426 | __masm-> movflt(Address(c_rarg0, 0), xmm0); | ||||
| 427 | __masm-> jmp(exit); | ||||
| 428 | |||||
| 429 | __masm-> BIND(is_double)bind(is_double); masm-> block_comment("is_double" ":"); | ||||
| 430 | __masm-> movdbl(Address(c_rarg0, 0), xmm0); | ||||
| 431 | __masm-> jmp(exit); | ||||
| 432 | |||||
| 433 | return start; | ||||
| 434 | } | ||||
| 435 | |||||
| 436 | // Return point for a Java call if there's an exception thrown in | ||||
| 437 | // Java code. The exception is caught and transformed into a | ||||
| 438 | // pending exception stored in JavaThread that can be tested from | ||||
| 439 | // within the VM. | ||||
| 440 | // | ||||
| 441 | // Note: Usually the parameters are removed by the callee. In case | ||||
| 442 | // of an exception crossing an activation frame boundary, that is | ||||
| 443 | // not the case if the callee is compiled code => need to setup the | ||||
| 444 | // rsp. | ||||
| 445 | // | ||||
| 446 | // rax: exception oop | ||||
| 447 | |||||
| 448 | address generate_catch_exception() { | ||||
| 449 | StubCodeMark mark(this, "StubRoutines", "catch_exception"); | ||||
| 450 | address start = __masm-> pc(); | ||||
| 451 | |||||
| 452 | // same as in generate_call_stub(): | ||||
| 453 | const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); | ||||
| 454 | const Address thread (rbp, thread_off * wordSize); | ||||
| 455 | |||||
| 456 | #ifdef ASSERT1 | ||||
| 457 | // verify that threads correspond | ||||
| 458 | { | ||||
| 459 | Label L1, L2, L3; | ||||
| 460 | __masm-> cmpptr(r15_thread, thread); | ||||
| 461 | __masm-> jcc(Assembler::equal, L1); | ||||
| 462 | __masm-> stop("StubRoutines::catch_exception: r15_thread is corrupted"); | ||||
| 463 | __masm-> bind(L1); | ||||
| 464 | __masm-> get_thread(rbx); | ||||
| 465 | __masm-> cmpptr(r15_thread, thread); | ||||
| 466 | __masm-> jcc(Assembler::equal, L2); | ||||
| 467 | __masm-> stop("StubRoutines::catch_exception: r15_thread is modified by call"); | ||||
| 468 | __masm-> bind(L2); | ||||
| 469 | __masm-> cmpptr(r15_thread, rbx); | ||||
| 470 | __masm-> jcc(Assembler::equal, L3); | ||||
| 471 | __masm-> stop("StubRoutines::catch_exception: threads must correspond"); | ||||
| 472 | __masm-> bind(L3); | ||||
| 473 | } | ||||
| 474 | #endif | ||||
| 475 | |||||
| 476 | // set pending exception | ||||
| 477 | __masm-> verify_oop(rax)_verify_oop_checked(rax, "broken oop " "rax", "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 477); | ||||
| 478 | |||||
| 479 | __masm-> movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); | ||||
| 480 | __masm-> lea(rscratch1, ExternalAddress((address)__FILE__"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp")); | ||||
| 481 | __masm-> movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); | ||||
| 482 | __masm-> movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__482); | ||||
| 483 | |||||
| 484 | // complete return to VM | ||||
| 485 | assert(StubRoutines::_call_stub_return_address != NULL,do { if (!(StubRoutines::_call_stub_return_address != __null) ) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 486, "assert(" "StubRoutines::_call_stub_return_address != __null" ") failed", "_call_stub_return_address must have been generated before" ); ::breakpoint(); } } while (0) | ||||
| 486 | "_call_stub_return_address must have been generated before")do { if (!(StubRoutines::_call_stub_return_address != __null) ) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 486, "assert(" "StubRoutines::_call_stub_return_address != __null" ") failed", "_call_stub_return_address must have been generated before" ); ::breakpoint(); } } while (0); | ||||
| 487 | __masm-> jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); | ||||
| 488 | |||||
| 489 | return start; | ||||
| 490 | } | ||||
| 491 | |||||
| 492 | // Continuation point for runtime calls returning with a pending | ||||
| 493 | // exception. The pending exception check happened in the runtime | ||||
| 494 | // or native call stub. The pending exception in Thread is | ||||
| 495 | // converted into a Java-level exception. | ||||
| 496 | // | ||||
| 497 | // Contract with Java-level exception handlers: | ||||
| 498 | // rax: exception | ||||
| 499 | // rdx: throwing pc | ||||
| 500 | // | ||||
| 501 | // NOTE: At entry of this stub, exception-pc must be on stack !! | ||||
| 502 | |||||
| 503 | address generate_forward_exception() { | ||||
| 504 | StubCodeMark mark(this, "StubRoutines", "forward exception"); | ||||
| 505 | address start = __masm-> pc(); | ||||
| 506 | |||||
| 507 | // Upon entry, the sp points to the return address returning into | ||||
| 508 | // Java (interpreted or compiled) code; i.e., the return address | ||||
| 509 | // becomes the throwing pc. | ||||
| 510 | // | ||||
| 511 | // Arguments pushed before the runtime call are still on the stack | ||||
| 512 | // but the exception handler will reset the stack pointer -> | ||||
| 513 | // ignore them. A potential result in registers can be ignored as | ||||
| 514 | // well. | ||||
| 515 | |||||
| 516 | #ifdef ASSERT1 | ||||
| 517 | // make sure this code is only executed if there is a pending exception | ||||
| 518 | { | ||||
| 519 | Label L; | ||||
| 520 | __masm-> cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL__null); | ||||
| 521 | __masm-> jcc(Assembler::notEqual, L); | ||||
| 522 | __masm-> stop("StubRoutines::forward exception: no pending exception (1)"); | ||||
| 523 | __masm-> bind(L); | ||||
| 524 | } | ||||
| 525 | #endif | ||||
| 526 | |||||
| 527 | // compute exception handler into rbx | ||||
| 528 | __masm-> movptr(c_rarg0, Address(rsp, 0)); | ||||
| 529 | BLOCK_COMMENT("call exception_handler_for_return_address")masm-> block_comment("call exception_handler_for_return_address" ); | ||||
| 530 | __masm-> call_VM_leaf(CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime::exception_handler_for_return_address ))) | ||||
| 531 | SharedRuntime::exception_handler_for_return_address)((address)((address_word)(SharedRuntime::exception_handler_for_return_address ))), | ||||
| 532 | r15_thread, c_rarg0); | ||||
| 533 | __masm-> mov(rbx, rax); | ||||
| 534 | |||||
| 535 | // setup rax & rdx, remove return address & clear pending exception | ||||
| 536 | __masm-> pop(rdx); | ||||
| 537 | __masm-> movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); | ||||
| 538 | __masm-> movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD0L); | ||||
| 539 | |||||
| 540 | #ifdef ASSERT1 | ||||
| 541 | // make sure exception is set | ||||
| 542 | { | ||||
| 543 | Label L; | ||||
| 544 | __masm-> testptr(rax, rax); | ||||
| 545 | __masm-> jcc(Assembler::notEqual, L); | ||||
| 546 | __masm-> stop("StubRoutines::forward exception: no pending exception (2)"); | ||||
| 547 | __masm-> bind(L); | ||||
| 548 | } | ||||
| 549 | #endif | ||||
| 550 | |||||
| 551 | // continue at exception handler (return address removed) | ||||
| 552 | // rax: exception | ||||
| 553 | // rbx: exception handler | ||||
| 554 | // rdx: throwing pc | ||||
| 555 | __masm-> verify_oop(rax)_verify_oop_checked(rax, "broken oop " "rax", "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 555); | ||||
| 556 | __masm-> jmp(rbx); | ||||
| 557 | |||||
| 558 | return start; | ||||
| 559 | } | ||||
| 560 | |||||
| 561 | // Support for intptr_t OrderAccess::fence() | ||||
| 562 | // | ||||
| 563 | // Arguments : | ||||
| 564 | // | ||||
| 565 | // Result: | ||||
| 566 | address generate_orderaccess_fence() { | ||||
| 567 | StubCodeMark mark(this, "StubRoutines", "orderaccess_fence"); | ||||
| 568 | address start = __masm-> pc(); | ||||
| 569 | __masm-> membar(Assembler::StoreLoad); | ||||
| 570 | __masm-> ret(0); | ||||
| 571 | |||||
| 572 | return start; | ||||
| 573 | } | ||||
| 574 | |||||
| 575 | |||||
| 576 | // Support for intptr_t get_previous_sp() | ||||
| 577 | // | ||||
| 578 | // This routine is used to find the previous stack pointer for the | ||||
| 579 | // caller. | ||||
| 580 | address generate_get_previous_sp() { | ||||
| 581 | StubCodeMark mark(this, "StubRoutines", "get_previous_sp"); | ||||
| 582 | address start = __masm-> pc(); | ||||
| 583 | |||||
| 584 | __masm-> movptr(rax, rsp); | ||||
| 585 | __masm-> addptr(rax, 8); // return address is at the top of the stack. | ||||
| 586 | __masm-> ret(0); | ||||
| 587 | |||||
| 588 | return start; | ||||
| 589 | } | ||||
| 590 | |||||
| 591 | //---------------------------------------------------------------------------------------------------- | ||||
| 592 | // Support for void verify_mxcsr() | ||||
| 593 | // | ||||
| 594 | // This routine is used with -Xcheck:jni to verify that native | ||||
| 595 | // JNI code does not return to Java code without restoring the | ||||
| 596 | // MXCSR register to our expected state. | ||||
| 597 | |||||
| 598 | address generate_verify_mxcsr() { | ||||
| 599 | StubCodeMark mark(this, "StubRoutines", "verify_mxcsr"); | ||||
| 600 | address start = __masm-> pc(); | ||||
| 601 | |||||
| 602 | const Address mxcsr_save(rsp, 0); | ||||
| 603 | |||||
| 604 | if (CheckJNICalls) { | ||||
| 605 | Label ok_ret; | ||||
| 606 | ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std()); | ||||
| 607 | __masm-> push(rax); | ||||
| 608 | __masm-> subptr(rsp, wordSize); // allocate a temp location | ||||
| 609 | __masm-> stmxcsr(mxcsr_save); | ||||
| 610 | __masm-> movl(rax, mxcsr_save); | ||||
| 611 | __masm-> andl(rax, MXCSR_MASK); // Only check control and mask bits | ||||
| 612 | __masm-> cmp32(rax, mxcsr_std); | ||||
| 613 | __masm-> jcc(Assembler::equal, ok_ret); | ||||
| 614 | |||||
| 615 | __masm-> warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall"); | ||||
| 616 | |||||
| 617 | __masm-> ldmxcsr(mxcsr_std); | ||||
| 618 | |||||
| 619 | __masm-> bind(ok_ret); | ||||
| 620 | __masm-> addptr(rsp, wordSize); | ||||
| 621 | __masm-> pop(rax); | ||||
| 622 | } | ||||
| 623 | |||||
| 624 | __masm-> ret(0); | ||||
| 625 | |||||
| 626 | return start; | ||||
| 627 | } | ||||
| 628 | |||||
| 629 | address generate_f2i_fixup() { | ||||
| 630 | StubCodeMark mark(this, "StubRoutines", "f2i_fixup"); | ||||
| 631 | Address inout(rsp, 5 * wordSize); // return address + 4 saves | ||||
| 632 | |||||
| 633 | address start = __masm-> pc(); | ||||
| 634 | |||||
| 635 | Label L; | ||||
| 636 | |||||
| 637 | __masm-> push(rax); | ||||
| 638 | __masm-> push(c_rarg3); | ||||
| 639 | __masm-> push(c_rarg2); | ||||
| 640 | __masm-> push(c_rarg1); | ||||
| 641 | |||||
| 642 | __masm-> movl(rax, 0x7f800000); | ||||
| 643 | __masm-> xorl(c_rarg3, c_rarg3); | ||||
| 644 | __masm-> movl(c_rarg2, inout); | ||||
| 645 | __masm-> movl(c_rarg1, c_rarg2); | ||||
| 646 | __masm-> andl(c_rarg1, 0x7fffffff); | ||||
| 647 | __masm-> cmpl(rax, c_rarg1); // NaN? -> 0 | ||||
| 648 | __masm-> jcc(Assembler::negative, L); | ||||
| 649 | __masm-> testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint | ||||
| 650 | __masm-> movl(c_rarg3, 0x80000000); | ||||
| 651 | __masm-> movl(rax, 0x7fffffff); | ||||
| 652 | __masm-> cmovl(Assembler::positive, c_rarg3, rax); | ||||
| 653 | |||||
| 654 | __masm-> bind(L); | ||||
| 655 | __masm-> movptr(inout, c_rarg3); | ||||
| 656 | |||||
| 657 | __masm-> pop(c_rarg1); | ||||
| 658 | __masm-> pop(c_rarg2); | ||||
| 659 | __masm-> pop(c_rarg3); | ||||
| 660 | __masm-> pop(rax); | ||||
| 661 | |||||
| 662 | __masm-> ret(0); | ||||
| 663 | |||||
| 664 | return start; | ||||
| 665 | } | ||||
| 666 | |||||
| 667 | address generate_f2l_fixup() { | ||||
| 668 | StubCodeMark mark(this, "StubRoutines", "f2l_fixup"); | ||||
| 669 | Address inout(rsp, 5 * wordSize); // return address + 4 saves | ||||
| 670 | address start = __masm-> pc(); | ||||
| 671 | |||||
| 672 | Label L; | ||||
| 673 | |||||
| 674 | __masm-> push(rax); | ||||
| 675 | __masm-> push(c_rarg3); | ||||
| 676 | __masm-> push(c_rarg2); | ||||
| 677 | __masm-> push(c_rarg1); | ||||
| 678 | |||||
| 679 | __masm-> movl(rax, 0x7f800000); | ||||
| 680 | __masm-> xorl(c_rarg3, c_rarg3); | ||||
| 681 | __masm-> movl(c_rarg2, inout); | ||||
| 682 | __masm-> movl(c_rarg1, c_rarg2); | ||||
| 683 | __masm-> andl(c_rarg1, 0x7fffffff); | ||||
| 684 | __masm-> cmpl(rax, c_rarg1); // NaN? -> 0 | ||||
| 685 | __masm-> jcc(Assembler::negative, L); | ||||
| 686 | __masm-> testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong | ||||
| 687 | __masm-> mov64(c_rarg3, 0x8000000000000000); | ||||
| 688 | __masm-> mov64(rax, 0x7fffffffffffffff); | ||||
| 689 | __masm-> cmov(Assembler::positive, c_rarg3, rax); | ||||
| 690 | |||||
| 691 | __masm-> bind(L); | ||||
| 692 | __masm-> movptr(inout, c_rarg3); | ||||
| 693 | |||||
| 694 | __masm-> pop(c_rarg1); | ||||
| 695 | __masm-> pop(c_rarg2); | ||||
| 696 | __masm-> pop(c_rarg3); | ||||
| 697 | __masm-> pop(rax); | ||||
| 698 | |||||
| 699 | __masm-> ret(0); | ||||
| 700 | |||||
| 701 | return start; | ||||
| 702 | } | ||||
| 703 | |||||
| 704 | address generate_d2i_fixup() { | ||||
| 705 | StubCodeMark mark(this, "StubRoutines", "d2i_fixup"); | ||||
| 706 | Address inout(rsp, 6 * wordSize); // return address + 5 saves | ||||
| 707 | |||||
| 708 | address start = __masm-> pc(); | ||||
| 709 | |||||
| 710 | Label L; | ||||
| 711 | |||||
| 712 | __masm-> push(rax); | ||||
| 713 | __masm-> push(c_rarg3); | ||||
| 714 | __masm-> push(c_rarg2); | ||||
| 715 | __masm-> push(c_rarg1); | ||||
| 716 | __masm-> push(c_rarg0); | ||||
| 717 | |||||
| 718 | __masm-> movl(rax, 0x7ff00000); | ||||
| 719 | __masm-> movq(c_rarg2, inout); | ||||
| 720 | __masm-> movl(c_rarg3, c_rarg2); | ||||
| 721 | __masm-> mov(c_rarg1, c_rarg2); | ||||
| 722 | __masm-> mov(c_rarg0, c_rarg2); | ||||
| 723 | __masm-> negl(c_rarg3); | ||||
| 724 | __masm-> shrptr(c_rarg1, 0x20); | ||||
| 725 | __masm-> orl(c_rarg3, c_rarg2); | ||||
| 726 | __masm-> andl(c_rarg1, 0x7fffffff); | ||||
| 727 | __masm-> xorl(c_rarg2, c_rarg2); | ||||
| 728 | __masm-> shrl(c_rarg3, 0x1f); | ||||
| 729 | __masm-> orl(c_rarg1, c_rarg3); | ||||
| 730 | __masm-> cmpl(rax, c_rarg1); | ||||
| 731 | __masm-> jcc(Assembler::negative, L); // NaN -> 0 | ||||
| 732 | __masm-> testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint | ||||
| 733 | __masm-> movl(c_rarg2, 0x80000000); | ||||
| 734 | __masm-> movl(rax, 0x7fffffff); | ||||
| 735 | __masm-> cmov(Assembler::positive, c_rarg2, rax); | ||||
| 736 | |||||
| 737 | __masm-> bind(L); | ||||
| 738 | __masm-> movptr(inout, c_rarg2); | ||||
| 739 | |||||
| 740 | __masm-> pop(c_rarg0); | ||||
| 741 | __masm-> pop(c_rarg1); | ||||
| 742 | __masm-> pop(c_rarg2); | ||||
| 743 | __masm-> pop(c_rarg3); | ||||
| 744 | __masm-> pop(rax); | ||||
| 745 | |||||
| 746 | __masm-> ret(0); | ||||
| 747 | |||||
| 748 | return start; | ||||
| 749 | } | ||||
| 750 | |||||
| 751 | address generate_d2l_fixup() { | ||||
| 752 | StubCodeMark mark(this, "StubRoutines", "d2l_fixup"); | ||||
| 753 | Address inout(rsp, 6 * wordSize); // return address + 5 saves | ||||
| 754 | |||||
| 755 | address start = __masm-> pc(); | ||||
| 756 | |||||
| 757 | Label L; | ||||
| 758 | |||||
| 759 | __masm-> push(rax); | ||||
| 760 | __masm-> push(c_rarg3); | ||||
| 761 | __masm-> push(c_rarg2); | ||||
| 762 | __masm-> push(c_rarg1); | ||||
| 763 | __masm-> push(c_rarg0); | ||||
| 764 | |||||
| 765 | __masm-> movl(rax, 0x7ff00000); | ||||
| 766 | __masm-> movq(c_rarg2, inout); | ||||
| 767 | __masm-> movl(c_rarg3, c_rarg2); | ||||
| 768 | __masm-> mov(c_rarg1, c_rarg2); | ||||
| 769 | __masm-> mov(c_rarg0, c_rarg2); | ||||
| 770 | __masm-> negl(c_rarg3); | ||||
| 771 | __masm-> shrptr(c_rarg1, 0x20); | ||||
| 772 | __masm-> orl(c_rarg3, c_rarg2); | ||||
| 773 | __masm-> andl(c_rarg1, 0x7fffffff); | ||||
| 774 | __masm-> xorl(c_rarg2, c_rarg2); | ||||
| 775 | __masm-> shrl(c_rarg3, 0x1f); | ||||
| 776 | __masm-> orl(c_rarg1, c_rarg3); | ||||
| 777 | __masm-> cmpl(rax, c_rarg1); | ||||
| 778 | __masm-> jcc(Assembler::negative, L); // NaN -> 0 | ||||
| 779 | __masm-> testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong | ||||
| 780 | __masm-> mov64(c_rarg2, 0x8000000000000000); | ||||
| 781 | __masm-> mov64(rax, 0x7fffffffffffffff); | ||||
| 782 | __masm-> cmovq(Assembler::positive, c_rarg2, rax); | ||||
| 783 | |||||
| 784 | __masm-> bind(L); | ||||
| 785 | __masm-> movq(inout, c_rarg2); | ||||
| 786 | |||||
| 787 | __masm-> pop(c_rarg0); | ||||
| 788 | __masm-> pop(c_rarg1); | ||||
| 789 | __masm-> pop(c_rarg2); | ||||
| 790 | __masm-> pop(c_rarg3); | ||||
| 791 | __masm-> pop(rax); | ||||
| 792 | |||||
| 793 | __masm-> ret(0); | ||||
| 794 | |||||
| 795 | return start; | ||||
| 796 | } | ||||
| 797 | |||||
| 798 | address generate_iota_indices(const char *stub_name) { | ||||
| 799 | __masm-> align(CodeEntryAlignment); | ||||
| 800 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 801 | address start = __masm-> pc(); | ||||
| 802 | __masm-> emit_data64(0x0706050403020100, relocInfo::none); | ||||
| 803 | __masm-> emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none); | ||||
| 804 | __masm-> emit_data64(0x1716151413121110, relocInfo::none); | ||||
| 805 | __masm-> emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none); | ||||
| 806 | __masm-> emit_data64(0x2726252423222120, relocInfo::none); | ||||
| 807 | __masm-> emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none); | ||||
| 808 | __masm-> emit_data64(0x3736353433323130, relocInfo::none); | ||||
| 809 | __masm-> emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none); | ||||
| 810 | return start; | ||||
| 811 | } | ||||
| 812 | |||||
| 813 | address generate_vector_byte_shuffle_mask(const char *stub_name) { | ||||
| 814 | __masm-> align(CodeEntryAlignment); | ||||
| 815 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 816 | address start = __masm-> pc(); | ||||
| 817 | __masm-> emit_data64(0x7070707070707070, relocInfo::none); | ||||
| 818 | __masm-> emit_data64(0x7070707070707070, relocInfo::none); | ||||
| 819 | __masm-> emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); | ||||
| 820 | __masm-> emit_data64(0xF0F0F0F0F0F0F0F0, relocInfo::none); | ||||
| 821 | return start; | ||||
| 822 | } | ||||
| 823 | |||||
| 824 | address generate_fp_mask(const char *stub_name, int64_t mask) { | ||||
| 825 | __masm-> align(CodeEntryAlignment); | ||||
| 826 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 827 | address start = __masm-> pc(); | ||||
| 828 | |||||
| 829 | __masm-> emit_data64( mask, relocInfo::none ); | ||||
| 830 | __masm-> emit_data64( mask, relocInfo::none ); | ||||
| 831 | |||||
| 832 | return start; | ||||
| 833 | } | ||||
| 834 | |||||
| 835 | address generate_vector_mask(const char *stub_name, int64_t mask) { | ||||
| 836 | __masm-> align(CodeEntryAlignment); | ||||
| 837 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 838 | address start = __masm-> pc(); | ||||
| 839 | |||||
| 840 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 841 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 842 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 843 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 844 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 845 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 846 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 847 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 848 | |||||
| 849 | return start; | ||||
| 850 | } | ||||
| 851 | |||||
| 852 | address generate_vector_byte_perm_mask(const char *stub_name) { | ||||
| 853 | __masm-> align(CodeEntryAlignment); | ||||
| 854 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 855 | address start = __masm-> pc(); | ||||
| 856 | |||||
| 857 | __masm-> emit_data64(0x0000000000000001, relocInfo::none); | ||||
| 858 | __masm-> emit_data64(0x0000000000000003, relocInfo::none); | ||||
| 859 | __masm-> emit_data64(0x0000000000000005, relocInfo::none); | ||||
| 860 | __masm-> emit_data64(0x0000000000000007, relocInfo::none); | ||||
| 861 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 862 | __masm-> emit_data64(0x0000000000000002, relocInfo::none); | ||||
| 863 | __masm-> emit_data64(0x0000000000000004, relocInfo::none); | ||||
| 864 | __masm-> emit_data64(0x0000000000000006, relocInfo::none); | ||||
| 865 | |||||
| 866 | return start; | ||||
| 867 | } | ||||
| 868 | |||||
| 869 | address generate_vector_fp_mask(const char *stub_name, int64_t mask) { | ||||
| 870 | __masm-> align(CodeEntryAlignment); | ||||
| 871 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 872 | address start = __masm-> pc(); | ||||
| 873 | |||||
| 874 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 875 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 876 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 877 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 878 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 879 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 880 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 881 | __masm-> emit_data64(mask, relocInfo::none); | ||||
| 882 | |||||
| 883 | return start; | ||||
| 884 | } | ||||
| 885 | |||||
| 886 | address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len, | ||||
| 887 | int32_t val0, int32_t val1, int32_t val2, int32_t val3, | ||||
| 888 | int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0, | ||||
| 889 | int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0, | ||||
| 890 | int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) { | ||||
| 891 | __masm-> align(CodeEntryAlignment); | ||||
| 892 | StubCodeMark mark(this, "StubRoutines", stub_name); | ||||
| 893 | address start = __masm-> pc(); | ||||
| 894 | |||||
| 895 | assert(len != Assembler::AVX_NoVec, "vector len must be specified")do { if (!(len != Assembler::AVX_NoVec)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 895, "assert(" "len != Assembler::AVX_NoVec" ") failed", "vector len must be specified" ); ::breakpoint(); } } while (0); | ||||
| 896 | __masm-> emit_data(val0, relocInfo::none, 0); | ||||
| 897 | __masm-> emit_data(val1, relocInfo::none, 0); | ||||
| 898 | __masm-> emit_data(val2, relocInfo::none, 0); | ||||
| 899 | __masm-> emit_data(val3, relocInfo::none, 0); | ||||
| 900 | if (len >= Assembler::AVX_256bit) { | ||||
| 901 | __masm-> emit_data(val4, relocInfo::none, 0); | ||||
| 902 | __masm-> emit_data(val5, relocInfo::none, 0); | ||||
| 903 | __masm-> emit_data(val6, relocInfo::none, 0); | ||||
| 904 | __masm-> emit_data(val7, relocInfo::none, 0); | ||||
| 905 | if (len >= Assembler::AVX_512bit) { | ||||
| 906 | __masm-> emit_data(val8, relocInfo::none, 0); | ||||
| 907 | __masm-> emit_data(val9, relocInfo::none, 0); | ||||
| 908 | __masm-> emit_data(val10, relocInfo::none, 0); | ||||
| 909 | __masm-> emit_data(val11, relocInfo::none, 0); | ||||
| 910 | __masm-> emit_data(val12, relocInfo::none, 0); | ||||
| 911 | __masm-> emit_data(val13, relocInfo::none, 0); | ||||
| 912 | __masm-> emit_data(val14, relocInfo::none, 0); | ||||
| 913 | __masm-> emit_data(val15, relocInfo::none, 0); | ||||
| 914 | } | ||||
| 915 | } | ||||
| 916 | |||||
| 917 | return start; | ||||
| 918 | } | ||||
| 919 | |||||
| 920 | // Non-destructive plausibility checks for oops | ||||
| 921 | // | ||||
| 922 | // Arguments: | ||||
| 923 | // all args on stack! | ||||
| 924 | // | ||||
| 925 | // Stack after saving c_rarg3: | ||||
| 926 | // [tos + 0]: saved c_rarg3 | ||||
| 927 | // [tos + 1]: saved c_rarg2 | ||||
| 928 | // [tos + 2]: saved r12 (several TemplateTable methods use it) | ||||
| 929 | // [tos + 3]: saved flags | ||||
| 930 | // [tos + 4]: return address | ||||
| 931 | // * [tos + 5]: error message (char*) | ||||
| 932 | // * [tos + 6]: object to verify (oop) | ||||
| 933 | // * [tos + 7]: saved rax - saved by caller and bashed | ||||
| 934 | // * [tos + 8]: saved r10 (rscratch1) - saved by caller | ||||
| 935 | // * = popped on exit | ||||
| 936 | address generate_verify_oop() { | ||||
| 937 | StubCodeMark mark(this, "StubRoutines", "verify_oop"); | ||||
| 938 | address start = __masm-> pc(); | ||||
| 939 | |||||
| 940 | Label exit, error; | ||||
| 941 | |||||
| 942 | __masm-> pushf(); | ||||
| 943 | __masm-> incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr())); | ||||
| 944 | |||||
| 945 | __masm-> push(r12); | ||||
| 946 | |||||
| 947 | // save c_rarg2 and c_rarg3 | ||||
| 948 | __masm-> push(c_rarg2); | ||||
| 949 | __masm-> push(c_rarg3); | ||||
| 950 | |||||
| 951 | enum { | ||||
| 952 | // After previous pushes. | ||||
| 953 | oop_to_verify = 6 * wordSize, | ||||
| 954 | saved_rax = 7 * wordSize, | ||||
| 955 | saved_r10 = 8 * wordSize, | ||||
| 956 | |||||
| 957 | // Before the call to MacroAssembler::debug(), see below. | ||||
| 958 | return_addr = 16 * wordSize, | ||||
| 959 | error_msg = 17 * wordSize | ||||
| 960 | }; | ||||
| 961 | |||||
| 962 | // get object | ||||
| 963 | __masm-> movptr(rax, Address(rsp, oop_to_verify)); | ||||
| 964 | |||||
| 965 | // make sure object is 'reasonable' | ||||
| 966 | __masm-> testptr(rax, rax); | ||||
| 967 | __masm-> jcc(Assembler::zero, exit); // if obj is NULL it is OK | ||||
| 968 | |||||
| 969 | #if INCLUDE_ZGC1 | ||||
| 970 | if (UseZGC) { | ||||
| 971 | // Check if metadata bits indicate a bad oop | ||||
| 972 | __masm-> testptr(rax, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset())); | ||||
| 973 | __masm-> jcc(Assembler::notZero, error); | ||||
| 974 | } | ||||
| 975 | #endif | ||||
| 976 | |||||
| 977 | // Check if the oop is in the right area of memory | ||||
| 978 | __masm-> movptr(c_rarg2, rax); | ||||
| 979 | __masm-> movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask()); | ||||
| 980 | __masm-> andptr(c_rarg2, c_rarg3); | ||||
| 981 | __masm-> movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits()); | ||||
| 982 | __masm-> cmpptr(c_rarg2, c_rarg3); | ||||
| 983 | __masm-> jcc(Assembler::notZero, error); | ||||
| 984 | |||||
| 985 | // make sure klass is 'reasonable', which is not zero. | ||||
| 986 | __masm-> load_klass(rax, rax, rscratch1); // get klass | ||||
| 987 | __masm-> testptr(rax, rax); | ||||
| 988 | __masm-> jcc(Assembler::zero, error); // if klass is NULL it is broken | ||||
| 989 | |||||
| 990 | // return if everything seems ok | ||||
| 991 | __masm-> bind(exit); | ||||
| 992 | __masm-> movptr(rax, Address(rsp, saved_rax)); // get saved rax back | ||||
| 993 | __masm-> movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back | ||||
| 994 | __masm-> pop(c_rarg3); // restore c_rarg3 | ||||
| 995 | __masm-> pop(c_rarg2); // restore c_rarg2 | ||||
| 996 | __masm-> pop(r12); // restore r12 | ||||
| 997 | __masm-> popf(); // restore flags | ||||
| 998 | __masm-> ret(4 * wordSize); // pop caller saved stuff | ||||
| 999 | |||||
| 1000 | // handle errors | ||||
| 1001 | __masm-> bind(error); | ||||
| 1002 | __masm-> movptr(rax, Address(rsp, saved_rax)); // get saved rax back | ||||
| 1003 | __masm-> movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back | ||||
| 1004 | __masm-> pop(c_rarg3); // get saved c_rarg3 back | ||||
| 1005 | __masm-> pop(c_rarg2); // get saved c_rarg2 back | ||||
| 1006 | __masm-> pop(r12); // get saved r12 back | ||||
| 1007 | __masm-> popf(); // get saved flags off stack -- | ||||
| 1008 | // will be ignored | ||||
| 1009 | |||||
| 1010 | __masm-> pusha(); // push registers | ||||
| 1011 | // (rip is already | ||||
| 1012 | // already pushed) | ||||
| 1013 | // debug(char* msg, int64_t pc, int64_t regs[]) | ||||
| 1014 | // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and | ||||
| 1015 | // pushed all the registers, so now the stack looks like: | ||||
| 1016 | // [tos + 0] 16 saved registers | ||||
| 1017 | // [tos + 16] return address | ||||
| 1018 | // * [tos + 17] error message (char*) | ||||
| 1019 | // * [tos + 18] object to verify (oop) | ||||
| 1020 | // * [tos + 19] saved rax - saved by caller and bashed | ||||
| 1021 | // * [tos + 20] saved r10 (rscratch1) - saved by caller | ||||
| 1022 | // * = popped on exit | ||||
| 1023 | |||||
| 1024 | __masm-> movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message | ||||
| 1025 | __masm-> movptr(c_rarg1, Address(rsp, return_addr)); // pass return address | ||||
| 1026 | __masm-> movq(c_rarg2, rsp); // pass address of regs on stack | ||||
| 1027 | __masm-> mov(r12, rsp); // remember rsp | ||||
| 1028 | __masm-> subptr(rsp, frame::arg_reg_save_area_bytes); // windows | ||||
| 1029 | __masm-> andptr(rsp, -16); // align stack as required by ABI | ||||
| 1030 | BLOCK_COMMENT("call MacroAssembler::debug")masm-> block_comment("call MacroAssembler::debug"); | ||||
| 1031 | __masm-> call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)((address)((address_word)(MacroAssembler::debug64))))); | ||||
| 1032 | __masm-> hlt(); | ||||
| 1033 | return start; | ||||
| 1034 | } | ||||
| 1035 | |||||
| 1036 | // | ||||
| 1037 | // Verify that a register contains clean 32-bits positive value | ||||
| 1038 | // (high 32-bits are 0) so it could be used in 64-bits shifts. | ||||
| 1039 | // | ||||
| 1040 | // Input: | ||||
| 1041 | // Rint - 32-bits value | ||||
| 1042 | // Rtmp - scratch | ||||
| 1043 | // | ||||
| 1044 | void assert_clean_int(Register Rint, Register Rtmp) { | ||||
| 1045 | #ifdef ASSERT1 | ||||
| 1046 | Label L; | ||||
| 1047 | assert_different_registers(Rtmp, Rint); | ||||
| 1048 | __masm-> movslq(Rtmp, Rint); | ||||
| 1049 | __masm-> cmpq(Rtmp, Rint); | ||||
| 1050 | __masm-> jcc(Assembler::equal, L); | ||||
| 1051 | __masm-> stop("high 32-bits of int value are not 0"); | ||||
| 1052 | __masm-> bind(L); | ||||
| 1053 | #endif | ||||
| 1054 | } | ||||
| 1055 | |||||
| 1056 | // Generate overlap test for array copy stubs | ||||
| 1057 | // | ||||
| 1058 | // Input: | ||||
| 1059 | // c_rarg0 - from | ||||
| 1060 | // c_rarg1 - to | ||||
| 1061 | // c_rarg2 - element count | ||||
| 1062 | // | ||||
| 1063 | // Output: | ||||
| 1064 | // rax - &from[element count - 1] | ||||
| 1065 | // | ||||
| 1066 | void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) { | ||||
| 1067 | assert(no_overlap_target != NULL, "must be generated")do { if (!(no_overlap_target != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1067, "assert(" "no_overlap_target != __null" ") failed", "must be generated" ); ::breakpoint(); } } while (0); | ||||
| 1068 | array_overlap_test(no_overlap_target, NULL__null, sf); | ||||
| 1069 | } | ||||
| 1070 | void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) { | ||||
| 1071 | array_overlap_test(NULL__null, &L_no_overlap, sf); | ||||
| 1072 | } | ||||
| 1073 | void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) { | ||||
| 1074 | const Register from = c_rarg0; | ||||
| 1075 | const Register to = c_rarg1; | ||||
| 1076 | const Register count = c_rarg2; | ||||
| 1077 | const Register end_from = rax; | ||||
| 1078 | |||||
| 1079 | __masm-> cmpptr(to, from); | ||||
| 1080 | __masm-> lea(end_from, Address(from, count, sf, 0)); | ||||
| 1081 | if (NOLp
| ||||
| 1082 | ExternalAddress no_overlap(no_overlap_target); | ||||
| 1083 | __masm-> jump_cc(Assembler::belowEqual, no_overlap); | ||||
| 1084 | __masm-> cmpptr(to, end_from); | ||||
| 1085 | __masm-> jump_cc(Assembler::aboveEqual, no_overlap); | ||||
| 1086 | } else { | ||||
| 1087 | __masm-> jcc(Assembler::belowEqual, (*NOLp)); | ||||
| 1088 | __masm-> cmpptr(to, end_from); | ||||
| 1089 | __masm-> jcc(Assembler::aboveEqual, (*NOLp)); | ||||
| 1090 | } | ||||
| 1091 | } | ||||
| 1092 | |||||
| 1093 | // Shuffle first three arg regs on Windows into Linux/Solaris locations. | ||||
| 1094 | // | ||||
| 1095 | // Outputs: | ||||
| 1096 | // rdi - rcx | ||||
| 1097 | // rsi - rdx | ||||
| 1098 | // rdx - r8 | ||||
| 1099 | // rcx - r9 | ||||
| 1100 | // | ||||
| 1101 | // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter | ||||
| 1102 | // are non-volatile. r9 and r10 should not be used by the caller. | ||||
| 1103 | // | ||||
| 1104 | DEBUG_ONLY(bool regs_in_thread;)bool regs_in_thread; | ||||
| 1105 | |||||
| 1106 | void setup_arg_regs(int nargs = 3) { | ||||
| 1107 | const Register saved_rdi = r9; | ||||
| 1108 | const Register saved_rsi = r10; | ||||
| 1109 | assert(nargs == 3 || nargs == 4, "else fix")do { if (!(nargs == 3 || nargs == 4)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1109, "assert(" "nargs == 3 || nargs == 4" ") failed", "else fix" ); ::breakpoint(); } } while (0); | ||||
| 1110 | #ifdef _WIN64 | ||||
| 1111 | assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,do { if (!(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1112, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0) | ||||
| 1112 | "unexpected argument registers")do { if (!(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1112, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0); | ||||
| 1113 | if (nargs >= 4) | ||||
| 1114 | __masm-> mov(rax, r9); // r9 is also saved_rdi | ||||
| 1115 | __masm-> movptr(saved_rdi, rdi); | ||||
| 1116 | __masm-> movptr(saved_rsi, rsi); | ||||
| 1117 | __masm-> mov(rdi, rcx); // c_rarg0 | ||||
| 1118 | __masm-> mov(rsi, rdx); // c_rarg1 | ||||
| 1119 | __masm-> mov(rdx, r8); // c_rarg2 | ||||
| 1120 | if (nargs >= 4) | ||||
| 1121 | __masm-> mov(rcx, rax); // c_rarg3 (via rax) | ||||
| 1122 | #else | ||||
| 1123 | assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,do { if (!(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1124, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0) | ||||
| 1124 | "unexpected argument registers")do { if (!(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1124, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0); | ||||
| 1125 | #endif | ||||
| 1126 | DEBUG_ONLY(regs_in_thread = false;)regs_in_thread = false; | ||||
| 1127 | } | ||||
| 1128 | |||||
| 1129 | void restore_arg_regs() { | ||||
| 1130 | assert(!regs_in_thread, "wrong call to restore_arg_regs")do { if (!(!regs_in_thread)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1130, "assert(" "!regs_in_thread" ") failed", "wrong call to restore_arg_regs" ); ::breakpoint(); } } while (0); | ||||
| 1131 | const Register saved_rdi = r9; | ||||
| 1132 | const Register saved_rsi = r10; | ||||
| 1133 | #ifdef _WIN64 | ||||
| 1134 | __masm-> movptr(rdi, saved_rdi); | ||||
| 1135 | __masm-> movptr(rsi, saved_rsi); | ||||
| 1136 | #endif | ||||
| 1137 | } | ||||
| 1138 | |||||
| 1139 | // This is used in places where r10 is a scratch register, and can | ||||
| 1140 | // be adapted if r9 is needed also. | ||||
| 1141 | void setup_arg_regs_using_thread() { | ||||
| 1142 | const Register saved_r15 = r9; | ||||
| 1143 | #ifdef _WIN64 | ||||
| 1144 | __masm-> mov(saved_r15, r15); // r15 is callee saved and needs to be restored | ||||
| 1145 | __masm-> get_thread(r15_thread); | ||||
| 1146 | assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,do { if (!(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1147, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0) | ||||
| 1147 | "unexpected argument registers")do { if (!(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1147, "assert(" "c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0); | ||||
| 1148 | __masm-> movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())), rdi); | ||||
| 1149 | __masm-> movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())), rsi); | ||||
| 1150 | |||||
| 1151 | __masm-> mov(rdi, rcx); // c_rarg0 | ||||
| 1152 | __masm-> mov(rsi, rdx); // c_rarg1 | ||||
| 1153 | __masm-> mov(rdx, r8); // c_rarg2 | ||||
| 1154 | #else | ||||
| 1155 | assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,do { if (!(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1156, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0) | ||||
| 1156 | "unexpected argument registers")do { if (!(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1156, "assert(" "c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx" ") failed", "unexpected argument registers"); ::breakpoint() ; } } while (0); | ||||
| 1157 | #endif | ||||
| 1158 | DEBUG_ONLY(regs_in_thread = true;)regs_in_thread = true; | ||||
| 1159 | } | ||||
| 1160 | |||||
| 1161 | void restore_arg_regs_using_thread() { | ||||
| 1162 | assert(regs_in_thread, "wrong call to restore_arg_regs")do { if (!(regs_in_thread)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1162, "assert(" "regs_in_thread" ") failed", "wrong call to restore_arg_regs" ); ::breakpoint(); } } while (0); | ||||
| 1163 | const Register saved_r15 = r9; | ||||
| 1164 | #ifdef _WIN64 | ||||
| 1165 | __masm-> get_thread(r15_thread); | ||||
| 1166 | __masm-> movptr(rsi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset()))); | ||||
| 1167 | __masm-> movptr(rdi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset()))); | ||||
| 1168 | __masm-> mov(r15, saved_r15); // r15 is callee saved and needs to be restored | ||||
| 1169 | #endif | ||||
| 1170 | } | ||||
| 1171 | |||||
| 1172 | // Copy big chunks forward | ||||
| 1173 | // | ||||
| 1174 | // Inputs: | ||||
| 1175 | // end_from - source arrays end address | ||||
| 1176 | // end_to - destination array end address | ||||
| 1177 | // qword_count - 64-bits element count, negative | ||||
| 1178 | // to - scratch | ||||
| 1179 | // L_copy_bytes - entry label | ||||
| 1180 | // L_copy_8_bytes - exit label | ||||
| 1181 | // | ||||
| 1182 | void copy_bytes_forward(Register end_from, Register end_to, | ||||
| 1183 | Register qword_count, Register to, | ||||
| 1184 | Label& L_copy_bytes, Label& L_copy_8_bytes) { | ||||
| 1185 | DEBUG_ONLY(__ stop("enter at entry label, not here"))masm-> stop("enter at entry label, not here"); | ||||
| 1186 | Label L_loop; | ||||
| 1187 | __masm-> align(OptoLoopAlignment); | ||||
| 1188 | if (UseUnalignedLoadStores) { | ||||
| 1189 | Label L_end; | ||||
| 1190 | __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":"); | ||||
| 1191 | if (UseAVX >= 2) { | ||||
| 1192 | __masm-> vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56)); | ||||
| 1193 | __masm-> vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0); | ||||
| 1194 | __masm-> vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24)); | ||||
| 1195 | __masm-> vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1); | ||||
| 1196 | } else { | ||||
| 1197 | __masm-> movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56)); | ||||
| 1198 | __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0); | ||||
| 1199 | __masm-> movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40)); | ||||
| 1200 | __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1); | ||||
| 1201 | __masm-> movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24)); | ||||
| 1202 | __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2); | ||||
| 1203 | __masm-> movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8)); | ||||
| 1204 | __masm-> movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3); | ||||
| 1205 | } | ||||
| 1206 | |||||
| 1207 | __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":" ); | ||||
| 1208 | __masm-> addptr(qword_count, 8); | ||||
| 1209 | __masm-> jcc(Assembler::lessEqual, L_loop); | ||||
| 1210 | __masm-> subptr(qword_count, 4); // sub(8) and add(4) | ||||
| 1211 | __masm-> jccb(Assembler::greater, L_end)jccb_0(Assembler::greater, L_end, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1211); | ||||
| 1212 | // Copy trailing 32 bytes | ||||
| 1213 | if (UseAVX >= 2) { | ||||
| 1214 | __masm-> vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24)); | ||||
| 1215 | __masm-> vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0); | ||||
| 1216 | } else { | ||||
| 1217 | __masm-> movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24)); | ||||
| 1218 | __masm-> movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0); | ||||
| 1219 | __masm-> movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8)); | ||||
| 1220 | __masm-> movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1); | ||||
| 1221 | } | ||||
| 1222 | __masm-> addptr(qword_count, 4); | ||||
| 1223 | __masm-> BIND(L_end)bind(L_end); masm-> block_comment("L_end" ":"); | ||||
| 1224 | if (UseAVX >= 2) { | ||||
| 1225 | // clean upper bits of YMM registers | ||||
| 1226 | __masm-> vpxor(xmm0, xmm0); | ||||
| 1227 | __masm-> vpxor(xmm1, xmm1); | ||||
| 1228 | } | ||||
| 1229 | } else { | ||||
| 1230 | // Copy 32-bytes per iteration | ||||
| 1231 | __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":"); | ||||
| 1232 | __masm-> movq(to, Address(end_from, qword_count, Address::times_8, -24)); | ||||
| 1233 | __masm-> movq(Address(end_to, qword_count, Address::times_8, -24), to); | ||||
| 1234 | __masm-> movq(to, Address(end_from, qword_count, Address::times_8, -16)); | ||||
| 1235 | __masm-> movq(Address(end_to, qword_count, Address::times_8, -16), to); | ||||
| 1236 | __masm-> movq(to, Address(end_from, qword_count, Address::times_8, - 8)); | ||||
| 1237 | __masm-> movq(Address(end_to, qword_count, Address::times_8, - 8), to); | ||||
| 1238 | __masm-> movq(to, Address(end_from, qword_count, Address::times_8, - 0)); | ||||
| 1239 | __masm-> movq(Address(end_to, qword_count, Address::times_8, - 0), to); | ||||
| 1240 | |||||
| 1241 | __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":" ); | ||||
| 1242 | __masm-> addptr(qword_count, 4); | ||||
| 1243 | __masm-> jcc(Assembler::lessEqual, L_loop); | ||||
| 1244 | } | ||||
| 1245 | __masm-> subptr(qword_count, 4); | ||||
| 1246 | __masm-> jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords | ||||
| 1247 | } | ||||
| 1248 | |||||
| 1249 | // Copy big chunks backward | ||||
| 1250 | // | ||||
| 1251 | // Inputs: | ||||
| 1252 | // from - source arrays address | ||||
| 1253 | // dest - destination array address | ||||
| 1254 | // qword_count - 64-bits element count | ||||
| 1255 | // to - scratch | ||||
| 1256 | // L_copy_bytes - entry label | ||||
| 1257 | // L_copy_8_bytes - exit label | ||||
| 1258 | // | ||||
| 1259 | void copy_bytes_backward(Register from, Register dest, | ||||
| 1260 | Register qword_count, Register to, | ||||
| 1261 | Label& L_copy_bytes, Label& L_copy_8_bytes) { | ||||
| 1262 | DEBUG_ONLY(__ stop("enter at entry label, not here"))masm-> stop("enter at entry label, not here"); | ||||
| 1263 | Label L_loop; | ||||
| 1264 | __masm-> align(OptoLoopAlignment); | ||||
| 1265 | if (UseUnalignedLoadStores) { | ||||
| 1266 | Label L_end; | ||||
| 1267 | __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":"); | ||||
| 1268 | if (UseAVX >= 2) { | ||||
| 1269 | __masm-> vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32)); | ||||
| 1270 | __masm-> vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0); | ||||
| 1271 | __masm-> vmovdqu(xmm1, Address(from, qword_count, Address::times_8, 0)); | ||||
| 1272 | __masm-> vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm1); | ||||
| 1273 | } else { | ||||
| 1274 | __masm-> movdqu(xmm0, Address(from, qword_count, Address::times_8, 48)); | ||||
| 1275 | __masm-> movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0); | ||||
| 1276 | __masm-> movdqu(xmm1, Address(from, qword_count, Address::times_8, 32)); | ||||
| 1277 | __masm-> movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1); | ||||
| 1278 | __masm-> movdqu(xmm2, Address(from, qword_count, Address::times_8, 16)); | ||||
| 1279 | __masm-> movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2); | ||||
| 1280 | __masm-> movdqu(xmm3, Address(from, qword_count, Address::times_8, 0)); | ||||
| 1281 | __masm-> movdqu(Address(dest, qword_count, Address::times_8, 0), xmm3); | ||||
| 1282 | } | ||||
| 1283 | |||||
| 1284 | __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":" ); | ||||
| 1285 | __masm-> subptr(qword_count, 8); | ||||
| 1286 | __masm-> jcc(Assembler::greaterEqual, L_loop); | ||||
| 1287 | |||||
| 1288 | __masm-> addptr(qword_count, 4); // add(8) and sub(4) | ||||
| 1289 | __masm-> jccb(Assembler::less, L_end)jccb_0(Assembler::less, L_end, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1289); | ||||
| 1290 | // Copy trailing 32 bytes | ||||
| 1291 | if (UseAVX >= 2) { | ||||
| 1292 | __masm-> vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0)); | ||||
| 1293 | __masm-> vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0); | ||||
| 1294 | } else { | ||||
| 1295 | __masm-> movdqu(xmm0, Address(from, qword_count, Address::times_8, 16)); | ||||
| 1296 | __masm-> movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0); | ||||
| 1297 | __masm-> movdqu(xmm1, Address(from, qword_count, Address::times_8, 0)); | ||||
| 1298 | __masm-> movdqu(Address(dest, qword_count, Address::times_8, 0), xmm1); | ||||
| 1299 | } | ||||
| 1300 | __masm-> subptr(qword_count, 4); | ||||
| 1301 | __masm-> BIND(L_end)bind(L_end); masm-> block_comment("L_end" ":"); | ||||
| 1302 | if (UseAVX >= 2) { | ||||
| 1303 | // clean upper bits of YMM registers | ||||
| 1304 | __masm-> vpxor(xmm0, xmm0); | ||||
| 1305 | __masm-> vpxor(xmm1, xmm1); | ||||
| 1306 | } | ||||
| 1307 | } else { | ||||
| 1308 | // Copy 32-bytes per iteration | ||||
| 1309 | __masm-> BIND(L_loop)bind(L_loop); masm-> block_comment("L_loop" ":"); | ||||
| 1310 | __masm-> movq(to, Address(from, qword_count, Address::times_8, 24)); | ||||
| 1311 | __masm-> movq(Address(dest, qword_count, Address::times_8, 24), to); | ||||
| 1312 | __masm-> movq(to, Address(from, qword_count, Address::times_8, 16)); | ||||
| 1313 | __masm-> movq(Address(dest, qword_count, Address::times_8, 16), to); | ||||
| 1314 | __masm-> movq(to, Address(from, qword_count, Address::times_8, 8)); | ||||
| 1315 | __masm-> movq(Address(dest, qword_count, Address::times_8, 8), to); | ||||
| 1316 | __masm-> movq(to, Address(from, qword_count, Address::times_8, 0)); | ||||
| 1317 | __masm-> movq(Address(dest, qword_count, Address::times_8, 0), to); | ||||
| 1318 | |||||
| 1319 | __masm-> BIND(L_copy_bytes)bind(L_copy_bytes); masm-> block_comment("L_copy_bytes" ":" ); | ||||
| 1320 | __masm-> subptr(qword_count, 4); | ||||
| 1321 | __masm-> jcc(Assembler::greaterEqual, L_loop); | ||||
| 1322 | } | ||||
| 1323 | __masm-> addptr(qword_count, 4); | ||||
| 1324 | __masm-> jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords | ||||
| 1325 | } | ||||
| 1326 | |||||
| 1327 | #ifndef PRODUCT | ||||
| 1328 | int& get_profile_ctr(int shift) { | ||||
| 1329 | if ( 0 == shift) | ||||
| 1330 | return SharedRuntime::_jbyte_array_copy_ctr; | ||||
| 1331 | else if(1 == shift) | ||||
| 1332 | return SharedRuntime::_jshort_array_copy_ctr; | ||||
| 1333 | else if(2 == shift) | ||||
| 1334 | return SharedRuntime::_jint_array_copy_ctr; | ||||
| 1335 | else | ||||
| 1336 | return SharedRuntime::_jlong_array_copy_ctr; | ||||
| 1337 | } | ||||
| 1338 | #endif | ||||
| 1339 | |||||
| 1340 | void setup_argument_regs(BasicType type) { | ||||
| 1341 | if (type == T_BYTE || type == T_SHORT) { | ||||
| 1342 | setup_arg_regs(); // from => rdi, to => rsi, count => rdx | ||||
| 1343 | // r9 and r10 may be used to save non-volatile registers | ||||
| 1344 | } else { | ||||
| 1345 | setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | ||||
| 1346 | // r9 is used to save r15_thread | ||||
| 1347 | } | ||||
| 1348 | } | ||||
| 1349 | |||||
| 1350 | void restore_argument_regs(BasicType type) { | ||||
| 1351 | if (type == T_BYTE || type == T_SHORT) { | ||||
| 1352 | restore_arg_regs(); | ||||
| 1353 | } else { | ||||
| 1354 | restore_arg_regs_using_thread(); | ||||
| 1355 | } | ||||
| 1356 | } | ||||
| 1357 | |||||
| 1358 | #if COMPILER2_OR_JVMCI1 | ||||
| 1359 | // Note: Following rules apply to AVX3 optimized arraycopy stubs:- | ||||
| 1360 | // - If target supports AVX3 features (BW+VL+F) then implementation uses 32 byte vectors (YMMs) | ||||
| 1361 | // for both special cases (various small block sizes) and aligned copy loop. This is the | ||||
| 1362 | // default configuration. | ||||
| 1363 | // - If copy length is above AVX3Threshold, then implementation use 64 byte vectors (ZMMs) | ||||
| 1364 | // for main copy loop (and subsequent tail) since bulk of the cycles will be consumed in it. | ||||
| 1365 | // - If user forces MaxVectorSize=32 then above 4096 bytes its seen that REP MOVs shows a | ||||
| 1366 | // better performance for disjoint copies. For conjoint/backward copy vector based | ||||
| 1367 | // copy performs better. | ||||
| 1368 | // - If user sets AVX3Threshold=0, then special cases for small blocks sizes operate over | ||||
| 1369 | // 64 byte vector registers (ZMMs). | ||||
| 1370 | |||||
| 1371 | // Inputs: | ||||
| 1372 | // c_rarg0 - source array address | ||||
| 1373 | // c_rarg1 - destination array address | ||||
| 1374 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 1375 | // | ||||
| 1376 | // | ||||
| 1377 | // Side Effects: | ||||
| 1378 | // disjoint_copy_avx3_masked is set to the no-overlap entry point | ||||
| 1379 | // used by generate_conjoint_[byte/int/short/long]_copy(). | ||||
| 1380 | // | ||||
| 1381 | |||||
| 1382 | address generate_disjoint_copy_avx3_masked(address* entry, const char *name, int shift, | ||||
| 1383 | bool aligned, bool is_oop, bool dest_uninitialized) { | ||||
| 1384 | __masm-> align(CodeEntryAlignment); | ||||
| 1385 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 1386 | address start = __masm-> pc(); | ||||
| 1387 | int avx3threshold = VM_Version::avx3_threshold(); | ||||
| 1388 | bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0); | ||||
| 1389 | Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry; | ||||
| 1390 | Label L_repmovs, L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64; | ||||
| 1391 | const Register from = rdi; // source array address | ||||
| 1392 | const Register to = rsi; // destination array address | ||||
| 1393 | const Register count = rdx; // elements count | ||||
| 1394 | const Register temp1 = r8; | ||||
| 1395 | const Register temp2 = r11; | ||||
| 1396 | const Register temp3 = rax; | ||||
| 1397 | const Register temp4 = rcx; | ||||
| 1398 | // End pointers are inclusive, and if count is not zero they point | ||||
| 1399 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 1400 | |||||
| 1401 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1402 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 1403 | |||||
| 1404 | if (entry != NULL__null) { | ||||
| 1405 | *entry = __masm-> pc(); | ||||
| 1406 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 1407 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 1408 | } | ||||
| 1409 | |||||
| 1410 | BasicType type_vec[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; | ||||
| 1411 | BasicType type = is_oop ? T_OBJECT : type_vec[shift]; | ||||
| 1412 | |||||
| 1413 | setup_argument_regs(type); | ||||
| 1414 | |||||
| 1415 | DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | ||||
| 1416 | if (dest_uninitialized) { | ||||
| 1417 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 1418 | } | ||||
| 1419 | if (aligned) { | ||||
| 1420 | decorators |= ARRAYCOPY_ALIGNED; | ||||
| 1421 | } | ||||
| 1422 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 1423 | bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | ||||
| 1424 | |||||
| 1425 | { | ||||
| 1426 | // Type(shift) byte(0), short(1), int(2), long(3) | ||||
| 1427 | int loop_size[] = { 192, 96, 48, 24}; | ||||
| 1428 | int threshold[] = { 4096, 2048, 1024, 512}; | ||||
| 1429 | |||||
| 1430 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 1431 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 1432 | // 'from', 'to' and 'count' are now valid | ||||
| 1433 | |||||
| 1434 | // temp1 holds remaining count and temp4 holds running count used to compute | ||||
| 1435 | // next address offset for start of to/from addresses (temp4 * scale). | ||||
| 1436 | __masm-> mov64(temp4, 0); | ||||
| 1437 | __masm-> movq(temp1, count); | ||||
| 1438 | |||||
| 1439 | // Zero length check. | ||||
| 1440 | __masm-> BIND(L_tail)bind(L_tail); masm-> block_comment("L_tail" ":"); | ||||
| 1441 | __masm-> cmpq(temp1, 0); | ||||
| 1442 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 1443 | |||||
| 1444 | // Special cases using 32 byte [masked] vector copy operations. | ||||
| 1445 | __masm-> arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift, | ||||
| 1446 | temp4, temp3, use64byteVector, L_entry, L_exit); | ||||
| 1447 | |||||
| 1448 | // PRE-MAIN-POST loop for aligned copy. | ||||
| 1449 | __masm-> BIND(L_entry)bind(L_entry); masm-> block_comment("L_entry" ":"); | ||||
| 1450 | |||||
| 1451 | if (avx3threshold != 0) { | ||||
| 1452 | __masm-> cmpq(count, threshold[shift]); | ||||
| 1453 | if (MaxVectorSize == 64) { | ||||
| 1454 | // Copy using 64 byte vectors. | ||||
| 1455 | __masm-> jcc(Assembler::greaterEqual, L_pre_main_post_64); | ||||
| 1456 | } else { | ||||
| 1457 | assert(MaxVectorSize < 64, "vector size should be < 64 bytes")do { if (!(MaxVectorSize < 64)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1457, "assert(" "MaxVectorSize < 64" ") failed", "vector size should be < 64 bytes" ); ::breakpoint(); } } while (0); | ||||
| 1458 | // REP MOVS offer a faster copy path. | ||||
| 1459 | __masm-> jcc(Assembler::greaterEqual, L_repmovs); | ||||
| 1460 | } | ||||
| 1461 | } | ||||
| 1462 | |||||
| 1463 | if ((MaxVectorSize < 64) || (avx3threshold != 0)) { | ||||
| 1464 | // Partial copy to make dst address 32 byte aligned. | ||||
| 1465 | __masm-> movq(temp2, to); | ||||
| 1466 | __masm-> andq(temp2, 31); | ||||
| 1467 | __masm-> jcc(Assembler::equal, L_main_pre_loop); | ||||
| 1468 | |||||
| 1469 | __masm-> negptr(temp2); | ||||
| 1470 | __masm-> addq(temp2, 32); | ||||
| 1471 | if (shift) { | ||||
| 1472 | __masm-> shrq(temp2, shift); | ||||
| 1473 | } | ||||
| 1474 | __masm-> movq(temp3, temp2); | ||||
| 1475 | __masm-> copy32_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift); | ||||
| 1476 | __masm-> movq(temp4, temp2); | ||||
| 1477 | __masm-> movq(temp1, count); | ||||
| 1478 | __masm-> subq(temp1, temp2); | ||||
| 1479 | |||||
| 1480 | __masm-> cmpq(temp1, loop_size[shift]); | ||||
| 1481 | __masm-> jcc(Assembler::less, L_tail); | ||||
| 1482 | |||||
| 1483 | __masm-> BIND(L_main_pre_loop)bind(L_main_pre_loop); masm-> block_comment("L_main_pre_loop" ":"); | ||||
| 1484 | __masm-> subq(temp1, loop_size[shift]); | ||||
| 1485 | |||||
| 1486 | // Main loop with aligned copy block size of 192 bytes at 32 byte granularity. | ||||
| 1487 | __masm-> align32(); | ||||
| 1488 | __masm-> BIND(L_main_loop)bind(L_main_loop); masm-> block_comment("L_main_loop" ":"); | ||||
| 1489 | __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 0); | ||||
| 1490 | __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 64); | ||||
| 1491 | __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 128); | ||||
| 1492 | __masm-> addptr(temp4, loop_size[shift]); | ||||
| 1493 | __masm-> subq(temp1, loop_size[shift]); | ||||
| 1494 | __masm-> jcc(Assembler::greater, L_main_loop); | ||||
| 1495 | |||||
| 1496 | __masm-> addq(temp1, loop_size[shift]); | ||||
| 1497 | |||||
| 1498 | // Tail loop. | ||||
| 1499 | __masm-> jmp(L_tail); | ||||
| 1500 | |||||
| 1501 | __masm-> BIND(L_repmovs)bind(L_repmovs); masm-> block_comment("L_repmovs" ":"); | ||||
| 1502 | __masm-> movq(temp2, temp1); | ||||
| 1503 | // Swap to(RSI) and from(RDI) addresses to comply with REP MOVs semantics. | ||||
| 1504 | __masm-> movq(temp3, to); | ||||
| 1505 | __masm-> movq(to, from); | ||||
| 1506 | __masm-> movq(from, temp3); | ||||
| 1507 | // Save to/from for restoration post rep_mov. | ||||
| 1508 | __masm-> movq(temp1, to); | ||||
| 1509 | __masm-> movq(temp3, from); | ||||
| 1510 | if(shift < 3) { | ||||
| 1511 | __masm-> shrq(temp2, 3-shift); // quad word count | ||||
| 1512 | } | ||||
| 1513 | __masm-> movq(temp4 , temp2); // move quad ward count into temp4(RCX). | ||||
| 1514 | __masm-> rep_mov(); | ||||
| 1515 | __masm-> shlq(temp2, 3); // convert quad words into byte count. | ||||
| 1516 | if(shift) { | ||||
| 1517 | __masm-> shrq(temp2, shift); // type specific count. | ||||
| 1518 | } | ||||
| 1519 | // Restore original addresses in to/from. | ||||
| 1520 | __masm-> movq(to, temp3); | ||||
| 1521 | __masm-> movq(from, temp1); | ||||
| 1522 | __masm-> movq(temp4, temp2); | ||||
| 1523 | __masm-> movq(temp1, count); | ||||
| 1524 | __masm-> subq(temp1, temp2); // tailing part (less than a quad ward size). | ||||
| 1525 | __masm-> jmp(L_tail); | ||||
| 1526 | } | ||||
| 1527 | |||||
| 1528 | if (MaxVectorSize > 32) { | ||||
| 1529 | __masm-> BIND(L_pre_main_post_64)bind(L_pre_main_post_64); masm-> block_comment("L_pre_main_post_64" ":"); | ||||
| 1530 | // Partial copy to make dst address 64 byte aligned. | ||||
| 1531 | __masm-> movq(temp2, to); | ||||
| 1532 | __masm-> andq(temp2, 63); | ||||
| 1533 | __masm-> jcc(Assembler::equal, L_main_pre_loop_64bytes); | ||||
| 1534 | |||||
| 1535 | __masm-> negptr(temp2); | ||||
| 1536 | __masm-> addq(temp2, 64); | ||||
| 1537 | if (shift) { | ||||
| 1538 | __masm-> shrq(temp2, shift); | ||||
| 1539 | } | ||||
| 1540 | __masm-> movq(temp3, temp2); | ||||
| 1541 | __masm-> copy64_masked_avx(to, from, xmm1, k2, temp3, temp4, temp1, shift, 0 , true); | ||||
| 1542 | __masm-> movq(temp4, temp2); | ||||
| 1543 | __masm-> movq(temp1, count); | ||||
| 1544 | __masm-> subq(temp1, temp2); | ||||
| 1545 | |||||
| 1546 | __masm-> cmpq(temp1, loop_size[shift]); | ||||
| 1547 | __masm-> jcc(Assembler::less, L_tail64); | ||||
| 1548 | |||||
| 1549 | __masm-> BIND(L_main_pre_loop_64bytes)bind(L_main_pre_loop_64bytes); masm-> block_comment("L_main_pre_loop_64bytes" ":"); | ||||
| 1550 | __masm-> subq(temp1, loop_size[shift]); | ||||
| 1551 | |||||
| 1552 | // Main loop with aligned copy block size of 192 bytes at | ||||
| 1553 | // 64 byte copy granularity. | ||||
| 1554 | __masm-> align32(); | ||||
| 1555 | __masm-> BIND(L_main_loop_64bytes)bind(L_main_loop_64bytes); masm-> block_comment("L_main_loop_64bytes" ":"); | ||||
| 1556 | __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 0 , true); | ||||
| 1557 | __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 64, true); | ||||
| 1558 | __masm-> copy64_avx(to, from, temp4, xmm1, false, shift, 128, true); | ||||
| 1559 | __masm-> addptr(temp4, loop_size[shift]); | ||||
| 1560 | __masm-> subq(temp1, loop_size[shift]); | ||||
| 1561 | __masm-> jcc(Assembler::greater, L_main_loop_64bytes); | ||||
| 1562 | |||||
| 1563 | __masm-> addq(temp1, loop_size[shift]); | ||||
| 1564 | // Zero length check. | ||||
| 1565 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 1566 | |||||
| 1567 | __masm-> BIND(L_tail64)bind(L_tail64); masm-> block_comment("L_tail64" ":"); | ||||
| 1568 | |||||
| 1569 | // Tail handling using 64 byte [masked] vector copy operations. | ||||
| 1570 | use64byteVector = true; | ||||
| 1571 | __masm-> arraycopy_avx3_special_cases(xmm1, k2, from, to, temp1, shift, | ||||
| 1572 | temp4, temp3, use64byteVector, L_entry, L_exit); | ||||
| 1573 | } | ||||
| 1574 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 1575 | } | ||||
| 1576 | |||||
| 1577 | address ucme_exit_pc = __masm-> pc(); | ||||
| 1578 | // When called from generic_arraycopy r11 contains specific values | ||||
| 1579 | // used during arraycopy epilogue, re-initializing r11. | ||||
| 1580 | if (is_oop) { | ||||
| 1581 | __masm-> movq(r11, shift == 3 ? count : to); | ||||
| 1582 | } | ||||
| 1583 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, count); | ||||
| 1584 | restore_argument_regs(type); | ||||
| 1585 | inc_counter_np(get_profile_ctr(shift))masm-> block_comment("inc_counter " "get_profile_ctr(shift)" ); inc_counter_np_(get_profile_ctr(shift));; // Update counter after rscratch1 is free | ||||
| 1586 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 1587 | __masm-> vzeroupper(); | ||||
| 1588 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1589 | __masm-> ret(0); | ||||
| 1590 | return start; | ||||
| 1591 | } | ||||
| 1592 | |||||
| 1593 | // Inputs: | ||||
| 1594 | // c_rarg0 - source array address | ||||
| 1595 | // c_rarg1 - destination array address | ||||
| 1596 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 1597 | // | ||||
| 1598 | // | ||||
| 1599 | address generate_conjoint_copy_avx3_masked(address* entry, const char *name, int shift, | ||||
| 1600 | address nooverlap_target, bool aligned, bool is_oop, | ||||
| 1601 | bool dest_uninitialized) { | ||||
| 1602 | __masm-> align(CodeEntryAlignment); | ||||
| 1603 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 1604 | address start = __masm-> pc(); | ||||
| 1605 | |||||
| 1606 | int avx3threshold = VM_Version::avx3_threshold(); | ||||
| 1607 | bool use64byteVector = (MaxVectorSize > 32) && (avx3threshold == 0); | ||||
| |||||
| 1608 | |||||
| 1609 | Label L_main_pre_loop, L_main_pre_loop_64bytes, L_pre_main_post_64; | ||||
| 1610 | Label L_main_loop, L_main_loop_64bytes, L_tail, L_tail64, L_exit, L_entry; | ||||
| 1611 | const Register from = rdi; // source array address | ||||
| 1612 | const Register to = rsi; // destination array address | ||||
| 1613 | const Register count = rdx; // elements count | ||||
| 1614 | const Register temp1 = r8; | ||||
| 1615 | const Register temp2 = rcx; | ||||
| 1616 | const Register temp3 = r11; | ||||
| 1617 | const Register temp4 = rax; | ||||
| 1618 | // End pointers are inclusive, and if count is not zero they point | ||||
| 1619 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 1620 | |||||
| 1621 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1622 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 1623 | |||||
| 1624 | if (entry != NULL__null) { | ||||
| 1625 | *entry = __masm-> pc(); | ||||
| 1626 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 1627 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 1628 | } | ||||
| 1629 | |||||
| 1630 | array_overlap_test(nooverlap_target, (Address::ScaleFactor)(shift)); | ||||
| 1631 | |||||
| 1632 | BasicType type_vec[] = { T_BYTE, T_SHORT, T_INT, T_LONG}; | ||||
| 1633 | BasicType type = is_oop ? T_OBJECT : type_vec[shift]; | ||||
| 1634 | |||||
| 1635 | setup_argument_regs(type); | ||||
| 1636 | |||||
| 1637 | DecoratorSet decorators = IN_HEAP | IS_ARRAY; | ||||
| 1638 | if (dest_uninitialized) { | ||||
| 1639 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 1640 | } | ||||
| 1641 | if (aligned) { | ||||
| 1642 | decorators |= ARRAYCOPY_ALIGNED; | ||||
| 1643 | } | ||||
| 1644 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 1645 | bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | ||||
| 1646 | { | ||||
| 1647 | // Type(shift) byte(0), short(1), int(2), long(3) | ||||
| 1648 | int loop_size[] = { 192, 96, 48, 24}; | ||||
| 1649 | int threshold[] = { 4096, 2048, 1024, 512}; | ||||
| 1650 | |||||
| 1651 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 1652 | UnsafeCopyMemoryMark ucmm(this, !is_oop
| ||||
| 1653 | // 'from', 'to' and 'count' are now valid | ||||
| 1654 | |||||
| 1655 | // temp1 holds remaining count. | ||||
| 1656 | __masm-> movq(temp1, count); | ||||
| 1657 | |||||
| 1658 | // Zero length check. | ||||
| 1659 | __masm-> BIND(L_tail)bind(L_tail); masm-> block_comment("L_tail" ":"); | ||||
| 1660 | __masm-> cmpq(temp1, 0); | ||||
| 1661 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 1662 | |||||
| 1663 | __masm-> mov64(temp2, 0); | ||||
| 1664 | __masm-> movq(temp3, temp1); | ||||
| 1665 | // Special cases using 32 byte [masked] vector copy operations. | ||||
| 1666 | __masm-> arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift, | ||||
| 1667 | temp4, use64byteVector, L_entry, L_exit); | ||||
| 1668 | |||||
| 1669 | // PRE-MAIN-POST loop for aligned copy. | ||||
| 1670 | __masm-> BIND(L_entry)bind(L_entry); masm-> block_comment("L_entry" ":"); | ||||
| 1671 | |||||
| 1672 | if ((MaxVectorSize > 32) && (avx3threshold != 0)) { | ||||
| 1673 | __masm-> cmpq(temp1, threshold[shift]); | ||||
| |||||
| 1674 | __masm-> jcc(Assembler::greaterEqual, L_pre_main_post_64); | ||||
| 1675 | } | ||||
| 1676 | |||||
| 1677 | if ((MaxVectorSize < 64) || (avx3threshold != 0)) { | ||||
| 1678 | // Partial copy to make dst address 32 byte aligned. | ||||
| 1679 | __masm-> leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0)); | ||||
| 1680 | __masm-> andq(temp2, 31); | ||||
| 1681 | __masm-> jcc(Assembler::equal, L_main_pre_loop); | ||||
| 1682 | |||||
| 1683 | if (shift) { | ||||
| 1684 | __masm-> shrq(temp2, shift); | ||||
| 1685 | } | ||||
| 1686 | __masm-> subq(temp1, temp2); | ||||
| 1687 | __masm-> copy32_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift); | ||||
| 1688 | |||||
| 1689 | __masm-> cmpq(temp1, loop_size[shift]); | ||||
| 1690 | __masm-> jcc(Assembler::less, L_tail); | ||||
| 1691 | |||||
| 1692 | __masm-> BIND(L_main_pre_loop)bind(L_main_pre_loop); masm-> block_comment("L_main_pre_loop" ":"); | ||||
| 1693 | |||||
| 1694 | // Main loop with aligned copy block size of 192 bytes at 32 byte granularity. | ||||
| 1695 | __masm-> align32(); | ||||
| 1696 | __masm-> BIND(L_main_loop)bind(L_main_loop); masm-> block_comment("L_main_loop" ":"); | ||||
| 1697 | __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -64); | ||||
| 1698 | __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -128); | ||||
| 1699 | __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -192); | ||||
| 1700 | __masm-> subptr(temp1, loop_size[shift]); | ||||
| 1701 | __masm-> cmpq(temp1, loop_size[shift]); | ||||
| 1702 | __masm-> jcc(Assembler::greater, L_main_loop); | ||||
| 1703 | |||||
| 1704 | // Tail loop. | ||||
| 1705 | __masm-> jmp(L_tail); | ||||
| 1706 | } | ||||
| 1707 | |||||
| 1708 | if (MaxVectorSize > 32) { | ||||
| 1709 | __masm-> BIND(L_pre_main_post_64)bind(L_pre_main_post_64); masm-> block_comment("L_pre_main_post_64" ":"); | ||||
| 1710 | // Partial copy to make dst address 64 byte aligned. | ||||
| 1711 | __masm-> leaq(temp2, Address(to, temp1, (Address::ScaleFactor)(shift), 0)); | ||||
| 1712 | __masm-> andq(temp2, 63); | ||||
| 1713 | __masm-> jcc(Assembler::equal, L_main_pre_loop_64bytes); | ||||
| 1714 | |||||
| 1715 | if (shift) { | ||||
| 1716 | __masm-> shrq(temp2, shift); | ||||
| 1717 | } | ||||
| 1718 | __masm-> subq(temp1, temp2); | ||||
| 1719 | __masm-> copy64_masked_avx(to, from, xmm1, k2, temp2, temp1, temp3, shift, 0 , true); | ||||
| 1720 | |||||
| 1721 | __masm-> cmpq(temp1, loop_size[shift]); | ||||
| 1722 | __masm-> jcc(Assembler::less, L_tail64); | ||||
| 1723 | |||||
| 1724 | __masm-> BIND(L_main_pre_loop_64bytes)bind(L_main_pre_loop_64bytes); masm-> block_comment("L_main_pre_loop_64bytes" ":"); | ||||
| 1725 | |||||
| 1726 | // Main loop with aligned copy block size of 192 bytes at | ||||
| 1727 | // 64 byte copy granularity. | ||||
| 1728 | __masm-> align32(); | ||||
| 1729 | __masm-> BIND(L_main_loop_64bytes)bind(L_main_loop_64bytes); masm-> block_comment("L_main_loop_64bytes" ":"); | ||||
| 1730 | __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -64 , true); | ||||
| 1731 | __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -128, true); | ||||
| 1732 | __masm-> copy64_avx(to, from, temp1, xmm1, true, shift, -192, true); | ||||
| 1733 | __masm-> subq(temp1, loop_size[shift]); | ||||
| 1734 | __masm-> cmpq(temp1, loop_size[shift]); | ||||
| 1735 | __masm-> jcc(Assembler::greater, L_main_loop_64bytes); | ||||
| 1736 | |||||
| 1737 | // Zero length check. | ||||
| 1738 | __masm-> cmpq(temp1, 0); | ||||
| 1739 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 1740 | |||||
| 1741 | __masm-> BIND(L_tail64)bind(L_tail64); masm-> block_comment("L_tail64" ":"); | ||||
| 1742 | |||||
| 1743 | // Tail handling using 64 byte [masked] vector copy operations. | ||||
| 1744 | use64byteVector = true; | ||||
| 1745 | __masm-> mov64(temp2, 0); | ||||
| 1746 | __masm-> movq(temp3, temp1); | ||||
| 1747 | __masm-> arraycopy_avx3_special_cases_conjoint(xmm1, k2, from, to, temp2, temp3, temp1, shift, | ||||
| 1748 | temp4, use64byteVector, L_entry, L_exit); | ||||
| 1749 | } | ||||
| 1750 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 1751 | } | ||||
| 1752 | address ucme_exit_pc = __masm-> pc(); | ||||
| 1753 | // When called from generic_arraycopy r11 contains specific values | ||||
| 1754 | // used during arraycopy epilogue, re-initializing r11. | ||||
| 1755 | if(is_oop) { | ||||
| 1756 | __masm-> movq(r11, count); | ||||
| 1757 | } | ||||
| 1758 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, count); | ||||
| 1759 | restore_argument_regs(type); | ||||
| 1760 | inc_counter_np(get_profile_ctr(shift))masm-> block_comment("inc_counter " "get_profile_ctr(shift)" ); inc_counter_np_(get_profile_ctr(shift));; // Update counter after rscratch1 is free | ||||
| 1761 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 1762 | __masm-> vzeroupper(); | ||||
| 1763 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1764 | __masm-> ret(0); | ||||
| 1765 | return start; | ||||
| 1766 | } | ||||
| 1767 | #endif // COMPILER2_OR_JVMCI | ||||
| 1768 | |||||
| 1769 | |||||
| 1770 | // Arguments: | ||||
| 1771 | // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary | ||||
| 1772 | // ignored | ||||
| 1773 | // name - stub name string | ||||
| 1774 | // | ||||
| 1775 | // Inputs: | ||||
| 1776 | // c_rarg0 - source array address | ||||
| 1777 | // c_rarg1 - destination array address | ||||
| 1778 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 1779 | // | ||||
| 1780 | // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, | ||||
| 1781 | // we let the hardware handle it. The one to eight bytes within words, | ||||
| 1782 | // dwords or qwords that span cache line boundaries will still be loaded | ||||
| 1783 | // and stored atomically. | ||||
| 1784 | // | ||||
| 1785 | // Side Effects: | ||||
| 1786 | // disjoint_byte_copy_entry is set to the no-overlap entry point | ||||
| 1787 | // used by generate_conjoint_byte_copy(). | ||||
| 1788 | // | ||||
| 1789 | address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) { | ||||
| 1790 | #if COMPILER2_OR_JVMCI1 | ||||
| 1791 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 1792 | return generate_disjoint_copy_avx3_masked(entry, "jbyte_disjoint_arraycopy_avx3", 0, | ||||
| 1793 | aligned, false, false); | ||||
| 1794 | } | ||||
| 1795 | #endif | ||||
| 1796 | __masm-> align(CodeEntryAlignment); | ||||
| 1797 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 1798 | address start = __masm-> pc(); | ||||
| 1799 | |||||
| 1800 | Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; | ||||
| 1801 | Label L_copy_byte, L_exit; | ||||
| 1802 | const Register from = rdi; // source array address | ||||
| 1803 | const Register to = rsi; // destination array address | ||||
| 1804 | const Register count = rdx; // elements count | ||||
| 1805 | const Register byte_count = rcx; | ||||
| 1806 | const Register qword_count = count; | ||||
| 1807 | const Register end_from = from; // source array end address | ||||
| 1808 | const Register end_to = to; // destination array end address | ||||
| 1809 | // End pointers are inclusive, and if count is not zero they point | ||||
| 1810 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 1811 | |||||
| 1812 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1813 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 1814 | |||||
| 1815 | if (entry != NULL__null) { | ||||
| 1816 | *entry = __masm-> pc(); | ||||
| 1817 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 1818 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 1819 | } | ||||
| 1820 | |||||
| 1821 | setup_arg_regs(); // from => rdi, to => rsi, count => rdx | ||||
| 1822 | // r9 and r10 may be used to save non-volatile registers | ||||
| 1823 | |||||
| 1824 | { | ||||
| 1825 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 1826 | UnsafeCopyMemoryMark ucmm(this, !aligned, true); | ||||
| 1827 | // 'from', 'to' and 'count' are now valid | ||||
| 1828 | __masm-> movptr(byte_count, count); | ||||
| 1829 | __masm-> shrptr(count, 3); // count => qword_count | ||||
| 1830 | |||||
| 1831 | // Copy from low to high addresses. Use 'to' as scratch. | ||||
| 1832 | __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8)); | ||||
| 1833 | __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8)); | ||||
| 1834 | __masm-> negptr(qword_count); // make the count negative | ||||
| 1835 | __masm-> jmp(L_copy_bytes); | ||||
| 1836 | |||||
| 1837 | // Copy trailing qwords | ||||
| 1838 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 1839 | __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | ||||
| 1840 | __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax); | ||||
| 1841 | __masm-> increment(qword_count); | ||||
| 1842 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 1843 | |||||
| 1844 | // Check for and copy trailing dword | ||||
| 1845 | __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes" ":"); | ||||
| 1846 | __masm-> testl(byte_count, 4); | ||||
| 1847 | __masm-> jccb(Assembler::zero, L_copy_2_bytes)jccb_0(Assembler::zero, L_copy_2_bytes, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1847); | ||||
| 1848 | __masm-> movl(rax, Address(end_from, 8)); | ||||
| 1849 | __masm-> movl(Address(end_to, 8), rax); | ||||
| 1850 | |||||
| 1851 | __masm-> addptr(end_from, 4); | ||||
| 1852 | __masm-> addptr(end_to, 4); | ||||
| 1853 | |||||
| 1854 | // Check for and copy trailing word | ||||
| 1855 | __masm-> BIND(L_copy_2_bytes)bind(L_copy_2_bytes); masm-> block_comment("L_copy_2_bytes" ":"); | ||||
| 1856 | __masm-> testl(byte_count, 2); | ||||
| 1857 | __masm-> jccb(Assembler::zero, L_copy_byte)jccb_0(Assembler::zero, L_copy_byte, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1857); | ||||
| 1858 | __masm-> movw(rax, Address(end_from, 8)); | ||||
| 1859 | __masm-> movw(Address(end_to, 8), rax); | ||||
| 1860 | |||||
| 1861 | __masm-> addptr(end_from, 2); | ||||
| 1862 | __masm-> addptr(end_to, 2); | ||||
| 1863 | |||||
| 1864 | // Check for and copy trailing byte | ||||
| 1865 | __masm-> BIND(L_copy_byte)bind(L_copy_byte); masm-> block_comment("L_copy_byte" ":"); | ||||
| 1866 | __masm-> testl(byte_count, 1); | ||||
| 1867 | __masm-> jccb(Assembler::zero, L_exit)jccb_0(Assembler::zero, L_exit, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 1867); | ||||
| 1868 | __masm-> movb(rax, Address(end_from, 8)); | ||||
| 1869 | __masm-> movb(Address(end_to, 8), rax); | ||||
| 1870 | } | ||||
| 1871 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 1872 | address ucme_exit_pc = __masm-> pc(); | ||||
| 1873 | restore_arg_regs(); | ||||
| 1874 | inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jbyte_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jbyte_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 1875 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 1876 | __masm-> vzeroupper(); | ||||
| 1877 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1878 | __masm-> ret(0); | ||||
| 1879 | |||||
| 1880 | { | ||||
| 1881 | UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc); | ||||
| 1882 | // Copy in multi-bytes chunks | ||||
| 1883 | copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 1884 | __masm-> jmp(L_copy_4_bytes); | ||||
| 1885 | } | ||||
| 1886 | return start; | ||||
| 1887 | } | ||||
| 1888 | |||||
| 1889 | // Arguments: | ||||
| 1890 | // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary | ||||
| 1891 | // ignored | ||||
| 1892 | // name - stub name string | ||||
| 1893 | // | ||||
| 1894 | // Inputs: | ||||
| 1895 | // c_rarg0 - source array address | ||||
| 1896 | // c_rarg1 - destination array address | ||||
| 1897 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 1898 | // | ||||
| 1899 | // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, | ||||
| 1900 | // we let the hardware handle it. The one to eight bytes within words, | ||||
| 1901 | // dwords or qwords that span cache line boundaries will still be loaded | ||||
| 1902 | // and stored atomically. | ||||
| 1903 | // | ||||
| 1904 | address generate_conjoint_byte_copy(bool aligned, address nooverlap_target, | ||||
| 1905 | address* entry, const char *name) { | ||||
| 1906 | #if COMPILER2_OR_JVMCI1 | ||||
| 1907 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 1908 | return generate_conjoint_copy_avx3_masked(entry, "jbyte_conjoint_arraycopy_avx3", 0, | ||||
| 1909 | nooverlap_target, aligned, false, false); | ||||
| 1910 | } | ||||
| 1911 | #endif | ||||
| 1912 | __masm-> align(CodeEntryAlignment); | ||||
| 1913 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 1914 | address start = __masm-> pc(); | ||||
| 1915 | |||||
| 1916 | Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; | ||||
| 1917 | const Register from = rdi; // source array address | ||||
| 1918 | const Register to = rsi; // destination array address | ||||
| 1919 | const Register count = rdx; // elements count | ||||
| 1920 | const Register byte_count = rcx; | ||||
| 1921 | const Register qword_count = count; | ||||
| 1922 | |||||
| 1923 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1924 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 1925 | |||||
| 1926 | if (entry != NULL__null) { | ||||
| 1927 | *entry = __masm-> pc(); | ||||
| 1928 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 1929 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 1930 | } | ||||
| 1931 | |||||
| 1932 | array_overlap_test(nooverlap_target, Address::times_1); | ||||
| 1933 | setup_arg_regs(); // from => rdi, to => rsi, count => rdx | ||||
| 1934 | // r9 and r10 may be used to save non-volatile registers | ||||
| 1935 | |||||
| 1936 | { | ||||
| 1937 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 1938 | UnsafeCopyMemoryMark ucmm(this, !aligned, true); | ||||
| 1939 | // 'from', 'to' and 'count' are now valid | ||||
| 1940 | __masm-> movptr(byte_count, count); | ||||
| 1941 | __masm-> shrptr(count, 3); // count => qword_count | ||||
| 1942 | |||||
| 1943 | // Copy from high to low addresses. | ||||
| 1944 | |||||
| 1945 | // Check for and copy trailing byte | ||||
| 1946 | __masm-> testl(byte_count, 1); | ||||
| 1947 | __masm-> jcc(Assembler::zero, L_copy_2_bytes); | ||||
| 1948 | __masm-> movb(rax, Address(from, byte_count, Address::times_1, -1)); | ||||
| 1949 | __masm-> movb(Address(to, byte_count, Address::times_1, -1), rax); | ||||
| 1950 | __masm-> decrement(byte_count); // Adjust for possible trailing word | ||||
| 1951 | |||||
| 1952 | // Check for and copy trailing word | ||||
| 1953 | __masm-> BIND(L_copy_2_bytes)bind(L_copy_2_bytes); masm-> block_comment("L_copy_2_bytes" ":"); | ||||
| 1954 | __masm-> testl(byte_count, 2); | ||||
| 1955 | __masm-> jcc(Assembler::zero, L_copy_4_bytes); | ||||
| 1956 | __masm-> movw(rax, Address(from, byte_count, Address::times_1, -2)); | ||||
| 1957 | __masm-> movw(Address(to, byte_count, Address::times_1, -2), rax); | ||||
| 1958 | |||||
| 1959 | // Check for and copy trailing dword | ||||
| 1960 | __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes" ":"); | ||||
| 1961 | __masm-> testl(byte_count, 4); | ||||
| 1962 | __masm-> jcc(Assembler::zero, L_copy_bytes); | ||||
| 1963 | __masm-> movl(rax, Address(from, qword_count, Address::times_8)); | ||||
| 1964 | __masm-> movl(Address(to, qword_count, Address::times_8), rax); | ||||
| 1965 | __masm-> jmp(L_copy_bytes); | ||||
| 1966 | |||||
| 1967 | // Copy trailing qwords | ||||
| 1968 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 1969 | __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8)); | ||||
| 1970 | __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax); | ||||
| 1971 | __masm-> decrement(qword_count); | ||||
| 1972 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 1973 | } | ||||
| 1974 | restore_arg_regs(); | ||||
| 1975 | inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jbyte_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jbyte_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 1976 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 1977 | __masm-> vzeroupper(); | ||||
| 1978 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1979 | __masm-> ret(0); | ||||
| 1980 | |||||
| 1981 | { | ||||
| 1982 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 1983 | UnsafeCopyMemoryMark ucmm(this, !aligned, true); | ||||
| 1984 | // Copy in multi-bytes chunks | ||||
| 1985 | copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 1986 | } | ||||
| 1987 | restore_arg_regs(); | ||||
| 1988 | inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jbyte_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jbyte_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 1989 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 1990 | __masm-> vzeroupper(); | ||||
| 1991 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 1992 | __masm-> ret(0); | ||||
| 1993 | |||||
| 1994 | return start; | ||||
| 1995 | } | ||||
| 1996 | |||||
| 1997 | // Arguments: | ||||
| 1998 | // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary | ||||
| 1999 | // ignored | ||||
| 2000 | // name - stub name string | ||||
| 2001 | // | ||||
| 2002 | // Inputs: | ||||
| 2003 | // c_rarg0 - source array address | ||||
| 2004 | // c_rarg1 - destination array address | ||||
| 2005 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2006 | // | ||||
| 2007 | // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we | ||||
| 2008 | // let the hardware handle it. The two or four words within dwords | ||||
| 2009 | // or qwords that span cache line boundaries will still be loaded | ||||
| 2010 | // and stored atomically. | ||||
| 2011 | // | ||||
| 2012 | // Side Effects: | ||||
| 2013 | // disjoint_short_copy_entry is set to the no-overlap entry point | ||||
| 2014 | // used by generate_conjoint_short_copy(). | ||||
| 2015 | // | ||||
| 2016 | address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) { | ||||
| 2017 | #if COMPILER2_OR_JVMCI1 | ||||
| 2018 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 2019 | return generate_disjoint_copy_avx3_masked(entry, "jshort_disjoint_arraycopy_avx3", 1, | ||||
| 2020 | aligned, false, false); | ||||
| 2021 | } | ||||
| 2022 | #endif | ||||
| 2023 | |||||
| 2024 | __masm-> align(CodeEntryAlignment); | ||||
| 2025 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2026 | address start = __masm-> pc(); | ||||
| 2027 | |||||
| 2028 | Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit; | ||||
| 2029 | const Register from = rdi; // source array address | ||||
| 2030 | const Register to = rsi; // destination array address | ||||
| 2031 | const Register count = rdx; // elements count | ||||
| 2032 | const Register word_count = rcx; | ||||
| 2033 | const Register qword_count = count; | ||||
| 2034 | const Register end_from = from; // source array end address | ||||
| 2035 | const Register end_to = to; // destination array end address | ||||
| 2036 | // End pointers are inclusive, and if count is not zero they point | ||||
| 2037 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 2038 | |||||
| 2039 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2040 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 2041 | |||||
| 2042 | if (entry != NULL__null) { | ||||
| 2043 | *entry = __masm-> pc(); | ||||
| 2044 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 2045 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2046 | } | ||||
| 2047 | |||||
| 2048 | setup_arg_regs(); // from => rdi, to => rsi, count => rdx | ||||
| 2049 | // r9 and r10 may be used to save non-volatile registers | ||||
| 2050 | |||||
| 2051 | { | ||||
| 2052 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2053 | UnsafeCopyMemoryMark ucmm(this, !aligned, true); | ||||
| 2054 | // 'from', 'to' and 'count' are now valid | ||||
| 2055 | __masm-> movptr(word_count, count); | ||||
| 2056 | __masm-> shrptr(count, 2); // count => qword_count | ||||
| 2057 | |||||
| 2058 | // Copy from low to high addresses. Use 'to' as scratch. | ||||
| 2059 | __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8)); | ||||
| 2060 | __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8)); | ||||
| 2061 | __masm-> negptr(qword_count); | ||||
| 2062 | __masm-> jmp(L_copy_bytes); | ||||
| 2063 | |||||
| 2064 | // Copy trailing qwords | ||||
| 2065 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 2066 | __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | ||||
| 2067 | __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax); | ||||
| 2068 | __masm-> increment(qword_count); | ||||
| 2069 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 2070 | |||||
| 2071 | // Original 'dest' is trashed, so we can't use it as a | ||||
| 2072 | // base register for a possible trailing word copy | ||||
| 2073 | |||||
| 2074 | // Check for and copy trailing dword | ||||
| 2075 | __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes" ":"); | ||||
| 2076 | __masm-> testl(word_count, 2); | ||||
| 2077 | __masm-> jccb(Assembler::zero, L_copy_2_bytes)jccb_0(Assembler::zero, L_copy_2_bytes, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2077); | ||||
| 2078 | __masm-> movl(rax, Address(end_from, 8)); | ||||
| 2079 | __masm-> movl(Address(end_to, 8), rax); | ||||
| 2080 | |||||
| 2081 | __masm-> addptr(end_from, 4); | ||||
| 2082 | __masm-> addptr(end_to, 4); | ||||
| 2083 | |||||
| 2084 | // Check for and copy trailing word | ||||
| 2085 | __masm-> BIND(L_copy_2_bytes)bind(L_copy_2_bytes); masm-> block_comment("L_copy_2_bytes" ":"); | ||||
| 2086 | __masm-> testl(word_count, 1); | ||||
| 2087 | __masm-> jccb(Assembler::zero, L_exit)jccb_0(Assembler::zero, L_exit, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2087); | ||||
| 2088 | __masm-> movw(rax, Address(end_from, 8)); | ||||
| 2089 | __masm-> movw(Address(end_to, 8), rax); | ||||
| 2090 | } | ||||
| 2091 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 2092 | address ucme_exit_pc = __masm-> pc(); | ||||
| 2093 | restore_arg_regs(); | ||||
| 2094 | inc_counter_np(SharedRuntime::_jshort_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jshort_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jshort_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2095 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2096 | __masm-> vzeroupper(); | ||||
| 2097 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2098 | __masm-> ret(0); | ||||
| 2099 | |||||
| 2100 | { | ||||
| 2101 | UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc); | ||||
| 2102 | // Copy in multi-bytes chunks | ||||
| 2103 | copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 2104 | __masm-> jmp(L_copy_4_bytes); | ||||
| 2105 | } | ||||
| 2106 | |||||
| 2107 | return start; | ||||
| 2108 | } | ||||
| 2109 | |||||
| 2110 | address generate_fill(BasicType t, bool aligned, const char *name) { | ||||
| 2111 | __masm-> align(CodeEntryAlignment); | ||||
| 2112 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2113 | address start = __masm-> pc(); | ||||
| 2114 | |||||
| 2115 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2116 | |||||
| 2117 | const Register to = c_rarg0; // destination array address | ||||
| 2118 | const Register value = c_rarg1; // value | ||||
| 2119 | const Register count = c_rarg2; // elements count | ||||
| 2120 | __masm-> mov(r11, count); | ||||
| 2121 | |||||
| 2122 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2123 | |||||
| 2124 | __masm-> generate_fill(t, aligned, to, value, r11, rax, xmm0); | ||||
| 2125 | |||||
| 2126 | __masm-> vzeroupper(); | ||||
| 2127 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2128 | __masm-> ret(0); | ||||
| 2129 | return start; | ||||
| 2130 | } | ||||
| 2131 | |||||
| 2132 | // Arguments: | ||||
| 2133 | // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary | ||||
| 2134 | // ignored | ||||
| 2135 | // name - stub name string | ||||
| 2136 | // | ||||
| 2137 | // Inputs: | ||||
| 2138 | // c_rarg0 - source array address | ||||
| 2139 | // c_rarg1 - destination array address | ||||
| 2140 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2141 | // | ||||
| 2142 | // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we | ||||
| 2143 | // let the hardware handle it. The two or four words within dwords | ||||
| 2144 | // or qwords that span cache line boundaries will still be loaded | ||||
| 2145 | // and stored atomically. | ||||
| 2146 | // | ||||
| 2147 | address generate_conjoint_short_copy(bool aligned, address nooverlap_target, | ||||
| 2148 | address *entry, const char *name) { | ||||
| 2149 | #if COMPILER2_OR_JVMCI1 | ||||
| 2150 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 2151 | return generate_conjoint_copy_avx3_masked(entry, "jshort_conjoint_arraycopy_avx3", 1, | ||||
| 2152 | nooverlap_target, aligned, false, false); | ||||
| 2153 | } | ||||
| 2154 | #endif | ||||
| 2155 | __masm-> align(CodeEntryAlignment); | ||||
| 2156 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2157 | address start = __masm-> pc(); | ||||
| 2158 | |||||
| 2159 | Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes; | ||||
| 2160 | const Register from = rdi; // source array address | ||||
| 2161 | const Register to = rsi; // destination array address | ||||
| 2162 | const Register count = rdx; // elements count | ||||
| 2163 | const Register word_count = rcx; | ||||
| 2164 | const Register qword_count = count; | ||||
| 2165 | |||||
| 2166 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2167 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 2168 | |||||
| 2169 | if (entry != NULL__null) { | ||||
| 2170 | *entry = __masm-> pc(); | ||||
| 2171 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 2172 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2173 | } | ||||
| 2174 | |||||
| 2175 | array_overlap_test(nooverlap_target, Address::times_2); | ||||
| 2176 | setup_arg_regs(); // from => rdi, to => rsi, count => rdx | ||||
| 2177 | // r9 and r10 may be used to save non-volatile registers | ||||
| 2178 | |||||
| 2179 | { | ||||
| 2180 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2181 | UnsafeCopyMemoryMark ucmm(this, !aligned, true); | ||||
| 2182 | // 'from', 'to' and 'count' are now valid | ||||
| 2183 | __masm-> movptr(word_count, count); | ||||
| 2184 | __masm-> shrptr(count, 2); // count => qword_count | ||||
| 2185 | |||||
| 2186 | // Copy from high to low addresses. Use 'to' as scratch. | ||||
| 2187 | |||||
| 2188 | // Check for and copy trailing word | ||||
| 2189 | __masm-> testl(word_count, 1); | ||||
| 2190 | __masm-> jccb(Assembler::zero, L_copy_4_bytes)jccb_0(Assembler::zero, L_copy_4_bytes, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2190); | ||||
| 2191 | __masm-> movw(rax, Address(from, word_count, Address::times_2, -2)); | ||||
| 2192 | __masm-> movw(Address(to, word_count, Address::times_2, -2), rax); | ||||
| 2193 | |||||
| 2194 | // Check for and copy trailing dword | ||||
| 2195 | __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes" ":"); | ||||
| 2196 | __masm-> testl(word_count, 2); | ||||
| 2197 | __masm-> jcc(Assembler::zero, L_copy_bytes); | ||||
| 2198 | __masm-> movl(rax, Address(from, qword_count, Address::times_8)); | ||||
| 2199 | __masm-> movl(Address(to, qword_count, Address::times_8), rax); | ||||
| 2200 | __masm-> jmp(L_copy_bytes); | ||||
| 2201 | |||||
| 2202 | // Copy trailing qwords | ||||
| 2203 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 2204 | __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8)); | ||||
| 2205 | __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax); | ||||
| 2206 | __masm-> decrement(qword_count); | ||||
| 2207 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 2208 | } | ||||
| 2209 | restore_arg_regs(); | ||||
| 2210 | inc_counter_np(SharedRuntime::_jshort_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jshort_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jshort_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2211 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2212 | __masm-> vzeroupper(); | ||||
| 2213 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2214 | __masm-> ret(0); | ||||
| 2215 | |||||
| 2216 | { | ||||
| 2217 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2218 | UnsafeCopyMemoryMark ucmm(this, !aligned, true); | ||||
| 2219 | // Copy in multi-bytes chunks | ||||
| 2220 | copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 2221 | } | ||||
| 2222 | restore_arg_regs(); | ||||
| 2223 | inc_counter_np(SharedRuntime::_jshort_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jshort_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jshort_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2224 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2225 | __masm-> vzeroupper(); | ||||
| 2226 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2227 | __masm-> ret(0); | ||||
| 2228 | |||||
| 2229 | return start; | ||||
| 2230 | } | ||||
| 2231 | |||||
| 2232 | // Arguments: | ||||
| 2233 | // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary | ||||
| 2234 | // ignored | ||||
| 2235 | // is_oop - true => oop array, so generate store check code | ||||
| 2236 | // name - stub name string | ||||
| 2237 | // | ||||
| 2238 | // Inputs: | ||||
| 2239 | // c_rarg0 - source array address | ||||
| 2240 | // c_rarg1 - destination array address | ||||
| 2241 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2242 | // | ||||
| 2243 | // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let | ||||
| 2244 | // the hardware handle it. The two dwords within qwords that span | ||||
| 2245 | // cache line boundaries will still be loaded and stored atomicly. | ||||
| 2246 | // | ||||
| 2247 | // Side Effects: | ||||
| 2248 | // disjoint_int_copy_entry is set to the no-overlap entry point | ||||
| 2249 | // used by generate_conjoint_int_oop_copy(). | ||||
| 2250 | // | ||||
| 2251 | address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry, | ||||
| 2252 | const char *name, bool dest_uninitialized = false) { | ||||
| 2253 | #if COMPILER2_OR_JVMCI1 | ||||
| 2254 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 2255 | return generate_disjoint_copy_avx3_masked(entry, "jint_disjoint_arraycopy_avx3", 2, | ||||
| 2256 | aligned, is_oop, dest_uninitialized); | ||||
| 2257 | } | ||||
| 2258 | #endif | ||||
| 2259 | |||||
| 2260 | __masm-> align(CodeEntryAlignment); | ||||
| 2261 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2262 | address start = __masm-> pc(); | ||||
| 2263 | |||||
| 2264 | Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit; | ||||
| 2265 | const Register from = rdi; // source array address | ||||
| 2266 | const Register to = rsi; // destination array address | ||||
| 2267 | const Register count = rdx; // elements count | ||||
| 2268 | const Register dword_count = rcx; | ||||
| 2269 | const Register qword_count = count; | ||||
| 2270 | const Register end_from = from; // source array end address | ||||
| 2271 | const Register end_to = to; // destination array end address | ||||
| 2272 | // End pointers are inclusive, and if count is not zero they point | ||||
| 2273 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 2274 | |||||
| 2275 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2276 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 2277 | |||||
| 2278 | if (entry != NULL__null) { | ||||
| 2279 | *entry = __masm-> pc(); | ||||
| 2280 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 2281 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2282 | } | ||||
| 2283 | |||||
| 2284 | setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | ||||
| 2285 | // r9 is used to save r15_thread | ||||
| 2286 | |||||
| 2287 | DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | ||||
| 2288 | if (dest_uninitialized) { | ||||
| 2289 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 2290 | } | ||||
| 2291 | if (aligned) { | ||||
| 2292 | decorators |= ARRAYCOPY_ALIGNED; | ||||
| 2293 | } | ||||
| 2294 | |||||
| 2295 | BasicType type = is_oop ? T_OBJECT : T_INT; | ||||
| 2296 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 2297 | bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | ||||
| 2298 | |||||
| 2299 | { | ||||
| 2300 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2301 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2302 | // 'from', 'to' and 'count' are now valid | ||||
| 2303 | __masm-> movptr(dword_count, count); | ||||
| 2304 | __masm-> shrptr(count, 1); // count => qword_count | ||||
| 2305 | |||||
| 2306 | // Copy from low to high addresses. Use 'to' as scratch. | ||||
| 2307 | __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8)); | ||||
| 2308 | __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8)); | ||||
| 2309 | __masm-> negptr(qword_count); | ||||
| 2310 | __masm-> jmp(L_copy_bytes); | ||||
| 2311 | |||||
| 2312 | // Copy trailing qwords | ||||
| 2313 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 2314 | __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | ||||
| 2315 | __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax); | ||||
| 2316 | __masm-> increment(qword_count); | ||||
| 2317 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 2318 | |||||
| 2319 | // Check for and copy trailing dword | ||||
| 2320 | __masm-> BIND(L_copy_4_bytes)bind(L_copy_4_bytes); masm-> block_comment("L_copy_4_bytes" ":"); | ||||
| 2321 | __masm-> testl(dword_count, 1); // Only byte test since the value is 0 or 1 | ||||
| 2322 | __masm-> jccb(Assembler::zero, L_exit)jccb_0(Assembler::zero, L_exit, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2322); | ||||
| 2323 | __masm-> movl(rax, Address(end_from, 8)); | ||||
| 2324 | __masm-> movl(Address(end_to, 8), rax); | ||||
| 2325 | } | ||||
| 2326 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 2327 | address ucme_exit_pc = __masm-> pc(); | ||||
| 2328 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); | ||||
| 2329 | restore_arg_regs_using_thread(); | ||||
| 2330 | inc_counter_np(SharedRuntime::_jint_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jint_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jint_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2331 | __masm-> vzeroupper(); | ||||
| 2332 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2333 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2334 | __masm-> ret(0); | ||||
| 2335 | |||||
| 2336 | { | ||||
| 2337 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, false, ucme_exit_pc); | ||||
| 2338 | // Copy in multi-bytes chunks | ||||
| 2339 | copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 2340 | __masm-> jmp(L_copy_4_bytes); | ||||
| 2341 | } | ||||
| 2342 | |||||
| 2343 | return start; | ||||
| 2344 | } | ||||
| 2345 | |||||
| 2346 | // Arguments: | ||||
| 2347 | // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary | ||||
| 2348 | // ignored | ||||
| 2349 | // is_oop - true => oop array, so generate store check code | ||||
| 2350 | // name - stub name string | ||||
| 2351 | // | ||||
| 2352 | // Inputs: | ||||
| 2353 | // c_rarg0 - source array address | ||||
| 2354 | // c_rarg1 - destination array address | ||||
| 2355 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2356 | // | ||||
| 2357 | // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let | ||||
| 2358 | // the hardware handle it. The two dwords within qwords that span | ||||
| 2359 | // cache line boundaries will still be loaded and stored atomicly. | ||||
| 2360 | // | ||||
| 2361 | address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target, | ||||
| 2362 | address *entry, const char *name, | ||||
| 2363 | bool dest_uninitialized = false) { | ||||
| 2364 | #if COMPILER2_OR_JVMCI1 | ||||
| 2365 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 2366 | return generate_conjoint_copy_avx3_masked(entry, "jint_conjoint_arraycopy_avx3", 2, | ||||
| 2367 | nooverlap_target, aligned, is_oop, dest_uninitialized); | ||||
| 2368 | } | ||||
| 2369 | #endif | ||||
| 2370 | __masm-> align(CodeEntryAlignment); | ||||
| 2371 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2372 | address start = __masm-> pc(); | ||||
| 2373 | |||||
| 2374 | Label L_copy_bytes, L_copy_8_bytes, L_exit; | ||||
| 2375 | const Register from = rdi; // source array address | ||||
| 2376 | const Register to = rsi; // destination array address | ||||
| 2377 | const Register count = rdx; // elements count | ||||
| 2378 | const Register dword_count = rcx; | ||||
| 2379 | const Register qword_count = count; | ||||
| 2380 | |||||
| 2381 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2382 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 2383 | |||||
| 2384 | if (entry != NULL__null) { | ||||
| 2385 | *entry = __masm-> pc(); | ||||
| 2386 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 2387 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2388 | } | ||||
| 2389 | |||||
| 2390 | array_overlap_test(nooverlap_target, Address::times_4); | ||||
| 2391 | setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | ||||
| 2392 | // r9 is used to save r15_thread | ||||
| 2393 | |||||
| 2394 | DecoratorSet decorators = IN_HEAP | IS_ARRAY; | ||||
| 2395 | if (dest_uninitialized) { | ||||
| 2396 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 2397 | } | ||||
| 2398 | if (aligned) { | ||||
| 2399 | decorators |= ARRAYCOPY_ALIGNED; | ||||
| 2400 | } | ||||
| 2401 | |||||
| 2402 | BasicType type = is_oop ? T_OBJECT : T_INT; | ||||
| 2403 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 2404 | // no registers are destroyed by this call | ||||
| 2405 | bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | ||||
| 2406 | |||||
| 2407 | assert_clean_int(count, rax); // Make sure 'count' is clean int. | ||||
| 2408 | { | ||||
| 2409 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2410 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2411 | // 'from', 'to' and 'count' are now valid | ||||
| 2412 | __masm-> movptr(dword_count, count); | ||||
| 2413 | __masm-> shrptr(count, 1); // count => qword_count | ||||
| 2414 | |||||
| 2415 | // Copy from high to low addresses. Use 'to' as scratch. | ||||
| 2416 | |||||
| 2417 | // Check for and copy trailing dword | ||||
| 2418 | __masm-> testl(dword_count, 1); | ||||
| 2419 | __masm-> jcc(Assembler::zero, L_copy_bytes); | ||||
| 2420 | __masm-> movl(rax, Address(from, dword_count, Address::times_4, -4)); | ||||
| 2421 | __masm-> movl(Address(to, dword_count, Address::times_4, -4), rax); | ||||
| 2422 | __masm-> jmp(L_copy_bytes); | ||||
| 2423 | |||||
| 2424 | // Copy trailing qwords | ||||
| 2425 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 2426 | __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8)); | ||||
| 2427 | __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax); | ||||
| 2428 | __masm-> decrement(qword_count); | ||||
| 2429 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 2430 | } | ||||
| 2431 | if (is_oop) { | ||||
| 2432 | __masm-> jmp(L_exit); | ||||
| 2433 | } | ||||
| 2434 | restore_arg_regs_using_thread(); | ||||
| 2435 | inc_counter_np(SharedRuntime::_jint_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jint_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jint_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2436 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2437 | __masm-> vzeroupper(); | ||||
| 2438 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2439 | __masm-> ret(0); | ||||
| 2440 | |||||
| 2441 | { | ||||
| 2442 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2443 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2444 | // Copy in multi-bytes chunks | ||||
| 2445 | copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 2446 | } | ||||
| 2447 | |||||
| 2448 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 2449 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count); | ||||
| 2450 | restore_arg_regs_using_thread(); | ||||
| 2451 | inc_counter_np(SharedRuntime::_jint_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jint_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jint_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2452 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2453 | __masm-> vzeroupper(); | ||||
| 2454 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2455 | __masm-> ret(0); | ||||
| 2456 | |||||
| 2457 | return start; | ||||
| 2458 | } | ||||
| 2459 | |||||
| 2460 | // Arguments: | ||||
| 2461 | // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes | ||||
| 2462 | // ignored | ||||
| 2463 | // is_oop - true => oop array, so generate store check code | ||||
| 2464 | // name - stub name string | ||||
| 2465 | // | ||||
| 2466 | // Inputs: | ||||
| 2467 | // c_rarg0 - source array address | ||||
| 2468 | // c_rarg1 - destination array address | ||||
| 2469 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2470 | // | ||||
| 2471 | // Side Effects: | ||||
| 2472 | // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the | ||||
| 2473 | // no-overlap entry point used by generate_conjoint_long_oop_copy(). | ||||
| 2474 | // | ||||
| 2475 | address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry, | ||||
| 2476 | const char *name, bool dest_uninitialized = false) { | ||||
| 2477 | #if COMPILER2_OR_JVMCI1 | ||||
| 2478 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 2479 | return generate_disjoint_copy_avx3_masked(entry, "jlong_disjoint_arraycopy_avx3", 3, | ||||
| 2480 | aligned, is_oop, dest_uninitialized); | ||||
| 2481 | } | ||||
| 2482 | #endif | ||||
| 2483 | __masm-> align(CodeEntryAlignment); | ||||
| 2484 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2485 | address start = __masm-> pc(); | ||||
| 2486 | |||||
| 2487 | Label L_copy_bytes, L_copy_8_bytes, L_exit; | ||||
| 2488 | const Register from = rdi; // source array address | ||||
| 2489 | const Register to = rsi; // destination array address | ||||
| 2490 | const Register qword_count = rdx; // elements count | ||||
| 2491 | const Register end_from = from; // source array end address | ||||
| 2492 | const Register end_to = rcx; // destination array end address | ||||
| 2493 | const Register saved_count = r11; | ||||
| 2494 | // End pointers are inclusive, and if count is not zero they point | ||||
| 2495 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 2496 | |||||
| 2497 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2498 | // Save no-overlap entry point for generate_conjoint_long_oop_copy() | ||||
| 2499 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 2500 | |||||
| 2501 | if (entry != NULL__null) { | ||||
| 2502 | *entry = __masm-> pc(); | ||||
| 2503 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 2504 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2505 | } | ||||
| 2506 | |||||
| 2507 | setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | ||||
| 2508 | // r9 is used to save r15_thread | ||||
| 2509 | // 'from', 'to' and 'qword_count' are now valid | ||||
| 2510 | |||||
| 2511 | DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT; | ||||
| 2512 | if (dest_uninitialized) { | ||||
| 2513 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 2514 | } | ||||
| 2515 | if (aligned) { | ||||
| 2516 | decorators |= ARRAYCOPY_ALIGNED; | ||||
| 2517 | } | ||||
| 2518 | |||||
| 2519 | BasicType type = is_oop ? T_OBJECT : T_LONG; | ||||
| 2520 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 2521 | bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); | ||||
| 2522 | { | ||||
| 2523 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2524 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2525 | |||||
| 2526 | // Copy from low to high addresses. Use 'to' as scratch. | ||||
| 2527 | __masm-> lea(end_from, Address(from, qword_count, Address::times_8, -8)); | ||||
| 2528 | __masm-> lea(end_to, Address(to, qword_count, Address::times_8, -8)); | ||||
| 2529 | __masm-> negptr(qword_count); | ||||
| 2530 | __masm-> jmp(L_copy_bytes); | ||||
| 2531 | |||||
| 2532 | // Copy trailing qwords | ||||
| 2533 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 2534 | __masm-> movq(rax, Address(end_from, qword_count, Address::times_8, 8)); | ||||
| 2535 | __masm-> movq(Address(end_to, qword_count, Address::times_8, 8), rax); | ||||
| 2536 | __masm-> increment(qword_count); | ||||
| 2537 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 2538 | } | ||||
| 2539 | if (is_oop) { | ||||
| 2540 | __masm-> jmp(L_exit); | ||||
| 2541 | } else { | ||||
| 2542 | restore_arg_regs_using_thread(); | ||||
| 2543 | inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2544 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2545 | __masm-> vzeroupper(); | ||||
| 2546 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2547 | __masm-> ret(0); | ||||
| 2548 | } | ||||
| 2549 | |||||
| 2550 | { | ||||
| 2551 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2552 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2553 | // Copy in multi-bytes chunks | ||||
| 2554 | copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 2555 | } | ||||
| 2556 | |||||
| 2557 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 2558 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); | ||||
| 2559 | restore_arg_regs_using_thread(); | ||||
| 2560 | if (is_oop) { | ||||
| 2561 | inc_counter_np(SharedRuntime::_oop_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_oop_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_oop_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2562 | } else { | ||||
| 2563 | inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2564 | } | ||||
| 2565 | __masm-> vzeroupper(); | ||||
| 2566 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2567 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2568 | __masm-> ret(0); | ||||
| 2569 | |||||
| 2570 | return start; | ||||
| 2571 | } | ||||
| 2572 | |||||
| 2573 | // Arguments: | ||||
| 2574 | // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes | ||||
| 2575 | // ignored | ||||
| 2576 | // is_oop - true => oop array, so generate store check code | ||||
| 2577 | // name - stub name string | ||||
| 2578 | // | ||||
| 2579 | // Inputs: | ||||
| 2580 | // c_rarg0 - source array address | ||||
| 2581 | // c_rarg1 - destination array address | ||||
| 2582 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2583 | // | ||||
| 2584 | address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, | ||||
| 2585 | address nooverlap_target, address *entry, | ||||
| 2586 | const char *name, bool dest_uninitialized = false) { | ||||
| 2587 | #if COMPILER2_OR_JVMCI1 | ||||
| 2588 | if (VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2() && MaxVectorSize >= 32) { | ||||
| 2589 | return generate_conjoint_copy_avx3_masked(entry, "jlong_conjoint_arraycopy_avx3", 3, | ||||
| 2590 | nooverlap_target, aligned, is_oop, dest_uninitialized); | ||||
| 2591 | } | ||||
| 2592 | #endif | ||||
| 2593 | __masm-> align(CodeEntryAlignment); | ||||
| 2594 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2595 | address start = __masm-> pc(); | ||||
| 2596 | |||||
| 2597 | Label L_copy_bytes, L_copy_8_bytes, L_exit; | ||||
| 2598 | const Register from = rdi; // source array address | ||||
| 2599 | const Register to = rsi; // destination array address | ||||
| 2600 | const Register qword_count = rdx; // elements count | ||||
| 2601 | const Register saved_count = rcx; | ||||
| 2602 | |||||
| 2603 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2604 | assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. | ||||
| 2605 | |||||
| 2606 | if (entry != NULL__null) { | ||||
| 2607 | *entry = __masm-> pc(); | ||||
| 2608 | // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) | ||||
| 2609 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2610 | } | ||||
| 2611 | |||||
| 2612 | array_overlap_test(nooverlap_target, Address::times_8); | ||||
| 2613 | setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx | ||||
| 2614 | // r9 is used to save r15_thread | ||||
| 2615 | // 'from', 'to' and 'qword_count' are now valid | ||||
| 2616 | |||||
| 2617 | DecoratorSet decorators = IN_HEAP | IS_ARRAY; | ||||
| 2618 | if (dest_uninitialized) { | ||||
| 2619 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 2620 | } | ||||
| 2621 | if (aligned) { | ||||
| 2622 | decorators |= ARRAYCOPY_ALIGNED; | ||||
| 2623 | } | ||||
| 2624 | |||||
| 2625 | BasicType type = is_oop ? T_OBJECT : T_LONG; | ||||
| 2626 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 2627 | bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count); | ||||
| 2628 | { | ||||
| 2629 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2630 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2631 | |||||
| 2632 | __masm-> jmp(L_copy_bytes); | ||||
| 2633 | |||||
| 2634 | // Copy trailing qwords | ||||
| 2635 | __masm-> BIND(L_copy_8_bytes)bind(L_copy_8_bytes); masm-> block_comment("L_copy_8_bytes" ":"); | ||||
| 2636 | __masm-> movq(rax, Address(from, qword_count, Address::times_8, -8)); | ||||
| 2637 | __masm-> movq(Address(to, qword_count, Address::times_8, -8), rax); | ||||
| 2638 | __masm-> decrement(qword_count); | ||||
| 2639 | __masm-> jcc(Assembler::notZero, L_copy_8_bytes); | ||||
| 2640 | } | ||||
| 2641 | if (is_oop) { | ||||
| 2642 | __masm-> jmp(L_exit); | ||||
| 2643 | } else { | ||||
| 2644 | restore_arg_regs_using_thread(); | ||||
| 2645 | inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2646 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2647 | __masm-> vzeroupper(); | ||||
| 2648 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2649 | __masm-> ret(0); | ||||
| 2650 | } | ||||
| 2651 | { | ||||
| 2652 | // UnsafeCopyMemory page error: continue after ucm | ||||
| 2653 | UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true); | ||||
| 2654 | |||||
| 2655 | // Copy in multi-bytes chunks | ||||
| 2656 | copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); | ||||
| 2657 | } | ||||
| 2658 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 2659 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count); | ||||
| 2660 | restore_arg_regs_using_thread(); | ||||
| 2661 | if (is_oop) { | ||||
| 2662 | inc_counter_np(SharedRuntime::_oop_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_oop_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_oop_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2663 | } else { | ||||
| 2664 | inc_counter_np(SharedRuntime::_jlong_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_jlong_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_jlong_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2665 | } | ||||
| 2666 | __masm-> vzeroupper(); | ||||
| 2667 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 2668 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2669 | __masm-> ret(0); | ||||
| 2670 | |||||
| 2671 | return start; | ||||
| 2672 | } | ||||
| 2673 | |||||
| 2674 | |||||
| 2675 | // Helper for generating a dynamic type check. | ||||
| 2676 | // Smashes no registers. | ||||
| 2677 | void generate_type_check(Register sub_klass, | ||||
| 2678 | Register super_check_offset, | ||||
| 2679 | Register super_klass, | ||||
| 2680 | Label& L_success) { | ||||
| 2681 | assert_different_registers(sub_klass, super_check_offset, super_klass); | ||||
| 2682 | |||||
| 2683 | BLOCK_COMMENT("type_check:")masm-> block_comment("type_check:"); | ||||
| 2684 | |||||
| 2685 | Label L_miss; | ||||
| 2686 | |||||
| 2687 | __masm-> check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, NULL__null, | ||||
| 2688 | super_check_offset); | ||||
| 2689 | __masm-> check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL__null); | ||||
| 2690 | |||||
| 2691 | // Fall through on failure! | ||||
| 2692 | __masm-> BIND(L_miss)bind(L_miss); masm-> block_comment("L_miss" ":"); | ||||
| 2693 | } | ||||
| 2694 | |||||
| 2695 | // | ||||
| 2696 | // Generate checkcasting array copy stub | ||||
| 2697 | // | ||||
| 2698 | // Input: | ||||
| 2699 | // c_rarg0 - source array address | ||||
| 2700 | // c_rarg1 - destination array address | ||||
| 2701 | // c_rarg2 - element count, treated as ssize_t, can be zero | ||||
| 2702 | // c_rarg3 - size_t ckoff (super_check_offset) | ||||
| 2703 | // not Win64 | ||||
| 2704 | // c_rarg4 - oop ckval (super_klass) | ||||
| 2705 | // Win64 | ||||
| 2706 | // rsp+40 - oop ckval (super_klass) | ||||
| 2707 | // | ||||
| 2708 | // Output: | ||||
| 2709 | // rax == 0 - success | ||||
| 2710 | // rax == -1^K - failure, where K is partial transfer count | ||||
| 2711 | // | ||||
| 2712 | address generate_checkcast_copy(const char *name, address *entry, | ||||
| 2713 | bool dest_uninitialized = false) { | ||||
| 2714 | |||||
| 2715 | Label L_load_element, L_store_element, L_do_card_marks, L_done; | ||||
| 2716 | |||||
| 2717 | // Input registers (after setup_arg_regs) | ||||
| 2718 | const Register from = rdi; // source array address | ||||
| 2719 | const Register to = rsi; // destination array address | ||||
| 2720 | const Register length = rdx; // elements count | ||||
| 2721 | const Register ckoff = rcx; // super_check_offset | ||||
| 2722 | const Register ckval = r8; // super_klass | ||||
| 2723 | |||||
| 2724 | // Registers used as temps (r13, r14 are save-on-entry) | ||||
| 2725 | const Register end_from = from; // source array end address | ||||
| 2726 | const Register end_to = r13; // destination array end address | ||||
| 2727 | const Register count = rdx; // -(count_remaining) | ||||
| 2728 | const Register r14_length = r14; // saved copy of length | ||||
| 2729 | // End pointers are inclusive, and if length is not zero they point | ||||
| 2730 | // to the last unit copied: end_to[0] := end_from[0] | ||||
| 2731 | |||||
| 2732 | const Register rax_oop = rax; // actual oop copied | ||||
| 2733 | const Register r11_klass = r11; // oop._klass | ||||
| 2734 | |||||
| 2735 | //--------------------------------------------------------------- | ||||
| 2736 | // Assembler stub will be used for this call to arraycopy | ||||
| 2737 | // if the two arrays are subtypes of Object[] but the | ||||
| 2738 | // destination array type is not equal to or a supertype | ||||
| 2739 | // of the source type. Each element must be separately | ||||
| 2740 | // checked. | ||||
| 2741 | |||||
| 2742 | __masm-> align(CodeEntryAlignment); | ||||
| 2743 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2744 | address start = __masm-> pc(); | ||||
| 2745 | |||||
| 2746 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2747 | |||||
| 2748 | #ifdef ASSERT1 | ||||
| 2749 | // caller guarantees that the arrays really are different | ||||
| 2750 | // otherwise, we would have to make conjoint checks | ||||
| 2751 | { Label L; | ||||
| 2752 | array_overlap_test(L, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8)); | ||||
| 2753 | __masm-> stop("checkcast_copy within a single array"); | ||||
| 2754 | __masm-> bind(L); | ||||
| 2755 | } | ||||
| 2756 | #endif //ASSERT | ||||
| 2757 | |||||
| 2758 | setup_arg_regs(4); // from => rdi, to => rsi, length => rdx | ||||
| 2759 | // ckoff => rcx, ckval => r8 | ||||
| 2760 | // r9 and r10 may be used to save non-volatile registers | ||||
| 2761 | #ifdef _WIN64 | ||||
| 2762 | // last argument (#4) is on stack on Win64 | ||||
| 2763 | __masm-> movptr(ckval, Address(rsp, 6 * wordSize)); | ||||
| 2764 | #endif | ||||
| 2765 | |||||
| 2766 | // Caller of this entry point must set up the argument registers. | ||||
| 2767 | if (entry != NULL__null) { | ||||
| 2768 | *entry = __masm-> pc(); | ||||
| 2769 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 2770 | } | ||||
| 2771 | |||||
| 2772 | // allocate spill slots for r13, r14 | ||||
| 2773 | enum { | ||||
| 2774 | saved_r13_offset, | ||||
| 2775 | saved_r14_offset, | ||||
| 2776 | saved_r10_offset, | ||||
| 2777 | saved_rbp_offset | ||||
| 2778 | }; | ||||
| 2779 | __masm-> subptr(rsp, saved_rbp_offset * wordSize); | ||||
| 2780 | __masm-> movptr(Address(rsp, saved_r13_offset * wordSize), r13); | ||||
| 2781 | __masm-> movptr(Address(rsp, saved_r14_offset * wordSize), r14); | ||||
| 2782 | __masm-> movptr(Address(rsp, saved_r10_offset * wordSize), r10); | ||||
| 2783 | |||||
| 2784 | #ifdef ASSERT1 | ||||
| 2785 | Label L2; | ||||
| 2786 | __masm-> get_thread(r14); | ||||
| 2787 | __masm-> cmpptr(r15_thread, r14); | ||||
| 2788 | __masm-> jcc(Assembler::equal, L2); | ||||
| 2789 | __masm-> stop("StubRoutines::call_stub: r15_thread is modified by call"); | ||||
| 2790 | __masm-> bind(L2); | ||||
| 2791 | #endif // ASSERT | ||||
| 2792 | |||||
| 2793 | // check that int operands are properly extended to size_t | ||||
| 2794 | assert_clean_int(length, rax); | ||||
| 2795 | assert_clean_int(ckoff, rax); | ||||
| 2796 | |||||
| 2797 | #ifdef ASSERT1 | ||||
| 2798 | BLOCK_COMMENT("assert consistent ckoff/ckval")masm-> block_comment("assert consistent ckoff/ckval"); | ||||
| 2799 | // The ckoff and ckval must be mutually consistent, | ||||
| 2800 | // even though caller generates both. | ||||
| 2801 | { Label L; | ||||
| 2802 | int sco_offset = in_bytes(Klass::super_check_offset_offset()); | ||||
| 2803 | __masm-> cmpl(ckoff, Address(ckval, sco_offset)); | ||||
| 2804 | __masm-> jcc(Assembler::equal, L); | ||||
| 2805 | __masm-> stop("super_check_offset inconsistent"); | ||||
| 2806 | __masm-> bind(L); | ||||
| 2807 | } | ||||
| 2808 | #endif //ASSERT | ||||
| 2809 | |||||
| 2810 | // Loop-invariant addresses. They are exclusive end pointers. | ||||
| 2811 | Address end_from_addr(from, length, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0); | ||||
| 2812 | Address end_to_addr(to, length, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0); | ||||
| 2813 | // Loop-variant addresses. They assume post-incremented count < 0. | ||||
| 2814 | Address from_element_addr(end_from, count, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0); | ||||
| 2815 | Address to_element_addr(end_to, count, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), 0); | ||||
| 2816 | |||||
| 2817 | DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT; | ||||
| 2818 | if (dest_uninitialized) { | ||||
| 2819 | decorators |= IS_DEST_UNINITIALIZED; | ||||
| 2820 | } | ||||
| 2821 | |||||
| 2822 | BasicType type = T_OBJECT; | ||||
| 2823 | BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); | ||||
| 2824 | bs->arraycopy_prologue(_masm, decorators, type, from, to, count); | ||||
| 2825 | |||||
| 2826 | // Copy from low to high addresses, indexed from the end of each array. | ||||
| 2827 | __masm-> lea(end_from, end_from_addr); | ||||
| 2828 | __masm-> lea(end_to, end_to_addr); | ||||
| 2829 | __masm-> movptr(r14_length, length); // save a copy of the length | ||||
| 2830 | assert(length == count, "")do { if (!(length == count)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2830, "assert(" "length == count" ") failed", ""); ::breakpoint (); } } while (0); // else fix next line: | ||||
| 2831 | __masm-> negptr(count); // negate and test the length | ||||
| 2832 | __masm-> jcc(Assembler::notZero, L_load_element); | ||||
| 2833 | |||||
| 2834 | // Empty array: Nothing to do. | ||||
| 2835 | __masm-> xorptr(rax, rax); // return 0 on (trivial) success | ||||
| 2836 | __masm-> jmp(L_done); | ||||
| 2837 | |||||
| 2838 | // ======== begin loop ======== | ||||
| 2839 | // (Loop is rotated; its entry is L_load_element.) | ||||
| 2840 | // Loop control: | ||||
| 2841 | // for (count = -count; count != 0; count++) | ||||
| 2842 | // Base pointers src, dst are biased by 8*(count-1),to last element. | ||||
| 2843 | __masm-> align(OptoLoopAlignment); | ||||
| 2844 | |||||
| 2845 | __masm-> BIND(L_store_element)bind(L_store_element); masm-> block_comment("L_store_element" ":"); | ||||
| 2846 | __masm-> store_heap_oop(to_element_addr, rax_oop, noreg, noreg, AS_RAW); // store the oop | ||||
| 2847 | __masm-> increment(count); // increment the count toward zero | ||||
| 2848 | __masm-> jcc(Assembler::zero, L_do_card_marks); | ||||
| 2849 | |||||
| 2850 | // ======== loop entry is here ======== | ||||
| 2851 | __masm-> BIND(L_load_element)bind(L_load_element); masm-> block_comment("L_load_element" ":"); | ||||
| 2852 | __masm-> load_heap_oop(rax_oop, from_element_addr, noreg, noreg, AS_RAW); // load the oop | ||||
| 2853 | __masm-> testptr(rax_oop, rax_oop); | ||||
| 2854 | __masm-> jcc(Assembler::zero, L_store_element); | ||||
| 2855 | |||||
| 2856 | __masm-> load_klass(r11_klass, rax_oop, rscratch1);// query the object klass | ||||
| 2857 | generate_type_check(r11_klass, ckoff, ckval, L_store_element); | ||||
| 2858 | // ======== end loop ======== | ||||
| 2859 | |||||
| 2860 | // It was a real error; we must depend on the caller to finish the job. | ||||
| 2861 | // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. | ||||
| 2862 | // Emit GC store barriers for the oops we have copied (r14 + rdx), | ||||
| 2863 | // and report their number to the caller. | ||||
| 2864 | assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1); | ||||
| 2865 | Label L_post_barrier; | ||||
| 2866 | __masm-> addptr(r14_length, count); // K = (original - remaining) oops | ||||
| 2867 | __masm-> movptr(rax, r14_length); // save the value | ||||
| 2868 | __masm-> notptr(rax); // report (-1^K) to caller (does not affect flags) | ||||
| 2869 | __masm-> jccb(Assembler::notZero, L_post_barrier)jccb_0(Assembler::notZero, L_post_barrier, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2869); | ||||
| 2870 | __masm-> jmp(L_done); // K == 0, nothing was copied, skip post barrier | ||||
| 2871 | |||||
| 2872 | // Come here on success only. | ||||
| 2873 | __masm-> BIND(L_do_card_marks)bind(L_do_card_marks); masm-> block_comment("L_do_card_marks" ":"); | ||||
| 2874 | __masm-> xorptr(rax, rax); // return 0 on success | ||||
| 2875 | |||||
| 2876 | __masm-> BIND(L_post_barrier)bind(L_post_barrier); masm-> block_comment("L_post_barrier" ":"); | ||||
| 2877 | bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length); | ||||
| 2878 | |||||
| 2879 | // Common exit point (success or failure). | ||||
| 2880 | __masm-> BIND(L_done)bind(L_done); masm-> block_comment("L_done" ":"); | ||||
| 2881 | __masm-> movptr(r13, Address(rsp, saved_r13_offset * wordSize)); | ||||
| 2882 | __masm-> movptr(r14, Address(rsp, saved_r14_offset * wordSize)); | ||||
| 2883 | __masm-> movptr(r10, Address(rsp, saved_r10_offset * wordSize)); | ||||
| 2884 | restore_arg_regs(); | ||||
| 2885 | inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_checkcast_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_checkcast_array_copy_ctr);; // Update counter after rscratch1 is free | ||||
| 2886 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2887 | __masm-> ret(0); | ||||
| 2888 | |||||
| 2889 | return start; | ||||
| 2890 | } | ||||
| 2891 | |||||
| 2892 | // | ||||
| 2893 | // Generate 'unsafe' array copy stub | ||||
| 2894 | // Though just as safe as the other stubs, it takes an unscaled | ||||
| 2895 | // size_t argument instead of an element count. | ||||
| 2896 | // | ||||
| 2897 | // Input: | ||||
| 2898 | // c_rarg0 - source array address | ||||
| 2899 | // c_rarg1 - destination array address | ||||
| 2900 | // c_rarg2 - byte count, treated as ssize_t, can be zero | ||||
| 2901 | // | ||||
| 2902 | // Examines the alignment of the operands and dispatches | ||||
| 2903 | // to a long, int, short, or byte copy loop. | ||||
| 2904 | // | ||||
| 2905 | address generate_unsafe_copy(const char *name, | ||||
| 2906 | address byte_copy_entry, address short_copy_entry, | ||||
| 2907 | address int_copy_entry, address long_copy_entry) { | ||||
| 2908 | |||||
| 2909 | Label L_long_aligned, L_int_aligned, L_short_aligned; | ||||
| 2910 | |||||
| 2911 | // Input registers (before setup_arg_regs) | ||||
| 2912 | const Register from = c_rarg0; // source array address | ||||
| 2913 | const Register to = c_rarg1; // destination array address | ||||
| 2914 | const Register size = c_rarg2; // byte count (size_t) | ||||
| 2915 | |||||
| 2916 | // Register used as a temp | ||||
| 2917 | const Register bits = rax; // test copy of low bits | ||||
| 2918 | |||||
| 2919 | __masm-> align(CodeEntryAlignment); | ||||
| 2920 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 2921 | address start = __masm-> pc(); | ||||
| 2922 | |||||
| 2923 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 2924 | |||||
| 2925 | // bump this on entry, not on exit: | ||||
| 2926 | inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_unsafe_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_unsafe_array_copy_ctr);; | ||||
| 2927 | |||||
| 2928 | __masm-> mov(bits, from); | ||||
| 2929 | __masm-> orptr(bits, to); | ||||
| 2930 | __masm-> orptr(bits, size); | ||||
| 2931 | |||||
| 2932 | __masm-> testb(bits, BytesPerLong-1); | ||||
| 2933 | __masm-> jccb(Assembler::zero, L_long_aligned)jccb_0(Assembler::zero, L_long_aligned, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2933); | ||||
| 2934 | |||||
| 2935 | __masm-> testb(bits, BytesPerInt-1); | ||||
| 2936 | __masm-> jccb(Assembler::zero, L_int_aligned)jccb_0(Assembler::zero, L_int_aligned, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 2936); | ||||
| 2937 | |||||
| 2938 | __masm-> testb(bits, BytesPerShort-1); | ||||
| 2939 | __masm-> jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry)); | ||||
| 2940 | |||||
| 2941 | __masm-> BIND(L_short_aligned)bind(L_short_aligned); masm-> block_comment("L_short_aligned" ":"); | ||||
| 2942 | __masm-> shrptr(size, LogBytesPerShort); // size => short_count | ||||
| 2943 | __masm-> jump(RuntimeAddress(short_copy_entry)); | ||||
| 2944 | |||||
| 2945 | __masm-> BIND(L_int_aligned)bind(L_int_aligned); masm-> block_comment("L_int_aligned" ":" ); | ||||
| 2946 | __masm-> shrptr(size, LogBytesPerInt); // size => int_count | ||||
| 2947 | __masm-> jump(RuntimeAddress(int_copy_entry)); | ||||
| 2948 | |||||
| 2949 | __masm-> BIND(L_long_aligned)bind(L_long_aligned); masm-> block_comment("L_long_aligned" ":"); | ||||
| 2950 | __masm-> shrptr(size, LogBytesPerLong); // size => qword_count | ||||
| 2951 | __masm-> jump(RuntimeAddress(long_copy_entry)); | ||||
| 2952 | |||||
| 2953 | return start; | ||||
| 2954 | } | ||||
| 2955 | |||||
| 2956 | // Perform range checks on the proposed arraycopy. | ||||
| 2957 | // Kills temp, but nothing else. | ||||
| 2958 | // Also, clean the sign bits of src_pos and dst_pos. | ||||
| 2959 | void arraycopy_range_checks(Register src, // source array oop (c_rarg0) | ||||
| 2960 | Register src_pos, // source position (c_rarg1) | ||||
| 2961 | Register dst, // destination array oo (c_rarg2) | ||||
| 2962 | Register dst_pos, // destination position (c_rarg3) | ||||
| 2963 | Register length, | ||||
| 2964 | Register temp, | ||||
| 2965 | Label& L_failed) { | ||||
| 2966 | BLOCK_COMMENT("arraycopy_range_checks:")masm-> block_comment("arraycopy_range_checks:"); | ||||
| 2967 | |||||
| 2968 | // if (src_pos + length > arrayOop(src)->length()) FAIL; | ||||
| 2969 | __masm-> movl(temp, length); | ||||
| 2970 | __masm-> addl(temp, src_pos); // src_pos + length | ||||
| 2971 | __masm-> cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes())); | ||||
| 2972 | __masm-> jcc(Assembler::above, L_failed); | ||||
| 2973 | |||||
| 2974 | // if (dst_pos + length > arrayOop(dst)->length()) FAIL; | ||||
| 2975 | __masm-> movl(temp, length); | ||||
| 2976 | __masm-> addl(temp, dst_pos); // dst_pos + length | ||||
| 2977 | __masm-> cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes())); | ||||
| 2978 | __masm-> jcc(Assembler::above, L_failed); | ||||
| 2979 | |||||
| 2980 | // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'. | ||||
| 2981 | // Move with sign extension can be used since they are positive. | ||||
| 2982 | __masm-> movslq(src_pos, src_pos); | ||||
| 2983 | __masm-> movslq(dst_pos, dst_pos); | ||||
| 2984 | |||||
| 2985 | BLOCK_COMMENT("arraycopy_range_checks done")masm-> block_comment("arraycopy_range_checks done"); | ||||
| 2986 | } | ||||
| 2987 | |||||
| 2988 | // | ||||
| 2989 | // Generate generic array copy stubs | ||||
| 2990 | // | ||||
| 2991 | // Input: | ||||
| 2992 | // c_rarg0 - src oop | ||||
| 2993 | // c_rarg1 - src_pos (32-bits) | ||||
| 2994 | // c_rarg2 - dst oop | ||||
| 2995 | // c_rarg3 - dst_pos (32-bits) | ||||
| 2996 | // not Win64 | ||||
| 2997 | // c_rarg4 - element count (32-bits) | ||||
| 2998 | // Win64 | ||||
| 2999 | // rsp+40 - element count (32-bits) | ||||
| 3000 | // | ||||
| 3001 | // Output: | ||||
| 3002 | // rax == 0 - success | ||||
| 3003 | // rax == -1^K - failure, where K is partial transfer count | ||||
| 3004 | // | ||||
| 3005 | address generate_generic_copy(const char *name, | ||||
| 3006 | address byte_copy_entry, address short_copy_entry, | ||||
| 3007 | address int_copy_entry, address oop_copy_entry, | ||||
| 3008 | address long_copy_entry, address checkcast_copy_entry) { | ||||
| 3009 | |||||
| 3010 | Label L_failed, L_failed_0, L_objArray; | ||||
| 3011 | Label L_copy_shorts, L_copy_ints, L_copy_longs; | ||||
| 3012 | |||||
| 3013 | // Input registers | ||||
| 3014 | const Register src = c_rarg0; // source array oop | ||||
| 3015 | const Register src_pos = c_rarg1; // source position | ||||
| 3016 | const Register dst = c_rarg2; // destination array oop | ||||
| 3017 | const Register dst_pos = c_rarg3; // destination position | ||||
| 3018 | #ifndef _WIN64 | ||||
| 3019 | const Register length = c_rarg4; | ||||
| 3020 | const Register rklass_tmp = r9; // load_klass | ||||
| 3021 | #else | ||||
| 3022 | const Address length(rsp, 7 * wordSize); // elements count is on stack on Win64 | ||||
| 3023 | const Register rklass_tmp = rdi; // load_klass | ||||
| 3024 | #endif | ||||
| 3025 | |||||
| 3026 | { int modulus = CodeEntryAlignment; | ||||
| 3027 | int target = modulus - 5; // 5 = sizeof jmp(L_failed) | ||||
| 3028 | int advance = target - (__masm-> offset() % modulus); | ||||
| 3029 | if (advance < 0) advance += modulus; | ||||
| 3030 | if (advance > 0) __masm-> nop(advance); | ||||
| 3031 | } | ||||
| 3032 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 3033 | |||||
| 3034 | // Short-hop target to L_failed. Makes for denser prologue code. | ||||
| 3035 | __masm-> BIND(L_failed_0)bind(L_failed_0); masm-> block_comment("L_failed_0" ":"); | ||||
| 3036 | __masm-> jmp(L_failed); | ||||
| 3037 | assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed")do { if (!(masm-> offset() % CodeEntryAlignment == 0)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3037, "assert(" "masm-> offset() % CodeEntryAlignment == 0" ") failed", "no further alignment needed"); ::breakpoint(); } } while (0); | ||||
| 3038 | |||||
| 3039 | __masm-> align(CodeEntryAlignment); | ||||
| 3040 | address start = __masm-> pc(); | ||||
| 3041 | |||||
| 3042 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3043 | |||||
| 3044 | #ifdef _WIN64 | ||||
| 3045 | __masm-> push(rklass_tmp); // rdi is callee-save on Windows | ||||
| 3046 | #endif | ||||
| 3047 | |||||
| 3048 | // bump this on entry, not on exit: | ||||
| 3049 | inc_counter_np(SharedRuntime::_generic_array_copy_ctr)masm-> block_comment("inc_counter " "SharedRuntime::_generic_array_copy_ctr" ); inc_counter_np_(SharedRuntime::_generic_array_copy_ctr);; | ||||
| 3050 | |||||
| 3051 | //----------------------------------------------------------------------- | ||||
| 3052 | // Assembler stub will be used for this call to arraycopy | ||||
| 3053 | // if the following conditions are met: | ||||
| 3054 | // | ||||
| 3055 | // (1) src and dst must not be null. | ||||
| 3056 | // (2) src_pos must not be negative. | ||||
| 3057 | // (3) dst_pos must not be negative. | ||||
| 3058 | // (4) length must not be negative. | ||||
| 3059 | // (5) src klass and dst klass should be the same and not NULL. | ||||
| 3060 | // (6) src and dst should be arrays. | ||||
| 3061 | // (7) src_pos + length must not exceed length of src. | ||||
| 3062 | // (8) dst_pos + length must not exceed length of dst. | ||||
| 3063 | // | ||||
| 3064 | |||||
| 3065 | // if (src == NULL) return -1; | ||||
| 3066 | __masm-> testptr(src, src); // src oop | ||||
| 3067 | size_t j1off = __masm-> offset(); | ||||
| 3068 | __masm-> jccb(Assembler::zero, L_failed_0)jccb_0(Assembler::zero, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3068); | ||||
| 3069 | |||||
| 3070 | // if (src_pos < 0) return -1; | ||||
| 3071 | __masm-> testl(src_pos, src_pos); // src_pos (32-bits) | ||||
| 3072 | __masm-> jccb(Assembler::negative, L_failed_0)jccb_0(Assembler::negative, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3072); | ||||
| 3073 | |||||
| 3074 | // if (dst == NULL) return -1; | ||||
| 3075 | __masm-> testptr(dst, dst); // dst oop | ||||
| 3076 | __masm-> jccb(Assembler::zero, L_failed_0)jccb_0(Assembler::zero, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3076); | ||||
| 3077 | |||||
| 3078 | // if (dst_pos < 0) return -1; | ||||
| 3079 | __masm-> testl(dst_pos, dst_pos); // dst_pos (32-bits) | ||||
| 3080 | size_t j4off = __masm-> offset(); | ||||
| 3081 | __masm-> jccb(Assembler::negative, L_failed_0)jccb_0(Assembler::negative, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3081); | ||||
| 3082 | |||||
| 3083 | // The first four tests are very dense code, | ||||
| 3084 | // but not quite dense enough to put four | ||||
| 3085 | // jumps in a 16-byte instruction fetch buffer. | ||||
| 3086 | // That's good, because some branch predicters | ||||
| 3087 | // do not like jumps so close together. | ||||
| 3088 | // Make sure of this. | ||||
| 3089 | guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps")do { if (!(((j1off ^ j4off) & ~15) != 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3089, "guarantee(" "((j1off ^ j4off) & ~15) != 0" ") failed" , "I$ line of 1st & 4th jumps"); ::breakpoint(); } } while (0); | ||||
| 3090 | |||||
| 3091 | // registers used as temp | ||||
| 3092 | const Register r11_length = r11; // elements count to copy | ||||
| 3093 | const Register r10_src_klass = r10; // array klass | ||||
| 3094 | |||||
| 3095 | // if (length < 0) return -1; | ||||
| 3096 | __masm-> movl(r11_length, length); // length (elements count, 32-bits value) | ||||
| 3097 | __masm-> testl(r11_length, r11_length); | ||||
| 3098 | __masm-> jccb(Assembler::negative, L_failed_0)jccb_0(Assembler::negative, L_failed_0, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3098); | ||||
| 3099 | |||||
| 3100 | __masm-> load_klass(r10_src_klass, src, rklass_tmp); | ||||
| 3101 | #ifdef ASSERT1 | ||||
| 3102 | // assert(src->klass() != NULL); | ||||
| 3103 | { | ||||
| 3104 | BLOCK_COMMENT("assert klasses not null {")masm-> block_comment("assert klasses not null {"); | ||||
| 3105 | Label L1, L2; | ||||
| 3106 | __masm-> testptr(r10_src_klass, r10_src_klass); | ||||
| 3107 | __masm-> jcc(Assembler::notZero, L2); // it is broken if klass is NULL | ||||
| 3108 | __masm-> bind(L1); | ||||
| 3109 | __masm-> stop("broken null klass"); | ||||
| 3110 | __masm-> bind(L2); | ||||
| 3111 | __masm-> load_klass(rax, dst, rklass_tmp); | ||||
| 3112 | __masm-> cmpq(rax, 0); | ||||
| 3113 | __masm-> jcc(Assembler::equal, L1); // this would be broken also | ||||
| 3114 | BLOCK_COMMENT("} assert klasses not null done")masm-> block_comment("} assert klasses not null done"); | ||||
| 3115 | } | ||||
| 3116 | #endif | ||||
| 3117 | |||||
| 3118 | // Load layout helper (32-bits) | ||||
| 3119 | // | ||||
| 3120 | // |array_tag| | header_size | element_type | |log2_element_size| | ||||
| 3121 | // 32 30 24 16 8 2 0 | ||||
| 3122 | // | ||||
| 3123 | // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 | ||||
| 3124 | // | ||||
| 3125 | |||||
| 3126 | const int lh_offset = in_bytes(Klass::layout_helper_offset()); | ||||
| 3127 | |||||
| 3128 | // Handle objArrays completely differently... | ||||
| 3129 | const jint objArray_lh = Klass::array_layout_helper(T_OBJECT); | ||||
| 3130 | __masm-> cmpl(Address(r10_src_klass, lh_offset), objArray_lh); | ||||
| 3131 | __masm-> jcc(Assembler::equal, L_objArray); | ||||
| 3132 | |||||
| 3133 | // if (src->klass() != dst->klass()) return -1; | ||||
| 3134 | __masm-> load_klass(rax, dst, rklass_tmp); | ||||
| 3135 | __masm-> cmpq(r10_src_klass, rax); | ||||
| 3136 | __masm-> jcc(Assembler::notEqual, L_failed); | ||||
| 3137 | |||||
| 3138 | const Register rax_lh = rax; // layout helper | ||||
| 3139 | __masm-> movl(rax_lh, Address(r10_src_klass, lh_offset)); | ||||
| 3140 | |||||
| 3141 | // if (!src->is_Array()) return -1; | ||||
| 3142 | __masm-> cmpl(rax_lh, Klass::_lh_neutral_value); | ||||
| 3143 | __masm-> jcc(Assembler::greaterEqual, L_failed); | ||||
| 3144 | |||||
| 3145 | // At this point, it is known to be a typeArray (array_tag 0x3). | ||||
| 3146 | #ifdef ASSERT1 | ||||
| 3147 | { | ||||
| 3148 | BLOCK_COMMENT("assert primitive array {")masm-> block_comment("assert primitive array {"); | ||||
| 3149 | Label L; | ||||
| 3150 | __masm-> cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); | ||||
| 3151 | __masm-> jcc(Assembler::greaterEqual, L); | ||||
| 3152 | __masm-> stop("must be a primitive array"); | ||||
| 3153 | __masm-> bind(L); | ||||
| 3154 | BLOCK_COMMENT("} assert primitive array done")masm-> block_comment("} assert primitive array done"); | ||||
| 3155 | } | ||||
| 3156 | #endif | ||||
| 3157 | |||||
| 3158 | arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, | ||||
| 3159 | r10, L_failed); | ||||
| 3160 | |||||
| 3161 | // TypeArrayKlass | ||||
| 3162 | // | ||||
| 3163 | // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize); | ||||
| 3164 | // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize); | ||||
| 3165 | // | ||||
| 3166 | |||||
| 3167 | const Register r10_offset = r10; // array offset | ||||
| 3168 | const Register rax_elsize = rax_lh; // element size | ||||
| 3169 | |||||
| 3170 | __masm-> movl(r10_offset, rax_lh); | ||||
| 3171 | __masm-> shrl(r10_offset, Klass::_lh_header_size_shift); | ||||
| 3172 | __masm-> andptr(r10_offset, Klass::_lh_header_size_mask); // array_offset | ||||
| 3173 | __masm-> addptr(src, r10_offset); // src array offset | ||||
| 3174 | __masm-> addptr(dst, r10_offset); // dst array offset | ||||
| 3175 | BLOCK_COMMENT("choose copy loop based on element size")masm-> block_comment("choose copy loop based on element size" ); | ||||
| 3176 | __masm-> andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize | ||||
| 3177 | |||||
| 3178 | #ifdef _WIN64 | ||||
| 3179 | __masm-> pop(rklass_tmp); // Restore callee-save rdi | ||||
| 3180 | #endif | ||||
| 3181 | |||||
| 3182 | // next registers should be set before the jump to corresponding stub | ||||
| 3183 | const Register from = c_rarg0; // source array address | ||||
| 3184 | const Register to = c_rarg1; // destination array address | ||||
| 3185 | const Register count = c_rarg2; // elements count | ||||
| 3186 | |||||
| 3187 | // 'from', 'to', 'count' registers should be set in such order | ||||
| 3188 | // since they are the same as 'src', 'src_pos', 'dst'. | ||||
| 3189 | |||||
| 3190 | __masm-> cmpl(rax_elsize, 0); | ||||
| 3191 | __masm-> jccb(Assembler::notEqual, L_copy_shorts)jccb_0(Assembler::notEqual, L_copy_shorts, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3191); | ||||
| 3192 | __masm-> lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr | ||||
| 3193 | __masm-> lea(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr | ||||
| 3194 | __masm-> movl2ptr(count, r11_length); // length | ||||
| 3195 | __masm-> jump(RuntimeAddress(byte_copy_entry)); | ||||
| 3196 | |||||
| 3197 | __masm-> BIND(L_copy_shorts)bind(L_copy_shorts); masm-> block_comment("L_copy_shorts" ":" ); | ||||
| 3198 | __masm-> cmpl(rax_elsize, LogBytesPerShort); | ||||
| 3199 | __masm-> jccb(Assembler::notEqual, L_copy_ints)jccb_0(Assembler::notEqual, L_copy_ints, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3199); | ||||
| 3200 | __masm-> lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr | ||||
| 3201 | __masm-> lea(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr | ||||
| 3202 | __masm-> movl2ptr(count, r11_length); // length | ||||
| 3203 | __masm-> jump(RuntimeAddress(short_copy_entry)); | ||||
| 3204 | |||||
| 3205 | __masm-> BIND(L_copy_ints)bind(L_copy_ints); masm-> block_comment("L_copy_ints" ":"); | ||||
| 3206 | __masm-> cmpl(rax_elsize, LogBytesPerInt); | ||||
| 3207 | __masm-> jccb(Assembler::notEqual, L_copy_longs)jccb_0(Assembler::notEqual, L_copy_longs, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3207); | ||||
| 3208 | __masm-> lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr | ||||
| 3209 | __masm-> lea(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr | ||||
| 3210 | __masm-> movl2ptr(count, r11_length); // length | ||||
| 3211 | __masm-> jump(RuntimeAddress(int_copy_entry)); | ||||
| 3212 | |||||
| 3213 | __masm-> BIND(L_copy_longs)bind(L_copy_longs); masm-> block_comment("L_copy_longs" ":" ); | ||||
| 3214 | #ifdef ASSERT1 | ||||
| 3215 | { | ||||
| 3216 | BLOCK_COMMENT("assert long copy {")masm-> block_comment("assert long copy {"); | ||||
| 3217 | Label L; | ||||
| 3218 | __masm-> cmpl(rax_elsize, LogBytesPerLong); | ||||
| 3219 | __masm-> jcc(Assembler::equal, L); | ||||
| 3220 | __masm-> stop("must be long copy, but elsize is wrong"); | ||||
| 3221 | __masm-> bind(L); | ||||
| 3222 | BLOCK_COMMENT("} assert long copy done")masm-> block_comment("} assert long copy done"); | ||||
| 3223 | } | ||||
| 3224 | #endif | ||||
| 3225 | __masm-> lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr | ||||
| 3226 | __masm-> lea(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr | ||||
| 3227 | __masm-> movl2ptr(count, r11_length); // length | ||||
| 3228 | __masm-> jump(RuntimeAddress(long_copy_entry)); | ||||
| 3229 | |||||
| 3230 | // ObjArrayKlass | ||||
| 3231 | __masm-> BIND(L_objArray)bind(L_objArray); masm-> block_comment("L_objArray" ":"); | ||||
| 3232 | // live at this point: r10_src_klass, r11_length, src[_pos], dst[_pos] | ||||
| 3233 | |||||
| 3234 | Label L_plain_copy, L_checkcast_copy; | ||||
| 3235 | // test array classes for subtyping | ||||
| 3236 | __masm-> load_klass(rax, dst, rklass_tmp); | ||||
| 3237 | __masm-> cmpq(r10_src_klass, rax); // usual case is exact equality | ||||
| 3238 | __masm-> jcc(Assembler::notEqual, L_checkcast_copy); | ||||
| 3239 | |||||
| 3240 | // Identically typed arrays can be copied without element-wise checks. | ||||
| 3241 | arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, | ||||
| 3242 | r10, L_failed); | ||||
| 3243 | |||||
| 3244 | __masm-> lea(from, Address(src, src_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), | ||||
| 3245 | arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr | ||||
| 3246 | __masm-> lea(to, Address(dst, dst_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), | ||||
| 3247 | arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr | ||||
| 3248 | __masm-> movl2ptr(count, r11_length); // length | ||||
| 3249 | __masm-> BIND(L_plain_copy)bind(L_plain_copy); masm-> block_comment("L_plain_copy" ":" ); | ||||
| 3250 | #ifdef _WIN64 | ||||
| 3251 | __masm-> pop(rklass_tmp); // Restore callee-save rdi | ||||
| 3252 | #endif | ||||
| 3253 | __masm-> jump(RuntimeAddress(oop_copy_entry)); | ||||
| 3254 | |||||
| 3255 | __masm-> BIND(L_checkcast_copy)bind(L_checkcast_copy); masm-> block_comment("L_checkcast_copy" ":"); | ||||
| 3256 | // live at this point: r10_src_klass, r11_length, rax (dst_klass) | ||||
| 3257 | { | ||||
| 3258 | // Before looking at dst.length, make sure dst is also an objArray. | ||||
| 3259 | __masm-> cmpl(Address(rax, lh_offset), objArray_lh); | ||||
| 3260 | __masm-> jcc(Assembler::notEqual, L_failed); | ||||
| 3261 | |||||
| 3262 | // It is safe to examine both src.length and dst.length. | ||||
| 3263 | arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, | ||||
| 3264 | rax, L_failed); | ||||
| 3265 | |||||
| 3266 | const Register r11_dst_klass = r11; | ||||
| 3267 | __masm-> load_klass(r11_dst_klass, dst, rklass_tmp); // reload | ||||
| 3268 | |||||
| 3269 | // Marshal the base address arguments now, freeing registers. | ||||
| 3270 | __masm-> lea(from, Address(src, src_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), | ||||
| 3271 | arrayOopDesc::base_offset_in_bytes(T_OBJECT))); | ||||
| 3272 | __masm-> lea(to, Address(dst, dst_pos, TIMES_OOP(UseCompressedOops ? Address::times_4 : Address::times_8), | ||||
| 3273 | arrayOopDesc::base_offset_in_bytes(T_OBJECT))); | ||||
| 3274 | __masm-> movl(count, length); // length (reloaded) | ||||
| 3275 | Register sco_temp = c_rarg3; // this register is free now | ||||
| 3276 | assert_different_registers(from, to, count, sco_temp, | ||||
| 3277 | r11_dst_klass, r10_src_klass); | ||||
| 3278 | assert_clean_int(count, sco_temp); | ||||
| 3279 | |||||
| 3280 | // Generate the type check. | ||||
| 3281 | const int sco_offset = in_bytes(Klass::super_check_offset_offset()); | ||||
| 3282 | __masm-> movl(sco_temp, Address(r11_dst_klass, sco_offset)); | ||||
| 3283 | assert_clean_int(sco_temp, rax); | ||||
| 3284 | generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy); | ||||
| 3285 | |||||
| 3286 | // Fetch destination element klass from the ObjArrayKlass header. | ||||
| 3287 | int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); | ||||
| 3288 | __masm-> movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset)); | ||||
| 3289 | __masm-> movl( sco_temp, Address(r11_dst_klass, sco_offset)); | ||||
| 3290 | assert_clean_int(sco_temp, rax); | ||||
| 3291 | |||||
| 3292 | #ifdef _WIN64 | ||||
| 3293 | __masm-> pop(rklass_tmp); // Restore callee-save rdi | ||||
| 3294 | #endif | ||||
| 3295 | |||||
| 3296 | // the checkcast_copy loop needs two extra arguments: | ||||
| 3297 | assert(c_rarg3 == sco_temp, "#3 already in place")do { if (!(c_rarg3 == sco_temp)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3297, "assert(" "c_rarg3 == sco_temp" ") failed", "#3 already in place" ); ::breakpoint(); } } while (0); | ||||
| 3298 | // Set up arguments for checkcast_copy_entry. | ||||
| 3299 | setup_arg_regs(4); | ||||
| 3300 | __masm-> movptr(r8, r11_dst_klass); // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris | ||||
| 3301 | __masm-> jump(RuntimeAddress(checkcast_copy_entry)); | ||||
| 3302 | } | ||||
| 3303 | |||||
| 3304 | __masm-> BIND(L_failed)bind(L_failed); masm-> block_comment("L_failed" ":"); | ||||
| 3305 | #ifdef _WIN64 | ||||
| 3306 | __masm-> pop(rklass_tmp); // Restore callee-save rdi | ||||
| 3307 | #endif | ||||
| 3308 | __masm-> xorptr(rax, rax); | ||||
| 3309 | __masm-> notptr(rax); // return -1 | ||||
| 3310 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3311 | __masm-> ret(0); | ||||
| 3312 | |||||
| 3313 | return start; | ||||
| 3314 | } | ||||
| 3315 | |||||
| 3316 | address generate_data_cache_writeback() { | ||||
| 3317 | const Register src = c_rarg0; // source address | ||||
| 3318 | |||||
| 3319 | __masm-> align(CodeEntryAlignment); | ||||
| 3320 | |||||
| 3321 | StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback"); | ||||
| 3322 | |||||
| 3323 | address start = __masm-> pc(); | ||||
| 3324 | __masm-> enter(); | ||||
| 3325 | __masm-> cache_wb(Address(src, 0)); | ||||
| 3326 | __masm-> leave(); | ||||
| 3327 | __masm-> ret(0); | ||||
| 3328 | |||||
| 3329 | return start; | ||||
| 3330 | } | ||||
| 3331 | |||||
| 3332 | address generate_data_cache_writeback_sync() { | ||||
| 3333 | const Register is_pre = c_rarg0; // pre or post sync | ||||
| 3334 | |||||
| 3335 | __masm-> align(CodeEntryAlignment); | ||||
| 3336 | |||||
| 3337 | StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync"); | ||||
| 3338 | |||||
| 3339 | // pre wbsync is a no-op | ||||
| 3340 | // post wbsync translates to an sfence | ||||
| 3341 | |||||
| 3342 | Label skip; | ||||
| 3343 | address start = __masm-> pc(); | ||||
| 3344 | __masm-> enter(); | ||||
| 3345 | __masm-> cmpl(is_pre, 0); | ||||
| 3346 | __masm-> jcc(Assembler::notEqual, skip); | ||||
| 3347 | __masm-> cache_wbsync(false); | ||||
| 3348 | __masm-> bind(skip); | ||||
| 3349 | __masm-> leave(); | ||||
| 3350 | __masm-> ret(0); | ||||
| 3351 | |||||
| 3352 | return start; | ||||
| 3353 | } | ||||
| 3354 | |||||
| 3355 | void generate_arraycopy_stubs() { | ||||
| 3356 | address entry; | ||||
| 3357 | address entry_jbyte_arraycopy; | ||||
| 3358 | address entry_jshort_arraycopy; | ||||
| 3359 | address entry_jint_arraycopy; | ||||
| 3360 | address entry_oop_arraycopy; | ||||
| 3361 | address entry_jlong_arraycopy; | ||||
| 3362 | address entry_checkcast_arraycopy; | ||||
| 3363 | |||||
| 3364 | StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry, | ||||
| 3365 | "jbyte_disjoint_arraycopy"); | ||||
| 3366 | StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy, | ||||
| 3367 | "jbyte_arraycopy"); | ||||
| 3368 | |||||
| 3369 | StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry, | ||||
| 3370 | "jshort_disjoint_arraycopy"); | ||||
| 3371 | StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy, | ||||
| 3372 | "jshort_arraycopy"); | ||||
| 3373 | |||||
| 3374 | StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, &entry, | ||||
| 3375 | "jint_disjoint_arraycopy"); | ||||
| 3376 | StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, entry, | ||||
| 3377 | &entry_jint_arraycopy, "jint_arraycopy"); | ||||
| 3378 | |||||
| 3379 | StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, &entry, | ||||
| 3380 | "jlong_disjoint_arraycopy"); | ||||
| 3381 | StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, entry, | ||||
| 3382 | &entry_jlong_arraycopy, "jlong_arraycopy"); | ||||
| 3383 | |||||
| 3384 | |||||
| 3385 | if (UseCompressedOops) { | ||||
| 3386 | StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, &entry, | ||||
| 3387 | "oop_disjoint_arraycopy"); | ||||
| 3388 | StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, entry, | ||||
| 3389 | &entry_oop_arraycopy, "oop_arraycopy"); | ||||
| 3390 | StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(false, true, &entry, | ||||
| 3391 | "oop_disjoint_arraycopy_uninit", | ||||
| 3392 | /*dest_uninitialized*/true); | ||||
| 3393 | StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(false, true, entry, | ||||
| 3394 | NULL__null, "oop_arraycopy_uninit", | ||||
| 3395 | /*dest_uninitialized*/true); | ||||
| 3396 | } else { | ||||
| 3397 | StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, &entry, | ||||
| 3398 | "oop_disjoint_arraycopy"); | ||||
| 3399 | StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, entry, | ||||
| 3400 | &entry_oop_arraycopy, "oop_arraycopy"); | ||||
| 3401 | StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(false, true, &entry, | ||||
| 3402 | "oop_disjoint_arraycopy_uninit", | ||||
| 3403 | /*dest_uninitialized*/true); | ||||
| 3404 | StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(false, true, entry, | ||||
| 3405 | NULL__null, "oop_arraycopy_uninit", | ||||
| 3406 | /*dest_uninitialized*/true); | ||||
| 3407 | } | ||||
| 3408 | |||||
| 3409 | StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); | ||||
| 3410 | StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL__null, | ||||
| 3411 | /*dest_uninitialized*/true); | ||||
| 3412 | |||||
| 3413 | StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", | ||||
| 3414 | entry_jbyte_arraycopy, | ||||
| 3415 | entry_jshort_arraycopy, | ||||
| 3416 | entry_jint_arraycopy, | ||||
| 3417 | entry_jlong_arraycopy); | ||||
| 3418 | StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy", | ||||
| 3419 | entry_jbyte_arraycopy, | ||||
| 3420 | entry_jshort_arraycopy, | ||||
| 3421 | entry_jint_arraycopy, | ||||
| 3422 | entry_oop_arraycopy, | ||||
| 3423 | entry_jlong_arraycopy, | ||||
| 3424 | entry_checkcast_arraycopy); | ||||
| 3425 | |||||
| 3426 | StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill"); | ||||
| 3427 | StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill"); | ||||
| 3428 | StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill"); | ||||
| 3429 | StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill"); | ||||
| 3430 | StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill"); | ||||
| 3431 | StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill"); | ||||
| 3432 | |||||
| 3433 | // We don't generate specialized code for HeapWord-aligned source | ||||
| 3434 | // arrays, so just use the code we've already generated | ||||
| 3435 | StubRoutines::_arrayof_jbyte_disjoint_arraycopy = StubRoutines::_jbyte_disjoint_arraycopy; | ||||
| 3436 | StubRoutines::_arrayof_jbyte_arraycopy = StubRoutines::_jbyte_arraycopy; | ||||
| 3437 | |||||
| 3438 | StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy; | ||||
| 3439 | StubRoutines::_arrayof_jshort_arraycopy = StubRoutines::_jshort_arraycopy; | ||||
| 3440 | |||||
| 3441 | StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy; | ||||
| 3442 | StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; | ||||
| 3443 | |||||
| 3444 | StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy; | ||||
| 3445 | StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; | ||||
| 3446 | |||||
| 3447 | StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy; | ||||
| 3448 | StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; | ||||
| 3449 | |||||
| 3450 | StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit; | ||||
| 3451 | StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit; | ||||
| 3452 | } | ||||
| 3453 | |||||
| 3454 | // AES intrinsic stubs | ||||
| 3455 | enum {AESBlockSize = 16}; | ||||
| 3456 | |||||
| 3457 | address generate_key_shuffle_mask() { | ||||
| 3458 | __masm-> align(16); | ||||
| 3459 | StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask"); | ||||
| 3460 | address start = __masm-> pc(); | ||||
| 3461 | __masm-> emit_data64( 0x0405060700010203, relocInfo::none ); | ||||
| 3462 | __masm-> emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none ); | ||||
| 3463 | return start; | ||||
| 3464 | } | ||||
| 3465 | |||||
| 3466 | address generate_counter_shuffle_mask() { | ||||
| 3467 | __masm-> align(16); | ||||
| 3468 | StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask"); | ||||
| 3469 | address start = __masm-> pc(); | ||||
| 3470 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); | ||||
| 3471 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); | ||||
| 3472 | return start; | ||||
| 3473 | } | ||||
| 3474 | |||||
| 3475 | // Utility routine for loading a 128-bit key word in little endian format | ||||
| 3476 | // can optionally specify that the shuffle mask is already in an xmmregister | ||||
| 3477 | void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL__null) { | ||||
| 3478 | __masm-> movdqu(xmmdst, Address(key, offset)); | ||||
| 3479 | if (xmm_shuf_mask != NULL__null) { | ||||
| 3480 | __masm-> pshufb(xmmdst, xmm_shuf_mask); | ||||
| 3481 | } else { | ||||
| 3482 | __masm-> pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 3483 | } | ||||
| 3484 | } | ||||
| 3485 | |||||
| 3486 | // Utility routine for increase 128bit counter (iv in CTR mode) | ||||
| 3487 | void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) { | ||||
| 3488 | __masm-> pextrq(reg, xmmdst, 0x0); | ||||
| 3489 | __masm-> addq(reg, inc_delta); | ||||
| 3490 | __masm-> pinsrq(xmmdst, reg, 0x0); | ||||
| 3491 | __masm-> jcc(Assembler::carryClear, next_block); // jump if no carry | ||||
| 3492 | __masm-> pextrq(reg, xmmdst, 0x01); // Carry | ||||
| 3493 | __masm-> addq(reg, 0x01); | ||||
| 3494 | __masm-> pinsrq(xmmdst, reg, 0x01); //Carry end | ||||
| 3495 | __masm-> BIND(next_block)bind(next_block); masm-> block_comment("next_block" ":"); // next instruction | ||||
| 3496 | } | ||||
| 3497 | |||||
| 3498 | // Arguments: | ||||
| 3499 | // | ||||
| 3500 | // Inputs: | ||||
| 3501 | // c_rarg0 - source byte array address | ||||
| 3502 | // c_rarg1 - destination byte array address | ||||
| 3503 | // c_rarg2 - K (key) in little endian int array | ||||
| 3504 | // | ||||
| 3505 | address generate_aescrypt_encryptBlock() { | ||||
| 3506 | assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3506, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support" ); ::breakpoint(); } } while (0); | ||||
| 3507 | __masm-> align(CodeEntryAlignment); | ||||
| 3508 | StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock"); | ||||
| 3509 | Label L_doLast; | ||||
| 3510 | address start = __masm-> pc(); | ||||
| 3511 | |||||
| 3512 | const Register from = c_rarg0; // source array address | ||||
| 3513 | const Register to = c_rarg1; // destination array address | ||||
| 3514 | const Register key = c_rarg2; // key array address | ||||
| 3515 | const Register keylen = rax; | ||||
| 3516 | |||||
| 3517 | const XMMRegister xmm_result = xmm0; | ||||
| 3518 | const XMMRegister xmm_key_shuf_mask = xmm1; | ||||
| 3519 | // On win64 xmm6-xmm15 must be preserved so don't use them. | ||||
| 3520 | const XMMRegister xmm_temp1 = xmm2; | ||||
| 3521 | const XMMRegister xmm_temp2 = xmm3; | ||||
| 3522 | const XMMRegister xmm_temp3 = xmm4; | ||||
| 3523 | const XMMRegister xmm_temp4 = xmm5; | ||||
| 3524 | |||||
| 3525 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3526 | |||||
| 3527 | // keylen could be only {11, 13, 15} * 4 = {44, 52, 60} | ||||
| 3528 | __masm-> movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); | ||||
| 3529 | |||||
| 3530 | __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 3531 | __masm-> movdqu(xmm_result, Address(from, 0)); // get 16 bytes of input | ||||
| 3532 | |||||
| 3533 | // For encryption, the java expanded key ordering is just what we need | ||||
| 3534 | // we don't know if the key is aligned, hence not using load-execute form | ||||
| 3535 | |||||
| 3536 | load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask); | ||||
| 3537 | __masm-> pxor(xmm_result, xmm_temp1); | ||||
| 3538 | |||||
| 3539 | load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask); | ||||
| 3540 | load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask); | ||||
| 3541 | load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask); | ||||
| 3542 | load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask); | ||||
| 3543 | |||||
| 3544 | __masm-> aesenc(xmm_result, xmm_temp1); | ||||
| 3545 | __masm-> aesenc(xmm_result, xmm_temp2); | ||||
| 3546 | __masm-> aesenc(xmm_result, xmm_temp3); | ||||
| 3547 | __masm-> aesenc(xmm_result, xmm_temp4); | ||||
| 3548 | |||||
| 3549 | load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask); | ||||
| 3550 | load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask); | ||||
| 3551 | load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask); | ||||
| 3552 | load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask); | ||||
| 3553 | |||||
| 3554 | __masm-> aesenc(xmm_result, xmm_temp1); | ||||
| 3555 | __masm-> aesenc(xmm_result, xmm_temp2); | ||||
| 3556 | __masm-> aesenc(xmm_result, xmm_temp3); | ||||
| 3557 | __masm-> aesenc(xmm_result, xmm_temp4); | ||||
| 3558 | |||||
| 3559 | load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask); | ||||
| 3560 | load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask); | ||||
| 3561 | |||||
| 3562 | __masm-> cmpl(keylen, 44); | ||||
| 3563 | __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3563); | ||||
| 3564 | |||||
| 3565 | __masm-> aesenc(xmm_result, xmm_temp1); | ||||
| 3566 | __masm-> aesenc(xmm_result, xmm_temp2); | ||||
| 3567 | |||||
| 3568 | load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask); | ||||
| 3569 | load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask); | ||||
| 3570 | |||||
| 3571 | __masm-> cmpl(keylen, 52); | ||||
| 3572 | __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3572); | ||||
| 3573 | |||||
| 3574 | __masm-> aesenc(xmm_result, xmm_temp1); | ||||
| 3575 | __masm-> aesenc(xmm_result, xmm_temp2); | ||||
| 3576 | |||||
| 3577 | load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask); | ||||
| 3578 | load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask); | ||||
| 3579 | |||||
| 3580 | __masm-> BIND(L_doLast)bind(L_doLast); masm-> block_comment("L_doLast" ":"); | ||||
| 3581 | __masm-> aesenc(xmm_result, xmm_temp1); | ||||
| 3582 | __masm-> aesenclast(xmm_result, xmm_temp2); | ||||
| 3583 | __masm-> movdqu(Address(to, 0), xmm_result); // store the result | ||||
| 3584 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 3585 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3586 | __masm-> ret(0); | ||||
| 3587 | |||||
| 3588 | return start; | ||||
| 3589 | } | ||||
| 3590 | |||||
| 3591 | |||||
| 3592 | // Arguments: | ||||
| 3593 | // | ||||
| 3594 | // Inputs: | ||||
| 3595 | // c_rarg0 - source byte array address | ||||
| 3596 | // c_rarg1 - destination byte array address | ||||
| 3597 | // c_rarg2 - K (key) in little endian int array | ||||
| 3598 | // | ||||
| 3599 | address generate_aescrypt_decryptBlock() { | ||||
| 3600 | assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3600, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support" ); ::breakpoint(); } } while (0); | ||||
| 3601 | __masm-> align(CodeEntryAlignment); | ||||
| 3602 | StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock"); | ||||
| 3603 | Label L_doLast; | ||||
| 3604 | address start = __masm-> pc(); | ||||
| 3605 | |||||
| 3606 | const Register from = c_rarg0; // source array address | ||||
| 3607 | const Register to = c_rarg1; // destination array address | ||||
| 3608 | const Register key = c_rarg2; // key array address | ||||
| 3609 | const Register keylen = rax; | ||||
| 3610 | |||||
| 3611 | const XMMRegister xmm_result = xmm0; | ||||
| 3612 | const XMMRegister xmm_key_shuf_mask = xmm1; | ||||
| 3613 | // On win64 xmm6-xmm15 must be preserved so don't use them. | ||||
| 3614 | const XMMRegister xmm_temp1 = xmm2; | ||||
| 3615 | const XMMRegister xmm_temp2 = xmm3; | ||||
| 3616 | const XMMRegister xmm_temp3 = xmm4; | ||||
| 3617 | const XMMRegister xmm_temp4 = xmm5; | ||||
| 3618 | |||||
| 3619 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3620 | |||||
| 3621 | // keylen could be only {11, 13, 15} * 4 = {44, 52, 60} | ||||
| 3622 | __masm-> movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); | ||||
| 3623 | |||||
| 3624 | __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 3625 | __masm-> movdqu(xmm_result, Address(from, 0)); | ||||
| 3626 | |||||
| 3627 | // for decryption java expanded key ordering is rotated one position from what we want | ||||
| 3628 | // so we start from 0x10 here and hit 0x00 last | ||||
| 3629 | // we don't know if the key is aligned, hence not using load-execute form | ||||
| 3630 | load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask); | ||||
| 3631 | load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask); | ||||
| 3632 | load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask); | ||||
| 3633 | load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask); | ||||
| 3634 | |||||
| 3635 | __masm-> pxor (xmm_result, xmm_temp1); | ||||
| 3636 | __masm-> aesdec(xmm_result, xmm_temp2); | ||||
| 3637 | __masm-> aesdec(xmm_result, xmm_temp3); | ||||
| 3638 | __masm-> aesdec(xmm_result, xmm_temp4); | ||||
| 3639 | |||||
| 3640 | load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask); | ||||
| 3641 | load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask); | ||||
| 3642 | load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask); | ||||
| 3643 | load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask); | ||||
| 3644 | |||||
| 3645 | __masm-> aesdec(xmm_result, xmm_temp1); | ||||
| 3646 | __masm-> aesdec(xmm_result, xmm_temp2); | ||||
| 3647 | __masm-> aesdec(xmm_result, xmm_temp3); | ||||
| 3648 | __masm-> aesdec(xmm_result, xmm_temp4); | ||||
| 3649 | |||||
| 3650 | load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask); | ||||
| 3651 | load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask); | ||||
| 3652 | load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask); | ||||
| 3653 | |||||
| 3654 | __masm-> cmpl(keylen, 44); | ||||
| 3655 | __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3655); | ||||
| 3656 | |||||
| 3657 | __masm-> aesdec(xmm_result, xmm_temp1); | ||||
| 3658 | __masm-> aesdec(xmm_result, xmm_temp2); | ||||
| 3659 | |||||
| 3660 | load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask); | ||||
| 3661 | load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask); | ||||
| 3662 | |||||
| 3663 | __masm-> cmpl(keylen, 52); | ||||
| 3664 | __masm-> jccb(Assembler::equal, L_doLast)jccb_0(Assembler::equal, L_doLast, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3664); | ||||
| 3665 | |||||
| 3666 | __masm-> aesdec(xmm_result, xmm_temp1); | ||||
| 3667 | __masm-> aesdec(xmm_result, xmm_temp2); | ||||
| 3668 | |||||
| 3669 | load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask); | ||||
| 3670 | load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask); | ||||
| 3671 | |||||
| 3672 | __masm-> BIND(L_doLast)bind(L_doLast); masm-> block_comment("L_doLast" ":"); | ||||
| 3673 | __masm-> aesdec(xmm_result, xmm_temp1); | ||||
| 3674 | __masm-> aesdec(xmm_result, xmm_temp2); | ||||
| 3675 | |||||
| 3676 | // for decryption the aesdeclast operation is always on key+0x00 | ||||
| 3677 | __masm-> aesdeclast(xmm_result, xmm_temp3); | ||||
| 3678 | __masm-> movdqu(Address(to, 0), xmm_result); // store the result | ||||
| 3679 | __masm-> xorptr(rax, rax); // return 0 | ||||
| 3680 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3681 | __masm-> ret(0); | ||||
| 3682 | |||||
| 3683 | return start; | ||||
| 3684 | } | ||||
| 3685 | |||||
| 3686 | |||||
| 3687 | // Arguments: | ||||
| 3688 | // | ||||
| 3689 | // Inputs: | ||||
| 3690 | // c_rarg0 - source byte array address | ||||
| 3691 | // c_rarg1 - destination byte array address | ||||
| 3692 | // c_rarg2 - K (key) in little endian int array | ||||
| 3693 | // c_rarg3 - r vector byte array address | ||||
| 3694 | // c_rarg4 - input length | ||||
| 3695 | // | ||||
| 3696 | // Output: | ||||
| 3697 | // rax - input length | ||||
| 3698 | // | ||||
| 3699 | address generate_cipherBlockChaining_encryptAESCrypt() { | ||||
| 3700 | assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3700, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support" ); ::breakpoint(); } } while (0); | ||||
| 3701 | __masm-> align(CodeEntryAlignment); | ||||
| 3702 | StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt"); | ||||
| 3703 | address start = __masm-> pc(); | ||||
| 3704 | |||||
| 3705 | Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256; | ||||
| 3706 | const Register from = c_rarg0; // source array address | ||||
| 3707 | const Register to = c_rarg1; // destination array address | ||||
| 3708 | const Register key = c_rarg2; // key array address | ||||
| 3709 | const Register rvec = c_rarg3; // r byte array initialized from initvector array address | ||||
| 3710 | // and left with the results of the last encryption block | ||||
| 3711 | #ifndef _WIN64 | ||||
| 3712 | const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) | ||||
| 3713 | #else | ||||
| 3714 | const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 3715 | const Register len_reg = r11; // pick the volatile windows register | ||||
| 3716 | #endif | ||||
| 3717 | const Register pos = rax; | ||||
| 3718 | |||||
| 3719 | // xmm register assignments for the loops below | ||||
| 3720 | const XMMRegister xmm_result = xmm0; | ||||
| 3721 | const XMMRegister xmm_temp = xmm1; | ||||
| 3722 | // keys 0-10 preloaded into xmm2-xmm12 | ||||
| 3723 | const int XMM_REG_NUM_KEY_FIRST = 2; | ||||
| 3724 | const int XMM_REG_NUM_KEY_LAST = 15; | ||||
| 3725 | const XMMRegister xmm_key0 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST); | ||||
| 3726 | const XMMRegister xmm_key10 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10); | ||||
| 3727 | const XMMRegister xmm_key11 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11); | ||||
| 3728 | const XMMRegister xmm_key12 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12); | ||||
| 3729 | const XMMRegister xmm_key13 = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13); | ||||
| 3730 | |||||
| 3731 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3732 | |||||
| 3733 | #ifdef _WIN64 | ||||
| 3734 | // on win64, fill len_reg from stack position | ||||
| 3735 | __masm-> movl(len_reg, len_mem); | ||||
| 3736 | #else | ||||
| 3737 | __masm-> push(len_reg); // Save | ||||
| 3738 | #endif | ||||
| 3739 | |||||
| 3740 | const XMMRegister xmm_key_shuf_mask = xmm_temp; // used temporarily to swap key bytes up front | ||||
| 3741 | __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 3742 | // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0 | ||||
| 3743 | for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) { | ||||
| 3744 | load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask); | ||||
| 3745 | offset += 0x10; | ||||
| 3746 | } | ||||
| 3747 | __masm-> movdqu(xmm_result, Address(rvec, 0x00)); // initialize xmm_result with r vec | ||||
| 3748 | |||||
| 3749 | // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256)) | ||||
| 3750 | __masm-> movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); | ||||
| 3751 | __masm-> cmpl(rax, 44); | ||||
| 3752 | __masm-> jcc(Assembler::notEqual, L_key_192_256); | ||||
| 3753 | |||||
| 3754 | // 128 bit code follows here | ||||
| 3755 | __masm-> movptr(pos, 0); | ||||
| 3756 | __masm-> align(OptoLoopAlignment); | ||||
| 3757 | |||||
| 3758 | __masm-> BIND(L_loopTop_128)bind(L_loopTop_128); masm-> block_comment("L_loopTop_128" ":" ); | ||||
| 3759 | __masm-> movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input | ||||
| 3760 | __masm-> pxor (xmm_result, xmm_temp); // xor with the current r vector | ||||
| 3761 | __masm-> pxor (xmm_result, xmm_key0); // do the aes rounds | ||||
| 3762 | for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) { | ||||
| 3763 | __masm-> aesenc(xmm_result, as_XMMRegister(rnum)); | ||||
| 3764 | } | ||||
| 3765 | __masm-> aesenclast(xmm_result, xmm_key10); | ||||
| 3766 | __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output | ||||
| 3767 | // no need to store r to memory until we exit | ||||
| 3768 | __masm-> addptr(pos, AESBlockSize); | ||||
| 3769 | __masm-> subptr(len_reg, AESBlockSize); | ||||
| 3770 | __masm-> jcc(Assembler::notEqual, L_loopTop_128); | ||||
| 3771 | |||||
| 3772 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 3773 | __masm-> movdqu(Address(rvec, 0), xmm_result); // final value of r stored in rvec of CipherBlockChaining object | ||||
| 3774 | |||||
| 3775 | #ifdef _WIN64 | ||||
| 3776 | __masm-> movl(rax, len_mem); | ||||
| 3777 | #else | ||||
| 3778 | __masm-> pop(rax); // return length | ||||
| 3779 | #endif | ||||
| 3780 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3781 | __masm-> ret(0); | ||||
| 3782 | |||||
| 3783 | __masm-> BIND(L_key_192_256)bind(L_key_192_256); masm-> block_comment("L_key_192_256" ":" ); | ||||
| 3784 | // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256) | ||||
| 3785 | load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask); | ||||
| 3786 | load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask); | ||||
| 3787 | __masm-> cmpl(rax, 52); | ||||
| 3788 | __masm-> jcc(Assembler::notEqual, L_key_256); | ||||
| 3789 | |||||
| 3790 | // 192-bit code follows here (could be changed to use more xmm registers) | ||||
| 3791 | __masm-> movptr(pos, 0); | ||||
| 3792 | __masm-> align(OptoLoopAlignment); | ||||
| 3793 | |||||
| 3794 | __masm-> BIND(L_loopTop_192)bind(L_loopTop_192); masm-> block_comment("L_loopTop_192" ":" ); | ||||
| 3795 | __masm-> movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input | ||||
| 3796 | __masm-> pxor (xmm_result, xmm_temp); // xor with the current r vector | ||||
| 3797 | __masm-> pxor (xmm_result, xmm_key0); // do the aes rounds | ||||
| 3798 | for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) { | ||||
| 3799 | __masm-> aesenc(xmm_result, as_XMMRegister(rnum)); | ||||
| 3800 | } | ||||
| 3801 | __masm-> aesenclast(xmm_result, xmm_key12); | ||||
| 3802 | __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output | ||||
| 3803 | // no need to store r to memory until we exit | ||||
| 3804 | __masm-> addptr(pos, AESBlockSize); | ||||
| 3805 | __masm-> subptr(len_reg, AESBlockSize); | ||||
| 3806 | __masm-> jcc(Assembler::notEqual, L_loopTop_192); | ||||
| 3807 | __masm-> jmp(L_exit); | ||||
| 3808 | |||||
| 3809 | __masm-> BIND(L_key_256)bind(L_key_256); masm-> block_comment("L_key_256" ":"); | ||||
| 3810 | // 256-bit code follows here (could be changed to use more xmm registers) | ||||
| 3811 | load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask); | ||||
| 3812 | __masm-> movptr(pos, 0); | ||||
| 3813 | __masm-> align(OptoLoopAlignment); | ||||
| 3814 | |||||
| 3815 | __masm-> BIND(L_loopTop_256)bind(L_loopTop_256); masm-> block_comment("L_loopTop_256" ":" ); | ||||
| 3816 | __masm-> movdqu(xmm_temp, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of input | ||||
| 3817 | __masm-> pxor (xmm_result, xmm_temp); // xor with the current r vector | ||||
| 3818 | __masm-> pxor (xmm_result, xmm_key0); // do the aes rounds | ||||
| 3819 | for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) { | ||||
| 3820 | __masm-> aesenc(xmm_result, as_XMMRegister(rnum)); | ||||
| 3821 | } | ||||
| 3822 | load_key(xmm_temp, key, 0xe0); | ||||
| 3823 | __masm-> aesenclast(xmm_result, xmm_temp); | ||||
| 3824 | __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output | ||||
| 3825 | // no need to store r to memory until we exit | ||||
| 3826 | __masm-> addptr(pos, AESBlockSize); | ||||
| 3827 | __masm-> subptr(len_reg, AESBlockSize); | ||||
| 3828 | __masm-> jcc(Assembler::notEqual, L_loopTop_256); | ||||
| 3829 | __masm-> jmp(L_exit); | ||||
| 3830 | |||||
| 3831 | return start; | ||||
| 3832 | } | ||||
| 3833 | |||||
| 3834 | // Safefetch stubs. | ||||
| 3835 | void generate_safefetch(const char* name, int size, address* entry, | ||||
| 3836 | address* fault_pc, address* continuation_pc) { | ||||
| 3837 | // safefetch signatures: | ||||
| 3838 | // int SafeFetch32(int* adr, int errValue); | ||||
| 3839 | // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); | ||||
| 3840 | // | ||||
| 3841 | // arguments: | ||||
| 3842 | // c_rarg0 = adr | ||||
| 3843 | // c_rarg1 = errValue | ||||
| 3844 | // | ||||
| 3845 | // result: | ||||
| 3846 | // PPC_RET = *adr or errValue | ||||
| 3847 | |||||
| 3848 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 3849 | |||||
| 3850 | // Entry point, pc or function descriptor. | ||||
| 3851 | *entry = __masm-> pc(); | ||||
| 3852 | |||||
| 3853 | // Load *adr into c_rarg1, may fault. | ||||
| 3854 | *fault_pc = __masm-> pc(); | ||||
| 3855 | switch (size) { | ||||
| 3856 | case 4: | ||||
| 3857 | // int32_t | ||||
| 3858 | __masm-> movl(c_rarg1, Address(c_rarg0, 0)); | ||||
| 3859 | break; | ||||
| 3860 | case 8: | ||||
| 3861 | // int64_t | ||||
| 3862 | __masm-> movq(c_rarg1, Address(c_rarg0, 0)); | ||||
| 3863 | break; | ||||
| 3864 | default: | ||||
| 3865 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3865); ::breakpoint(); } while (0); | ||||
| 3866 | } | ||||
| 3867 | |||||
| 3868 | // return errValue or *adr | ||||
| 3869 | *continuation_pc = __masm-> pc(); | ||||
| 3870 | __masm-> movq(rax, c_rarg1); | ||||
| 3871 | __masm-> ret(0); | ||||
| 3872 | } | ||||
| 3873 | |||||
| 3874 | // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time | ||||
| 3875 | // to hide instruction latency | ||||
| 3876 | // | ||||
| 3877 | // Arguments: | ||||
| 3878 | // | ||||
| 3879 | // Inputs: | ||||
| 3880 | // c_rarg0 - source byte array address | ||||
| 3881 | // c_rarg1 - destination byte array address | ||||
| 3882 | // c_rarg2 - K (key) in little endian int array | ||||
| 3883 | // c_rarg3 - r vector byte array address | ||||
| 3884 | // c_rarg4 - input length | ||||
| 3885 | // | ||||
| 3886 | // Output: | ||||
| 3887 | // rax - input length | ||||
| 3888 | // | ||||
| 3889 | address generate_cipherBlockChaining_decryptAESCrypt_Parallel() { | ||||
| 3890 | assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 3890, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support" ); ::breakpoint(); } } while (0); | ||||
| 3891 | __masm-> align(CodeEntryAlignment); | ||||
| 3892 | StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt"); | ||||
| 3893 | address start = __masm-> pc(); | ||||
| 3894 | |||||
| 3895 | const Register from = c_rarg0; // source array address | ||||
| 3896 | const Register to = c_rarg1; // destination array address | ||||
| 3897 | const Register key = c_rarg2; // key array address | ||||
| 3898 | const Register rvec = c_rarg3; // r byte array initialized from initvector array address | ||||
| 3899 | // and left with the results of the last encryption block | ||||
| 3900 | #ifndef _WIN64 | ||||
| 3901 | const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) | ||||
| 3902 | #else | ||||
| 3903 | const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 3904 | const Register len_reg = r11; // pick the volatile windows register | ||||
| 3905 | #endif | ||||
| 3906 | const Register pos = rax; | ||||
| 3907 | |||||
| 3908 | const int PARALLEL_FACTOR = 4; | ||||
| 3909 | const int ROUNDS[3] = { 10, 12, 14 }; // aes rounds for key128, key192, key256 | ||||
| 3910 | |||||
| 3911 | Label L_exit; | ||||
| 3912 | Label L_singleBlock_loopTopHead[3]; // 128, 192, 256 | ||||
| 3913 | Label L_singleBlock_loopTopHead2[3]; // 128, 192, 256 | ||||
| 3914 | Label L_singleBlock_loopTop[3]; // 128, 192, 256 | ||||
| 3915 | Label L_multiBlock_loopTopHead[3]; // 128, 192, 256 | ||||
| 3916 | Label L_multiBlock_loopTop[3]; // 128, 192, 256 | ||||
| 3917 | |||||
| 3918 | // keys 0-10 preloaded into xmm5-xmm15 | ||||
| 3919 | const int XMM_REG_NUM_KEY_FIRST = 5; | ||||
| 3920 | const int XMM_REG_NUM_KEY_LAST = 15; | ||||
| 3921 | const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST); | ||||
| 3922 | const XMMRegister xmm_key_last = as_XMMRegister(XMM_REG_NUM_KEY_LAST); | ||||
| 3923 | |||||
| 3924 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 3925 | |||||
| 3926 | #ifdef _WIN64 | ||||
| 3927 | // on win64, fill len_reg from stack position | ||||
| 3928 | __masm-> movl(len_reg, len_mem); | ||||
| 3929 | #else | ||||
| 3930 | __masm-> push(len_reg); // Save | ||||
| 3931 | #endif | ||||
| 3932 | __masm-> push(rbx); | ||||
| 3933 | // the java expanded key ordering is rotated one position from what we want | ||||
| 3934 | // so we start from 0x10 here and hit 0x00 last | ||||
| 3935 | const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front | ||||
| 3936 | __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 3937 | // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00 | ||||
| 3938 | for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) { | ||||
| 3939 | load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask); | ||||
| 3940 | offset += 0x10; | ||||
| 3941 | } | ||||
| 3942 | load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask); | ||||
| 3943 | |||||
| 3944 | const XMMRegister xmm_prev_block_cipher = xmm1; // holds cipher of previous block | ||||
| 3945 | |||||
| 3946 | // registers holding the four results in the parallelized loop | ||||
| 3947 | const XMMRegister xmm_result0 = xmm0; | ||||
| 3948 | const XMMRegister xmm_result1 = xmm2; | ||||
| 3949 | const XMMRegister xmm_result2 = xmm3; | ||||
| 3950 | const XMMRegister xmm_result3 = xmm4; | ||||
| 3951 | |||||
| 3952 | __masm-> movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec | ||||
| 3953 | |||||
| 3954 | __masm-> xorptr(pos, pos); | ||||
| 3955 | |||||
| 3956 | // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256)) | ||||
| 3957 | __masm-> movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); | ||||
| 3958 | __masm-> cmpl(rbx, 52); | ||||
| 3959 | __masm-> jcc(Assembler::equal, L_multiBlock_loopTopHead[1]); | ||||
| 3960 | __masm-> cmpl(rbx, 60); | ||||
| 3961 | __masm-> jcc(Assembler::equal, L_multiBlock_loopTopHead[2]); | ||||
| 3962 | |||||
| 3963 | #define DoFour(opc, src_reg)masm-> opc(xmm_result0, src_reg); masm-> opc(xmm_result1 , src_reg); masm-> opc(xmm_result2, src_reg); masm-> opc (xmm_result3, src_reg); \ | ||||
| 3964 | __masm-> opc(xmm_result0, src_reg); \ | ||||
| 3965 | __masm-> opc(xmm_result1, src_reg); \ | ||||
| 3966 | __masm-> opc(xmm_result2, src_reg); \ | ||||
| 3967 | __masm-> opc(xmm_result3, src_reg); \ | ||||
| 3968 | |||||
| 3969 | for (int k = 0; k < 3; ++k) { | ||||
| 3970 | __masm-> BIND(L_multiBlock_loopTopHead[k])bind(L_multiBlock_loopTopHead[k]); masm-> block_comment("L_multiBlock_loopTopHead[k]" ":"); | ||||
| 3971 | if (k != 0) { | ||||
| 3972 | __masm-> cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left | ||||
| 3973 | __masm-> jcc(Assembler::less, L_singleBlock_loopTopHead2[k]); | ||||
| 3974 | } | ||||
| 3975 | if (k == 1) { | ||||
| 3976 | __masm-> subptr(rsp, 6 * wordSize); | ||||
| 3977 | __masm-> movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15 | ||||
| 3978 | load_key(xmm15, key, 0xb0); // 0xb0; 192-bit key goes up to 0xc0 | ||||
| 3979 | __masm-> movdqu(Address(rsp, 2 * wordSize), xmm15); | ||||
| 3980 | load_key(xmm1, key, 0xc0); // 0xc0; | ||||
| 3981 | __masm-> movdqu(Address(rsp, 4 * wordSize), xmm1); | ||||
| 3982 | } else if (k == 2) { | ||||
| 3983 | __masm-> subptr(rsp, 10 * wordSize); | ||||
| 3984 | __masm-> movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15 | ||||
| 3985 | load_key(xmm15, key, 0xd0); // 0xd0; 256-bit key goes upto 0xe0 | ||||
| 3986 | __masm-> movdqu(Address(rsp, 6 * wordSize), xmm15); | ||||
| 3987 | load_key(xmm1, key, 0xe0); // 0xe0; | ||||
| 3988 | __masm-> movdqu(Address(rsp, 8 * wordSize), xmm1); | ||||
| 3989 | load_key(xmm15, key, 0xb0); // 0xb0; | ||||
| 3990 | __masm-> movdqu(Address(rsp, 2 * wordSize), xmm15); | ||||
| 3991 | load_key(xmm1, key, 0xc0); // 0xc0; | ||||
| 3992 | __masm-> movdqu(Address(rsp, 4 * wordSize), xmm1); | ||||
| 3993 | } | ||||
| 3994 | __masm-> align(OptoLoopAlignment); | ||||
| 3995 | __masm-> BIND(L_multiBlock_loopTop[k])bind(L_multiBlock_loopTop[k]); masm-> block_comment("L_multiBlock_loopTop[k]" ":"); | ||||
| 3996 | __masm-> cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left | ||||
| 3997 | __masm-> jcc(Assembler::less, L_singleBlock_loopTopHead[k]); | ||||
| 3998 | |||||
| 3999 | if (k != 0) { | ||||
| 4000 | __masm-> movdqu(xmm15, Address(rsp, 2 * wordSize)); | ||||
| 4001 | __masm-> movdqu(xmm1, Address(rsp, 4 * wordSize)); | ||||
| 4002 | } | ||||
| 4003 | |||||
| 4004 | __masm-> movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers | ||||
| 4005 | __masm-> movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize)); | ||||
| 4006 | __masm-> movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize)); | ||||
| 4007 | __masm-> movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize)); | ||||
| 4008 | |||||
| 4009 | DoFour(pxor, xmm_key_first)masm-> pxor(xmm_result0, xmm_key_first); masm-> pxor(xmm_result1 , xmm_key_first); masm-> pxor(xmm_result2, xmm_key_first); masm-> pxor(xmm_result3, xmm_key_first);; | ||||
| 4010 | if (k == 0) { | ||||
| 4011 | for (int rnum = 1; rnum < ROUNDS[k]; rnum++) { | ||||
| 4012 | DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST))masm-> aesdec(xmm_result0, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result1, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result2, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result3, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST ));; | ||||
| 4013 | } | ||||
| 4014 | DoFour(aesdeclast, xmm_key_last)masm-> aesdeclast(xmm_result0, xmm_key_last); masm-> aesdeclast (xmm_result1, xmm_key_last); masm-> aesdeclast(xmm_result2 , xmm_key_last); masm-> aesdeclast(xmm_result3, xmm_key_last );; | ||||
| 4015 | } else if (k == 1) { | ||||
| 4016 | for (int rnum = 1; rnum <= ROUNDS[k]-2; rnum++) { | ||||
| 4017 | DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST))masm-> aesdec(xmm_result0, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result1, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result2, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result3, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST ));; | ||||
| 4018 | } | ||||
| 4019 | __masm-> movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again. | ||||
| 4020 | DoFour(aesdec, xmm1)masm-> aesdec(xmm_result0, xmm1); masm-> aesdec(xmm_result1 , xmm1); masm-> aesdec(xmm_result2, xmm1); masm-> aesdec (xmm_result3, xmm1);; // key : 0xc0 | ||||
| 4021 | __masm-> movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // xmm1 needs to be loaded again | ||||
| 4022 | DoFour(aesdeclast, xmm_key_last)masm-> aesdeclast(xmm_result0, xmm_key_last); masm-> aesdeclast (xmm_result1, xmm_key_last); masm-> aesdeclast(xmm_result2 , xmm_key_last); masm-> aesdeclast(xmm_result3, xmm_key_last );; | ||||
| 4023 | } else if (k == 2) { | ||||
| 4024 | for (int rnum = 1; rnum <= ROUNDS[k] - 4; rnum++) { | ||||
| 4025 | DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST))masm-> aesdec(xmm_result0, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result1, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result2, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST )); masm-> aesdec(xmm_result3, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST ));; | ||||
| 4026 | } | ||||
| 4027 | DoFour(aesdec, xmm1)masm-> aesdec(xmm_result0, xmm1); masm-> aesdec(xmm_result1 , xmm1); masm-> aesdec(xmm_result2, xmm1); masm-> aesdec (xmm_result3, xmm1);; // key : 0xc0 | ||||
| 4028 | __masm-> movdqu(xmm15, Address(rsp, 6 * wordSize)); | ||||
| 4029 | __masm-> movdqu(xmm1, Address(rsp, 8 * wordSize)); | ||||
| 4030 | DoFour(aesdec, xmm15)masm-> aesdec(xmm_result0, xmm15); masm-> aesdec(xmm_result1 , xmm15); masm-> aesdec(xmm_result2, xmm15); masm-> aesdec (xmm_result3, xmm15);; // key : 0xd0 | ||||
| 4031 | __masm-> movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again. | ||||
| 4032 | DoFour(aesdec, xmm1)masm-> aesdec(xmm_result0, xmm1); masm-> aesdec(xmm_result1 , xmm1); masm-> aesdec(xmm_result2, xmm1); masm-> aesdec (xmm_result3, xmm1);; // key : 0xe0 | ||||
| 4033 | __masm-> movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // xmm1 needs to be loaded again | ||||
| 4034 | DoFour(aesdeclast, xmm_key_last)masm-> aesdeclast(xmm_result0, xmm_key_last); masm-> aesdeclast (xmm_result1, xmm_key_last); masm-> aesdeclast(xmm_result2 , xmm_key_last); masm-> aesdeclast(xmm_result3, xmm_key_last );; | ||||
| 4035 | } | ||||
| 4036 | |||||
| 4037 | // for each result, xor with the r vector of previous cipher block | ||||
| 4038 | __masm-> pxor(xmm_result0, xmm_prev_block_cipher); | ||||
| 4039 | __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize)); | ||||
| 4040 | __masm-> pxor(xmm_result1, xmm_prev_block_cipher); | ||||
| 4041 | __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize)); | ||||
| 4042 | __masm-> pxor(xmm_result2, xmm_prev_block_cipher); | ||||
| 4043 | __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize)); | ||||
| 4044 | __masm-> pxor(xmm_result3, xmm_prev_block_cipher); | ||||
| 4045 | __masm-> movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks | ||||
| 4046 | if (k != 0) { | ||||
| 4047 | __masm-> movdqu(Address(rvec, 0x00), xmm_prev_block_cipher); | ||||
| 4048 | } | ||||
| 4049 | |||||
| 4050 | __masm-> movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0); // store 4 results into the next 64 bytes of output | ||||
| 4051 | __masm-> movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1); | ||||
| 4052 | __masm-> movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2); | ||||
| 4053 | __masm-> movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3); | ||||
| 4054 | |||||
| 4055 | __masm-> addptr(pos, PARALLEL_FACTOR * AESBlockSize); | ||||
| 4056 | __masm-> subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); | ||||
| 4057 | __masm-> jmp(L_multiBlock_loopTop[k]); | ||||
| 4058 | |||||
| 4059 | // registers used in the non-parallelized loops | ||||
| 4060 | // xmm register assignments for the loops below | ||||
| 4061 | const XMMRegister xmm_result = xmm0; | ||||
| 4062 | const XMMRegister xmm_prev_block_cipher_save = xmm2; | ||||
| 4063 | const XMMRegister xmm_key11 = xmm3; | ||||
| 4064 | const XMMRegister xmm_key12 = xmm4; | ||||
| 4065 | const XMMRegister key_tmp = xmm4; | ||||
| 4066 | |||||
| 4067 | __masm-> BIND(L_singleBlock_loopTopHead[k])bind(L_singleBlock_loopTopHead[k]); masm-> block_comment("L_singleBlock_loopTopHead[k]" ":"); | ||||
| 4068 | if (k == 1) { | ||||
| 4069 | __masm-> addptr(rsp, 6 * wordSize); | ||||
| 4070 | } else if (k == 2) { | ||||
| 4071 | __masm-> addptr(rsp, 10 * wordSize); | ||||
| 4072 | } | ||||
| 4073 | __masm-> cmpptr(len_reg, 0); // any blocks left?? | ||||
| 4074 | __masm-> jcc(Assembler::equal, L_exit); | ||||
| 4075 | __masm-> BIND(L_singleBlock_loopTopHead2[k])bind(L_singleBlock_loopTopHead2[k]); masm-> block_comment( "L_singleBlock_loopTopHead2[k]" ":"); | ||||
| 4076 | if (k == 1) { | ||||
| 4077 | load_key(xmm_key11, key, 0xb0); // 0xb0; 192-bit key goes upto 0xc0 | ||||
| 4078 | load_key(xmm_key12, key, 0xc0); // 0xc0; 192-bit key goes upto 0xc0 | ||||
| 4079 | } | ||||
| 4080 | if (k == 2) { | ||||
| 4081 | load_key(xmm_key11, key, 0xb0); // 0xb0; 256-bit key goes upto 0xe0 | ||||
| 4082 | } | ||||
| 4083 | __masm-> align(OptoLoopAlignment); | ||||
| 4084 | __masm-> BIND(L_singleBlock_loopTop[k])bind(L_singleBlock_loopTop[k]); masm-> block_comment("L_singleBlock_loopTop[k]" ":"); | ||||
| 4085 | __masm-> movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input | ||||
| 4086 | __masm-> movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector | ||||
| 4087 | __masm-> pxor(xmm_result, xmm_key_first); // do the aes dec rounds | ||||
| 4088 | for (int rnum = 1; rnum <= 9 ; rnum++) { | ||||
| 4089 | __masm-> aesdec(xmm_result, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST)); | ||||
| 4090 | } | ||||
| 4091 | if (k == 1) { | ||||
| 4092 | __masm-> aesdec(xmm_result, xmm_key11); | ||||
| 4093 | __masm-> aesdec(xmm_result, xmm_key12); | ||||
| 4094 | } | ||||
| 4095 | if (k == 2) { | ||||
| 4096 | __masm-> aesdec(xmm_result, xmm_key11); | ||||
| 4097 | load_key(key_tmp, key, 0xc0); | ||||
| 4098 | __masm-> aesdec(xmm_result, key_tmp); | ||||
| 4099 | load_key(key_tmp, key, 0xd0); | ||||
| 4100 | __masm-> aesdec(xmm_result, key_tmp); | ||||
| 4101 | load_key(key_tmp, key, 0xe0); | ||||
| 4102 | __masm-> aesdec(xmm_result, key_tmp); | ||||
| 4103 | } | ||||
| 4104 | |||||
| 4105 | __masm-> aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0 | ||||
| 4106 | __masm-> pxor(xmm_result, xmm_prev_block_cipher); // xor with the current r vector | ||||
| 4107 | __masm-> movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output | ||||
| 4108 | // no need to store r to memory until we exit | ||||
| 4109 | __masm-> movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block | ||||
| 4110 | __masm-> addptr(pos, AESBlockSize); | ||||
| 4111 | __masm-> subptr(len_reg, AESBlockSize); | ||||
| 4112 | __masm-> jcc(Assembler::notEqual, L_singleBlock_loopTop[k]); | ||||
| 4113 | if (k != 2) { | ||||
| 4114 | __masm-> jmp(L_exit); | ||||
| 4115 | } | ||||
| 4116 | } //for 128/192/256 | ||||
| 4117 | |||||
| 4118 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 4119 | __masm-> movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object | ||||
| 4120 | __masm-> pop(rbx); | ||||
| 4121 | #ifdef _WIN64 | ||||
| 4122 | __masm-> movl(rax, len_mem); | ||||
| 4123 | #else | ||||
| 4124 | __masm-> pop(rax); // return length | ||||
| 4125 | #endif | ||||
| 4126 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4127 | __masm-> ret(0); | ||||
| 4128 | return start; | ||||
| 4129 | } | ||||
| 4130 | |||||
| 4131 | address generate_electronicCodeBook_encryptAESCrypt() { | ||||
| 4132 | __masm-> align(CodeEntryAlignment); | ||||
| 4133 | StubCodeMark mark(this, "StubRoutines", "electronicCodeBook_encryptAESCrypt"); | ||||
| 4134 | address start = __masm-> pc(); | ||||
| 4135 | const Register from = c_rarg0; // source array address | ||||
| 4136 | const Register to = c_rarg1; // destination array address | ||||
| 4137 | const Register key = c_rarg2; // key array address | ||||
| 4138 | const Register len = c_rarg3; // src len (must be multiple of blocksize 16) | ||||
| 4139 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4140 | __masm-> aesecb_encrypt(from, to, key, len); | ||||
| 4141 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4142 | __masm-> ret(0); | ||||
| 4143 | return start; | ||||
| 4144 | } | ||||
| 4145 | |||||
| 4146 | address generate_electronicCodeBook_decryptAESCrypt() { | ||||
| 4147 | __masm-> align(CodeEntryAlignment); | ||||
| 4148 | StubCodeMark mark(this, "StubRoutines", "electronicCodeBook_decryptAESCrypt"); | ||||
| 4149 | address start = __masm-> pc(); | ||||
| 4150 | const Register from = c_rarg0; // source array address | ||||
| 4151 | const Register to = c_rarg1; // destination array address | ||||
| 4152 | const Register key = c_rarg2; // key array address | ||||
| 4153 | const Register len = c_rarg3; // src len (must be multiple of blocksize 16) | ||||
| 4154 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4155 | __masm-> aesecb_decrypt(from, to, key, len); | ||||
| 4156 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4157 | __masm-> ret(0); | ||||
| 4158 | return start; | ||||
| 4159 | } | ||||
| 4160 | |||||
| 4161 | // ofs and limit are use for multi-block byte array. | ||||
| 4162 | // int com.sun.security.provider.MD5.implCompress(byte[] b, int ofs) | ||||
| 4163 | address generate_md5_implCompress(bool multi_block, const char *name) { | ||||
| 4164 | __masm-> align(CodeEntryAlignment); | ||||
| 4165 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 4166 | address start = __masm-> pc(); | ||||
| 4167 | |||||
| 4168 | const Register buf_param = r15; | ||||
| 4169 | const Address state_param(rsp, 0 * wordSize); | ||||
| 4170 | const Address ofs_param (rsp, 1 * wordSize ); | ||||
| 4171 | const Address limit_param(rsp, 1 * wordSize + 4); | ||||
| 4172 | |||||
| 4173 | __masm-> enter(); | ||||
| 4174 | __masm-> push(rbx); | ||||
| 4175 | __masm-> push(rdi); | ||||
| 4176 | __masm-> push(rsi); | ||||
| 4177 | __masm-> push(r15); | ||||
| 4178 | __masm-> subptr(rsp, 2 * wordSize); | ||||
| 4179 | |||||
| 4180 | __masm-> movptr(buf_param, c_rarg0); | ||||
| 4181 | __masm-> movptr(state_param, c_rarg1); | ||||
| 4182 | if (multi_block) { | ||||
| 4183 | __masm-> movl(ofs_param, c_rarg2); | ||||
| 4184 | __masm-> movl(limit_param, c_rarg3); | ||||
| 4185 | } | ||||
| 4186 | __masm-> fast_md5(buf_param, state_param, ofs_param, limit_param, multi_block); | ||||
| 4187 | |||||
| 4188 | __masm-> addptr(rsp, 2 * wordSize); | ||||
| 4189 | __masm-> pop(r15); | ||||
| 4190 | __masm-> pop(rsi); | ||||
| 4191 | __masm-> pop(rdi); | ||||
| 4192 | __masm-> pop(rbx); | ||||
| 4193 | __masm-> leave(); | ||||
| 4194 | __masm-> ret(0); | ||||
| 4195 | return start; | ||||
| 4196 | } | ||||
| 4197 | |||||
| 4198 | address generate_upper_word_mask() { | ||||
| 4199 | __masm-> align64(); | ||||
| 4200 | StubCodeMark mark(this, "StubRoutines", "upper_word_mask"); | ||||
| 4201 | address start = __masm-> pc(); | ||||
| 4202 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4203 | __masm-> emit_data64(0xFFFFFFFF00000000, relocInfo::none); | ||||
| 4204 | return start; | ||||
| 4205 | } | ||||
| 4206 | |||||
| 4207 | address generate_shuffle_byte_flip_mask() { | ||||
| 4208 | __masm-> align64(); | ||||
| 4209 | StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask"); | ||||
| 4210 | address start = __masm-> pc(); | ||||
| 4211 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); | ||||
| 4212 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); | ||||
| 4213 | return start; | ||||
| 4214 | } | ||||
| 4215 | |||||
| 4216 | // ofs and limit are use for multi-block byte array. | ||||
| 4217 | // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) | ||||
| 4218 | address generate_sha1_implCompress(bool multi_block, const char *name) { | ||||
| 4219 | __masm-> align(CodeEntryAlignment); | ||||
| 4220 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 4221 | address start = __masm-> pc(); | ||||
| 4222 | |||||
| 4223 | Register buf = c_rarg0; | ||||
| 4224 | Register state = c_rarg1; | ||||
| 4225 | Register ofs = c_rarg2; | ||||
| 4226 | Register limit = c_rarg3; | ||||
| 4227 | |||||
| 4228 | const XMMRegister abcd = xmm0; | ||||
| 4229 | const XMMRegister e0 = xmm1; | ||||
| 4230 | const XMMRegister e1 = xmm2; | ||||
| 4231 | const XMMRegister msg0 = xmm3; | ||||
| 4232 | |||||
| 4233 | const XMMRegister msg1 = xmm4; | ||||
| 4234 | const XMMRegister msg2 = xmm5; | ||||
| 4235 | const XMMRegister msg3 = xmm6; | ||||
| 4236 | const XMMRegister shuf_mask = xmm7; | ||||
| 4237 | |||||
| 4238 | __masm-> enter(); | ||||
| 4239 | |||||
| 4240 | __masm-> subptr(rsp, 4 * wordSize); | ||||
| 4241 | |||||
| 4242 | __masm-> fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask, | ||||
| 4243 | buf, state, ofs, limit, rsp, multi_block); | ||||
| 4244 | |||||
| 4245 | __masm-> addptr(rsp, 4 * wordSize); | ||||
| 4246 | |||||
| 4247 | __masm-> leave(); | ||||
| 4248 | __masm-> ret(0); | ||||
| 4249 | return start; | ||||
| 4250 | } | ||||
| 4251 | |||||
| 4252 | address generate_pshuffle_byte_flip_mask() { | ||||
| 4253 | __masm-> align64(); | ||||
| 4254 | StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask"); | ||||
| 4255 | address start = __masm-> pc(); | ||||
| 4256 | __masm-> emit_data64(0x0405060700010203, relocInfo::none); | ||||
| 4257 | __masm-> emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); | ||||
| 4258 | |||||
| 4259 | if (VM_Version::supports_avx2()) { | ||||
| 4260 | __masm-> emit_data64(0x0405060700010203, relocInfo::none); // second copy | ||||
| 4261 | __masm-> emit_data64(0x0c0d0e0f08090a0b, relocInfo::none); | ||||
| 4262 | // _SHUF_00BA | ||||
| 4263 | __masm-> emit_data64(0x0b0a090803020100, relocInfo::none); | ||||
| 4264 | __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); | ||||
| 4265 | __masm-> emit_data64(0x0b0a090803020100, relocInfo::none); | ||||
| 4266 | __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); | ||||
| 4267 | // _SHUF_DC00 | ||||
| 4268 | __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); | ||||
| 4269 | __masm-> emit_data64(0x0b0a090803020100, relocInfo::none); | ||||
| 4270 | __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); | ||||
| 4271 | __masm-> emit_data64(0x0b0a090803020100, relocInfo::none); | ||||
| 4272 | } | ||||
| 4273 | |||||
| 4274 | return start; | ||||
| 4275 | } | ||||
| 4276 | |||||
| 4277 | //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. | ||||
| 4278 | address generate_pshuffle_byte_flip_mask_sha512() { | ||||
| 4279 | __masm-> align32(); | ||||
| 4280 | StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512"); | ||||
| 4281 | address start = __masm-> pc(); | ||||
| 4282 | if (VM_Version::supports_avx2()) { | ||||
| 4283 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK | ||||
| 4284 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); | ||||
| 4285 | __masm-> emit_data64(0x1011121314151617, relocInfo::none); | ||||
| 4286 | __masm-> emit_data64(0x18191a1b1c1d1e1f, relocInfo::none); | ||||
| 4287 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO | ||||
| 4288 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4289 | __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); | ||||
| 4290 | __masm-> emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none); | ||||
| 4291 | } | ||||
| 4292 | |||||
| 4293 | return start; | ||||
| 4294 | } | ||||
| 4295 | |||||
| 4296 | // ofs and limit are use for multi-block byte array. | ||||
| 4297 | // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) | ||||
| 4298 | address generate_sha256_implCompress(bool multi_block, const char *name) { | ||||
| 4299 | assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "")do { if (!(VM_Version::supports_sha() || VM_Version::supports_avx2 ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 4299, "assert(" "VM_Version::supports_sha() || VM_Version::supports_avx2()" ") failed", ""); ::breakpoint(); } } while (0); | ||||
| 4300 | __masm-> align(CodeEntryAlignment); | ||||
| 4301 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 4302 | address start = __masm-> pc(); | ||||
| 4303 | |||||
| 4304 | Register buf = c_rarg0; | ||||
| 4305 | Register state = c_rarg1; | ||||
| 4306 | Register ofs = c_rarg2; | ||||
| 4307 | Register limit = c_rarg3; | ||||
| 4308 | |||||
| 4309 | const XMMRegister msg = xmm0; | ||||
| 4310 | const XMMRegister state0 = xmm1; | ||||
| 4311 | const XMMRegister state1 = xmm2; | ||||
| 4312 | const XMMRegister msgtmp0 = xmm3; | ||||
| 4313 | |||||
| 4314 | const XMMRegister msgtmp1 = xmm4; | ||||
| 4315 | const XMMRegister msgtmp2 = xmm5; | ||||
| 4316 | const XMMRegister msgtmp3 = xmm6; | ||||
| 4317 | const XMMRegister msgtmp4 = xmm7; | ||||
| 4318 | |||||
| 4319 | const XMMRegister shuf_mask = xmm8; | ||||
| 4320 | |||||
| 4321 | __masm-> enter(); | ||||
| 4322 | |||||
| 4323 | __masm-> subptr(rsp, 4 * wordSize); | ||||
| 4324 | |||||
| 4325 | if (VM_Version::supports_sha()) { | ||||
| 4326 | __masm-> fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, | ||||
| 4327 | buf, state, ofs, limit, rsp, multi_block, shuf_mask); | ||||
| 4328 | } else if (VM_Version::supports_avx2()) { | ||||
| 4329 | __masm-> sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, | ||||
| 4330 | buf, state, ofs, limit, rsp, multi_block, shuf_mask); | ||||
| 4331 | } | ||||
| 4332 | __masm-> addptr(rsp, 4 * wordSize); | ||||
| 4333 | __masm-> vzeroupper(); | ||||
| 4334 | __masm-> leave(); | ||||
| 4335 | __masm-> ret(0); | ||||
| 4336 | return start; | ||||
| 4337 | } | ||||
| 4338 | |||||
| 4339 | address generate_sha512_implCompress(bool multi_block, const char *name) { | ||||
| 4340 | assert(VM_Version::supports_avx2(), "")do { if (!(VM_Version::supports_avx2())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 4340, "assert(" "VM_Version::supports_avx2()" ") failed", "" ); ::breakpoint(); } } while (0); | ||||
| 4341 | assert(VM_Version::supports_bmi2(), "")do { if (!(VM_Version::supports_bmi2())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 4341, "assert(" "VM_Version::supports_bmi2()" ") failed", "" ); ::breakpoint(); } } while (0); | ||||
| 4342 | __masm-> align(CodeEntryAlignment); | ||||
| 4343 | StubCodeMark mark(this, "StubRoutines", name); | ||||
| 4344 | address start = __masm-> pc(); | ||||
| 4345 | |||||
| 4346 | Register buf = c_rarg0; | ||||
| 4347 | Register state = c_rarg1; | ||||
| 4348 | Register ofs = c_rarg2; | ||||
| 4349 | Register limit = c_rarg3; | ||||
| 4350 | |||||
| 4351 | const XMMRegister msg = xmm0; | ||||
| 4352 | const XMMRegister state0 = xmm1; | ||||
| 4353 | const XMMRegister state1 = xmm2; | ||||
| 4354 | const XMMRegister msgtmp0 = xmm3; | ||||
| 4355 | const XMMRegister msgtmp1 = xmm4; | ||||
| 4356 | const XMMRegister msgtmp2 = xmm5; | ||||
| 4357 | const XMMRegister msgtmp3 = xmm6; | ||||
| 4358 | const XMMRegister msgtmp4 = xmm7; | ||||
| 4359 | |||||
| 4360 | const XMMRegister shuf_mask = xmm8; | ||||
| 4361 | |||||
| 4362 | __masm-> enter(); | ||||
| 4363 | |||||
| 4364 | __masm-> sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4, | ||||
| 4365 | buf, state, ofs, limit, rsp, multi_block, shuf_mask); | ||||
| 4366 | |||||
| 4367 | __masm-> vzeroupper(); | ||||
| 4368 | __masm-> leave(); | ||||
| 4369 | __masm-> ret(0); | ||||
| 4370 | return start; | ||||
| 4371 | } | ||||
| 4372 | |||||
| 4373 | address ghash_polynomial512_addr() { | ||||
| 4374 | __masm-> align(CodeEntryAlignment); | ||||
| 4375 | StubCodeMark mark(this, "StubRoutines", "_ghash_poly512_addr"); | ||||
| 4376 | address start = __masm-> pc(); | ||||
| 4377 | __masm-> emit_data64(0x00000001C2000000, relocInfo::none); // POLY for reduction | ||||
| 4378 | __masm-> emit_data64(0xC200000000000000, relocInfo::none); | ||||
| 4379 | __masm-> emit_data64(0x00000001C2000000, relocInfo::none); | ||||
| 4380 | __masm-> emit_data64(0xC200000000000000, relocInfo::none); | ||||
| 4381 | __masm-> emit_data64(0x00000001C2000000, relocInfo::none); | ||||
| 4382 | __masm-> emit_data64(0xC200000000000000, relocInfo::none); | ||||
| 4383 | __masm-> emit_data64(0x00000001C2000000, relocInfo::none); | ||||
| 4384 | __masm-> emit_data64(0xC200000000000000, relocInfo::none); | ||||
| 4385 | __masm-> emit_data64(0x0000000000000001, relocInfo::none); // POLY | ||||
| 4386 | __masm-> emit_data64(0xC200000000000000, relocInfo::none); | ||||
| 4387 | __masm-> emit_data64(0x0000000000000001, relocInfo::none); // TWOONE | ||||
| 4388 | __masm-> emit_data64(0x0000000100000000, relocInfo::none); | ||||
| 4389 | return start; | ||||
| 4390 | } | ||||
| 4391 | |||||
| 4392 | // Vector AES Galois Counter Mode implementation. Parameters: | ||||
| 4393 | // Windows regs | Linux regs | ||||
| 4394 | // in = c_rarg0 (rcx) | c_rarg0 (rsi) | ||||
| 4395 | // len = c_rarg1 (rdx) | c_rarg1 (rdi) | ||||
| 4396 | // ct = c_rarg2 (r8) | c_rarg2 (rdx) | ||||
| 4397 | // out = c_rarg3 (r9) | c_rarg3 (rcx) | ||||
| 4398 | // key = r10 | c_rarg4 (r8) | ||||
| 4399 | // state = r13 | c_rarg5 (r9) | ||||
| 4400 | // subkeyHtbl = r14 | r11 | ||||
| 4401 | // counter = rsi | r12 | ||||
| 4402 | // return - number of processed bytes | ||||
| 4403 | address generate_galoisCounterMode_AESCrypt() { | ||||
| 4404 | __masm-> align(CodeEntryAlignment); | ||||
| 4405 | StubCodeMark mark(this, "StubRoutines", "galoisCounterMode_AESCrypt"); | ||||
| 4406 | address start = __masm-> pc(); | ||||
| 4407 | const Register in = c_rarg0; | ||||
| 4408 | const Register len = c_rarg1; | ||||
| 4409 | const Register ct = c_rarg2; | ||||
| 4410 | const Register out = c_rarg3; | ||||
| 4411 | // and updated with the incremented counter in the end | ||||
| 4412 | #ifndef _WIN64 | ||||
| 4413 | const Register key = c_rarg4; | ||||
| 4414 | const Register state = c_rarg5; | ||||
| 4415 | const Address subkeyH_mem(rbp, 2 * wordSize); | ||||
| 4416 | const Register subkeyHtbl = r11; | ||||
| 4417 | const Address avx512_subkeyH_mem(rbp, 3 * wordSize); | ||||
| 4418 | const Register avx512_subkeyHtbl = r13; | ||||
| 4419 | const Address counter_mem(rbp, 4 * wordSize); | ||||
| 4420 | const Register counter = r12; | ||||
| 4421 | #else | ||||
| 4422 | const Address key_mem(rbp, 6 * wordSize); | ||||
| 4423 | const Register key = r10; | ||||
| 4424 | const Address state_mem(rbp, 7 * wordSize); | ||||
| 4425 | const Register state = r13; | ||||
| 4426 | const Address subkeyH_mem(rbp, 8 * wordSize); | ||||
| 4427 | const Register subkeyHtbl = r14; | ||||
| 4428 | const Address avx512_subkeyH_mem(rbp, 9 * wordSize); | ||||
| 4429 | const Register avx512_subkeyHtbl = r12; | ||||
| 4430 | const Address counter_mem(rbp, 10 * wordSize); | ||||
| 4431 | const Register counter = rsi; | ||||
| 4432 | #endif | ||||
| 4433 | __masm-> enter(); | ||||
| 4434 | // Save state before entering routine | ||||
| 4435 | __masm-> push(r12); | ||||
| 4436 | __masm-> push(r13); | ||||
| 4437 | __masm-> push(r14); | ||||
| 4438 | __masm-> push(r15); | ||||
| 4439 | __masm-> push(rbx); | ||||
| 4440 | #ifdef _WIN64 | ||||
| 4441 | // on win64, fill len_reg from stack position | ||||
| 4442 | __masm-> push(rsi); | ||||
| 4443 | __masm-> movptr(key, key_mem); | ||||
| 4444 | __masm-> movptr(state, state_mem); | ||||
| 4445 | #endif | ||||
| 4446 | __masm-> movptr(subkeyHtbl, subkeyH_mem); | ||||
| 4447 | __masm-> movptr(avx512_subkeyHtbl, avx512_subkeyH_mem); | ||||
| 4448 | __masm-> movptr(counter, counter_mem); | ||||
| 4449 | |||||
| 4450 | __masm-> aesgcm_encrypt(in, len, ct, out, key, state, subkeyHtbl, avx512_subkeyHtbl, counter); | ||||
| 4451 | |||||
| 4452 | // Restore state before leaving routine | ||||
| 4453 | #ifdef _WIN64 | ||||
| 4454 | __masm-> pop(rsi); | ||||
| 4455 | #endif | ||||
| 4456 | __masm-> pop(rbx); | ||||
| 4457 | __masm-> pop(r15); | ||||
| 4458 | __masm-> pop(r14); | ||||
| 4459 | __masm-> pop(r13); | ||||
| 4460 | __masm-> pop(r12); | ||||
| 4461 | |||||
| 4462 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4463 | __masm-> ret(0); | ||||
| 4464 | return start; | ||||
| 4465 | } | ||||
| 4466 | |||||
| 4467 | // This mask is used for incrementing counter value(linc0, linc4, etc.) | ||||
| 4468 | address counter_mask_addr() { | ||||
| 4469 | __masm-> align64(); | ||||
| 4470 | StubCodeMark mark(this, "StubRoutines", "counter_mask_addr"); | ||||
| 4471 | address start = __masm-> pc(); | ||||
| 4472 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);//lbswapmask | ||||
| 4473 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); | ||||
| 4474 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); | ||||
| 4475 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); | ||||
| 4476 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); | ||||
| 4477 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); | ||||
| 4478 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none); | ||||
| 4479 | __masm-> emit_data64(0x0001020304050607, relocInfo::none); | ||||
| 4480 | __masm-> emit_data64(0x0000000000000000, relocInfo::none);//linc0 = counter_mask_addr+64 | ||||
| 4481 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4482 | __masm-> emit_data64(0x0000000000000001, relocInfo::none);//counter_mask_addr() + 80 | ||||
| 4483 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4484 | __masm-> emit_data64(0x0000000000000002, relocInfo::none); | ||||
| 4485 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4486 | __masm-> emit_data64(0x0000000000000003, relocInfo::none); | ||||
| 4487 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4488 | __masm-> emit_data64(0x0000000000000004, relocInfo::none);//linc4 = counter_mask_addr() + 128 | ||||
| 4489 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4490 | __masm-> emit_data64(0x0000000000000004, relocInfo::none); | ||||
| 4491 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4492 | __masm-> emit_data64(0x0000000000000004, relocInfo::none); | ||||
| 4493 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4494 | __masm-> emit_data64(0x0000000000000004, relocInfo::none); | ||||
| 4495 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4496 | __masm-> emit_data64(0x0000000000000008, relocInfo::none);//linc8 = counter_mask_addr() + 192 | ||||
| 4497 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4498 | __masm-> emit_data64(0x0000000000000008, relocInfo::none); | ||||
| 4499 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4500 | __masm-> emit_data64(0x0000000000000008, relocInfo::none); | ||||
| 4501 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4502 | __masm-> emit_data64(0x0000000000000008, relocInfo::none); | ||||
| 4503 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4504 | __masm-> emit_data64(0x0000000000000020, relocInfo::none);//linc32 = counter_mask_addr() + 256 | ||||
| 4505 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4506 | __masm-> emit_data64(0x0000000000000020, relocInfo::none); | ||||
| 4507 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4508 | __masm-> emit_data64(0x0000000000000020, relocInfo::none); | ||||
| 4509 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4510 | __masm-> emit_data64(0x0000000000000020, relocInfo::none); | ||||
| 4511 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4512 | __masm-> emit_data64(0x0000000000000010, relocInfo::none);//linc16 = counter_mask_addr() + 320 | ||||
| 4513 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4514 | __masm-> emit_data64(0x0000000000000010, relocInfo::none); | ||||
| 4515 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4516 | __masm-> emit_data64(0x0000000000000010, relocInfo::none); | ||||
| 4517 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4518 | __masm-> emit_data64(0x0000000000000010, relocInfo::none); | ||||
| 4519 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 4520 | return start; | ||||
| 4521 | } | ||||
| 4522 | |||||
| 4523 | // Vector AES Counter implementation | ||||
| 4524 | address generate_counterMode_VectorAESCrypt() { | ||||
| 4525 | __masm-> align(CodeEntryAlignment); | ||||
| 4526 | StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt"); | ||||
| 4527 | address start = __masm-> pc(); | ||||
| 4528 | const Register from = c_rarg0; // source array address | ||||
| 4529 | const Register to = c_rarg1; // destination array address | ||||
| 4530 | const Register key = c_rarg2; // key array address r8 | ||||
| 4531 | const Register counter = c_rarg3; // counter byte array initialized from counter array address | ||||
| 4532 | // and updated with the incremented counter in the end | ||||
| 4533 | #ifndef _WIN64 | ||||
| 4534 | const Register len_reg = c_rarg4; | ||||
| 4535 | const Register saved_encCounter_start = c_rarg5; | ||||
| 4536 | const Register used_addr = r10; | ||||
| 4537 | const Address used_mem(rbp, 2 * wordSize); | ||||
| 4538 | const Register used = r11; | ||||
| 4539 | #else | ||||
| 4540 | const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 4541 | const Address saved_encCounter_mem(rbp, 7 * wordSize); // saved encrypted counter is on stack on Win64 | ||||
| 4542 | const Address used_mem(rbp, 8 * wordSize); // used length is on stack on Win64 | ||||
| 4543 | const Register len_reg = r10; // pick the first volatile windows register | ||||
| 4544 | const Register saved_encCounter_start = r11; | ||||
| 4545 | const Register used_addr = r13; | ||||
| 4546 | const Register used = r14; | ||||
| 4547 | #endif | ||||
| 4548 | __masm-> enter(); | ||||
| 4549 | // Save state before entering routine | ||||
| 4550 | __masm-> push(r12); | ||||
| 4551 | __masm-> push(r13); | ||||
| 4552 | __masm-> push(r14); | ||||
| 4553 | __masm-> push(r15); | ||||
| 4554 | #ifdef _WIN64 | ||||
| 4555 | // on win64, fill len_reg from stack position | ||||
| 4556 | __masm-> movl(len_reg, len_mem); | ||||
| 4557 | __masm-> movptr(saved_encCounter_start, saved_encCounter_mem); | ||||
| 4558 | __masm-> movptr(used_addr, used_mem); | ||||
| 4559 | __masm-> movl(used, Address(used_addr, 0)); | ||||
| 4560 | #else | ||||
| 4561 | __masm-> push(len_reg); // Save | ||||
| 4562 | __masm-> movptr(used_addr, used_mem); | ||||
| 4563 | __masm-> movl(used, Address(used_addr, 0)); | ||||
| 4564 | #endif | ||||
| 4565 | __masm-> push(rbx); | ||||
| 4566 | __masm-> aesctr_encrypt(from, to, key, counter, len_reg, used, used_addr, saved_encCounter_start); | ||||
| 4567 | // Restore state before leaving routine | ||||
| 4568 | __masm-> pop(rbx); | ||||
| 4569 | #ifdef _WIN64 | ||||
| 4570 | __masm-> movl(rax, len_mem); // return length | ||||
| 4571 | #else | ||||
| 4572 | __masm-> pop(rax); // return length | ||||
| 4573 | #endif | ||||
| 4574 | __masm-> pop(r15); | ||||
| 4575 | __masm-> pop(r14); | ||||
| 4576 | __masm-> pop(r13); | ||||
| 4577 | __masm-> pop(r12); | ||||
| 4578 | |||||
| 4579 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4580 | __masm-> ret(0); | ||||
| 4581 | return start; | ||||
| 4582 | } | ||||
| 4583 | |||||
| 4584 | // This is a version of CTR/AES crypt which does 6 blocks in a loop at a time | ||||
| 4585 | // to hide instruction latency | ||||
| 4586 | // | ||||
| 4587 | // Arguments: | ||||
| 4588 | // | ||||
| 4589 | // Inputs: | ||||
| 4590 | // c_rarg0 - source byte array address | ||||
| 4591 | // c_rarg1 - destination byte array address | ||||
| 4592 | // c_rarg2 - K (key) in little endian int array | ||||
| 4593 | // c_rarg3 - counter vector byte array address | ||||
| 4594 | // Linux | ||||
| 4595 | // c_rarg4 - input length | ||||
| 4596 | // c_rarg5 - saved encryptedCounter start | ||||
| 4597 | // rbp + 6 * wordSize - saved used length | ||||
| 4598 | // Windows | ||||
| 4599 | // rbp + 6 * wordSize - input length | ||||
| 4600 | // rbp + 7 * wordSize - saved encryptedCounter start | ||||
| 4601 | // rbp + 8 * wordSize - saved used length | ||||
| 4602 | // | ||||
| 4603 | // Output: | ||||
| 4604 | // rax - input length | ||||
| 4605 | // | ||||
| 4606 | address generate_counterMode_AESCrypt_Parallel() { | ||||
| 4607 | assert(UseAES, "need AES instructions and misaligned SSE support")do { if (!(UseAES)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 4607, "assert(" "UseAES" ") failed", "need AES instructions and misaligned SSE support" ); ::breakpoint(); } } while (0); | ||||
| 4608 | __masm-> align(CodeEntryAlignment); | ||||
| 4609 | StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt"); | ||||
| 4610 | address start = __masm-> pc(); | ||||
| 4611 | const Register from = c_rarg0; // source array address | ||||
| 4612 | const Register to = c_rarg1; // destination array address | ||||
| 4613 | const Register key = c_rarg2; // key array address | ||||
| 4614 | const Register counter = c_rarg3; // counter byte array initialized from counter array address | ||||
| 4615 | // and updated with the incremented counter in the end | ||||
| 4616 | #ifndef _WIN64 | ||||
| 4617 | const Register len_reg = c_rarg4; | ||||
| 4618 | const Register saved_encCounter_start = c_rarg5; | ||||
| 4619 | const Register used_addr = r10; | ||||
| 4620 | const Address used_mem(rbp, 2 * wordSize); | ||||
| 4621 | const Register used = r11; | ||||
| 4622 | #else | ||||
| 4623 | const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 4624 | const Address saved_encCounter_mem(rbp, 7 * wordSize); // length is on stack on Win64 | ||||
| 4625 | const Address used_mem(rbp, 8 * wordSize); // length is on stack on Win64 | ||||
| 4626 | const Register len_reg = r10; // pick the first volatile windows register | ||||
| 4627 | const Register saved_encCounter_start = r11; | ||||
| 4628 | const Register used_addr = r13; | ||||
| 4629 | const Register used = r14; | ||||
| 4630 | #endif | ||||
| 4631 | const Register pos = rax; | ||||
| 4632 | |||||
| 4633 | const int PARALLEL_FACTOR = 6; | ||||
| 4634 | const XMMRegister xmm_counter_shuf_mask = xmm0; | ||||
| 4635 | const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front | ||||
| 4636 | const XMMRegister xmm_curr_counter = xmm2; | ||||
| 4637 | |||||
| 4638 | const XMMRegister xmm_key_tmp0 = xmm3; | ||||
| 4639 | const XMMRegister xmm_key_tmp1 = xmm4; | ||||
| 4640 | |||||
| 4641 | // registers holding the four results in the parallelized loop | ||||
| 4642 | const XMMRegister xmm_result0 = xmm5; | ||||
| 4643 | const XMMRegister xmm_result1 = xmm6; | ||||
| 4644 | const XMMRegister xmm_result2 = xmm7; | ||||
| 4645 | const XMMRegister xmm_result3 = xmm8; | ||||
| 4646 | const XMMRegister xmm_result4 = xmm9; | ||||
| 4647 | const XMMRegister xmm_result5 = xmm10; | ||||
| 4648 | |||||
| 4649 | const XMMRegister xmm_from0 = xmm11; | ||||
| 4650 | const XMMRegister xmm_from1 = xmm12; | ||||
| 4651 | const XMMRegister xmm_from2 = xmm13; | ||||
| 4652 | const XMMRegister xmm_from3 = xmm14; //the last one is xmm14. we have to preserve it on WIN64. | ||||
| 4653 | const XMMRegister xmm_from4 = xmm3; //reuse xmm3~4. Because xmm_key_tmp0~1 are useless when loading input text | ||||
| 4654 | const XMMRegister xmm_from5 = xmm4; | ||||
| 4655 | |||||
| 4656 | //for key_128, key_192, key_256 | ||||
| 4657 | const int rounds[3] = {10, 12, 14}; | ||||
| 4658 | Label L_exit_preLoop, L_preLoop_start; | ||||
| 4659 | Label L_multiBlock_loopTop[3]; | ||||
| 4660 | Label L_singleBlockLoopTop[3]; | ||||
| 4661 | Label L__incCounter[3][6]; //for 6 blocks | ||||
| 4662 | Label L__incCounter_single[3]; //for single block, key128, key192, key256 | ||||
| 4663 | Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3]; | ||||
| 4664 | Label L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3]; | ||||
| 4665 | |||||
| 4666 | Label L_exit; | ||||
| 4667 | |||||
| 4668 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4669 | |||||
| 4670 | #ifdef _WIN64 | ||||
| 4671 | // allocate spill slots for r13, r14 | ||||
| 4672 | enum { | ||||
| 4673 | saved_r13_offset, | ||||
| 4674 | saved_r14_offset | ||||
| 4675 | }; | ||||
| 4676 | __masm-> subptr(rsp, 2 * wordSize); | ||||
| 4677 | __masm-> movptr(Address(rsp, saved_r13_offset * wordSize), r13); | ||||
| 4678 | __masm-> movptr(Address(rsp, saved_r14_offset * wordSize), r14); | ||||
| 4679 | |||||
| 4680 | // on win64, fill len_reg from stack position | ||||
| 4681 | __masm-> movl(len_reg, len_mem); | ||||
| 4682 | __masm-> movptr(saved_encCounter_start, saved_encCounter_mem); | ||||
| 4683 | __masm-> movptr(used_addr, used_mem); | ||||
| 4684 | __masm-> movl(used, Address(used_addr, 0)); | ||||
| 4685 | #else | ||||
| 4686 | __masm-> push(len_reg); // Save | ||||
| 4687 | __masm-> movptr(used_addr, used_mem); | ||||
| 4688 | __masm-> movl(used, Address(used_addr, 0)); | ||||
| 4689 | #endif | ||||
| 4690 | |||||
| 4691 | __masm-> push(rbx); // Save RBX | ||||
| 4692 | __masm-> movdqu(xmm_curr_counter, Address(counter, 0x00)); // initialize counter with initial counter | ||||
| 4693 | __masm-> movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()), pos); // pos as scratch | ||||
| 4694 | __masm-> pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled | ||||
| 4695 | __masm-> movptr(pos, 0); | ||||
| 4696 | |||||
| 4697 | // Use the partially used encrpyted counter from last invocation | ||||
| 4698 | __masm-> BIND(L_preLoop_start)bind(L_preLoop_start); masm-> block_comment("L_preLoop_start" ":"); | ||||
| 4699 | __masm-> cmpptr(used, 16); | ||||
| 4700 | __masm-> jcc(Assembler::aboveEqual, L_exit_preLoop); | ||||
| 4701 | __masm-> cmpptr(len_reg, 0); | ||||
| 4702 | __masm-> jcc(Assembler::lessEqual, L_exit_preLoop); | ||||
| 4703 | __masm-> movb(rbx, Address(saved_encCounter_start, used)); | ||||
| 4704 | __masm-> xorb(rbx, Address(from, pos)); | ||||
| 4705 | __masm-> movb(Address(to, pos), rbx); | ||||
| 4706 | __masm-> addptr(pos, 1); | ||||
| 4707 | __masm-> addptr(used, 1); | ||||
| 4708 | __masm-> subptr(len_reg, 1); | ||||
| 4709 | |||||
| 4710 | __masm-> jmp(L_preLoop_start); | ||||
| 4711 | |||||
| 4712 | __masm-> BIND(L_exit_preLoop)bind(L_exit_preLoop); masm-> block_comment("L_exit_preLoop" ":"); | ||||
| 4713 | __masm-> movl(Address(used_addr, 0), used); | ||||
| 4714 | |||||
| 4715 | // key length could be only {11, 13, 15} * 4 = {44, 52, 60} | ||||
| 4716 | __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()), rbx); // rbx as scratch | ||||
| 4717 | __masm-> movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); | ||||
| 4718 | __masm-> cmpl(rbx, 52); | ||||
| 4719 | __masm-> jcc(Assembler::equal, L_multiBlock_loopTop[1]); | ||||
| 4720 | __masm-> cmpl(rbx, 60); | ||||
| 4721 | __masm-> jcc(Assembler::equal, L_multiBlock_loopTop[2]); | ||||
| 4722 | |||||
| 4723 | #define CTR_DoSix(opc, src_reg)masm-> opc(xmm_result0, src_reg); masm-> opc(xmm_result1 , src_reg); masm-> opc(xmm_result2, src_reg); masm-> opc (xmm_result3, src_reg); masm-> opc(xmm_result4, src_reg); masm -> opc(xmm_result5, src_reg); \ | ||||
| 4724 | __masm-> opc(xmm_result0, src_reg); \ | ||||
| 4725 | __masm-> opc(xmm_result1, src_reg); \ | ||||
| 4726 | __masm-> opc(xmm_result2, src_reg); \ | ||||
| 4727 | __masm-> opc(xmm_result3, src_reg); \ | ||||
| 4728 | __masm-> opc(xmm_result4, src_reg); \ | ||||
| 4729 | __masm-> opc(xmm_result5, src_reg); | ||||
| 4730 | |||||
| 4731 | // k == 0 : generate code for key_128 | ||||
| 4732 | // k == 1 : generate code for key_192 | ||||
| 4733 | // k == 2 : generate code for key_256 | ||||
| 4734 | for (int k = 0; k < 3; ++k) { | ||||
| 4735 | //multi blocks starts here | ||||
| 4736 | __masm-> align(OptoLoopAlignment); | ||||
| 4737 | __masm-> BIND(L_multiBlock_loopTop[k])bind(L_multiBlock_loopTop[k]); masm-> block_comment("L_multiBlock_loopTop[k]" ":"); | ||||
| 4738 | __masm-> cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left | ||||
| 4739 | __masm-> jcc(Assembler::less, L_singleBlockLoopTop[k]); | ||||
| 4740 | load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask); | ||||
| 4741 | |||||
| 4742 | //load, then increase counters | ||||
| 4743 | CTR_DoSix(movdqa, xmm_curr_counter)masm-> movdqa(xmm_result0, xmm_curr_counter); masm-> movdqa (xmm_result1, xmm_curr_counter); masm-> movdqa(xmm_result2 , xmm_curr_counter); masm-> movdqa(xmm_result3, xmm_curr_counter ); masm-> movdqa(xmm_result4, xmm_curr_counter); masm-> movdqa(xmm_result5, xmm_curr_counter);; | ||||
| 4744 | inc_counter(rbx, xmm_result1, 0x01, L__incCounter[k][0]); | ||||
| 4745 | inc_counter(rbx, xmm_result2, 0x02, L__incCounter[k][1]); | ||||
| 4746 | inc_counter(rbx, xmm_result3, 0x03, L__incCounter[k][2]); | ||||
| 4747 | inc_counter(rbx, xmm_result4, 0x04, L__incCounter[k][3]); | ||||
| 4748 | inc_counter(rbx, xmm_result5, 0x05, L__incCounter[k][4]); | ||||
| 4749 | inc_counter(rbx, xmm_curr_counter, 0x06, L__incCounter[k][5]); | ||||
| 4750 | CTR_DoSix(pshufb, xmm_counter_shuf_mask)masm-> pshufb(xmm_result0, xmm_counter_shuf_mask); masm-> pshufb(xmm_result1, xmm_counter_shuf_mask); masm-> pshufb (xmm_result2, xmm_counter_shuf_mask); masm-> pshufb(xmm_result3 , xmm_counter_shuf_mask); masm-> pshufb(xmm_result4, xmm_counter_shuf_mask ); masm-> pshufb(xmm_result5, xmm_counter_shuf_mask);; // after increased, shuffled counters back for PXOR | ||||
| 4751 | CTR_DoSix(pxor, xmm_key_tmp0)masm-> pxor(xmm_result0, xmm_key_tmp0); masm-> pxor(xmm_result1 , xmm_key_tmp0); masm-> pxor(xmm_result2, xmm_key_tmp0); masm -> pxor(xmm_result3, xmm_key_tmp0); masm-> pxor(xmm_result4 , xmm_key_tmp0); masm-> pxor(xmm_result5, xmm_key_tmp0);; //PXOR with Round 0 key | ||||
| 4752 | |||||
| 4753 | //load two ROUND_KEYs at a time | ||||
| 4754 | for (int i = 1; i < rounds[k]; ) { | ||||
| 4755 | load_key(xmm_key_tmp1, key, (0x10 * i), xmm_key_shuf_mask); | ||||
| 4756 | load_key(xmm_key_tmp0, key, (0x10 * (i+1)), xmm_key_shuf_mask); | ||||
| 4757 | CTR_DoSix(aesenc, xmm_key_tmp1)masm-> aesenc(xmm_result0, xmm_key_tmp1); masm-> aesenc (xmm_result1, xmm_key_tmp1); masm-> aesenc(xmm_result2, xmm_key_tmp1 ); masm-> aesenc(xmm_result3, xmm_key_tmp1); masm-> aesenc (xmm_result4, xmm_key_tmp1); masm-> aesenc(xmm_result5, xmm_key_tmp1 );; | ||||
| 4758 | i++; | ||||
| 4759 | if (i != rounds[k]) { | ||||
| 4760 | CTR_DoSix(aesenc, xmm_key_tmp0)masm-> aesenc(xmm_result0, xmm_key_tmp0); masm-> aesenc (xmm_result1, xmm_key_tmp0); masm-> aesenc(xmm_result2, xmm_key_tmp0 ); masm-> aesenc(xmm_result3, xmm_key_tmp0); masm-> aesenc (xmm_result4, xmm_key_tmp0); masm-> aesenc(xmm_result5, xmm_key_tmp0 );; | ||||
| 4761 | } else { | ||||
| 4762 | CTR_DoSix(aesenclast, xmm_key_tmp0)masm-> aesenclast(xmm_result0, xmm_key_tmp0); masm-> aesenclast (xmm_result1, xmm_key_tmp0); masm-> aesenclast(xmm_result2 , xmm_key_tmp0); masm-> aesenclast(xmm_result3, xmm_key_tmp0 ); masm-> aesenclast(xmm_result4, xmm_key_tmp0); masm-> aesenclast(xmm_result5, xmm_key_tmp0);; | ||||
| 4763 | } | ||||
| 4764 | i++; | ||||
| 4765 | } | ||||
| 4766 | |||||
| 4767 | // get next PARALLEL_FACTOR blocks into xmm_result registers | ||||
| 4768 | __masm-> movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); | ||||
| 4769 | __masm-> movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize)); | ||||
| 4770 | __masm-> movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize)); | ||||
| 4771 | __masm-> movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize)); | ||||
| 4772 | __masm-> movdqu(xmm_from4, Address(from, pos, Address::times_1, 4 * AESBlockSize)); | ||||
| 4773 | __masm-> movdqu(xmm_from5, Address(from, pos, Address::times_1, 5 * AESBlockSize)); | ||||
| 4774 | |||||
| 4775 | __masm-> pxor(xmm_result0, xmm_from0); | ||||
| 4776 | __masm-> pxor(xmm_result1, xmm_from1); | ||||
| 4777 | __masm-> pxor(xmm_result2, xmm_from2); | ||||
| 4778 | __masm-> pxor(xmm_result3, xmm_from3); | ||||
| 4779 | __masm-> pxor(xmm_result4, xmm_from4); | ||||
| 4780 | __masm-> pxor(xmm_result5, xmm_from5); | ||||
| 4781 | |||||
| 4782 | // store 6 results into the next 64 bytes of output | ||||
| 4783 | __masm-> movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0); | ||||
| 4784 | __masm-> movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1); | ||||
| 4785 | __masm-> movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2); | ||||
| 4786 | __masm-> movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3); | ||||
| 4787 | __masm-> movdqu(Address(to, pos, Address::times_1, 4 * AESBlockSize), xmm_result4); | ||||
| 4788 | __masm-> movdqu(Address(to, pos, Address::times_1, 5 * AESBlockSize), xmm_result5); | ||||
| 4789 | |||||
| 4790 | __masm-> addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text | ||||
| 4791 | __masm-> subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length | ||||
| 4792 | __masm-> jmp(L_multiBlock_loopTop[k]); | ||||
| 4793 | |||||
| 4794 | // singleBlock starts here | ||||
| 4795 | __masm-> align(OptoLoopAlignment); | ||||
| 4796 | __masm-> BIND(L_singleBlockLoopTop[k])bind(L_singleBlockLoopTop[k]); masm-> block_comment("L_singleBlockLoopTop[k]" ":"); | ||||
| 4797 | __masm-> cmpptr(len_reg, 0); | ||||
| 4798 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 4799 | load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask); | ||||
| 4800 | __masm-> movdqa(xmm_result0, xmm_curr_counter); | ||||
| 4801 | inc_counter(rbx, xmm_curr_counter, 0x01, L__incCounter_single[k]); | ||||
| 4802 | __masm-> pshufb(xmm_result0, xmm_counter_shuf_mask); | ||||
| 4803 | __masm-> pxor(xmm_result0, xmm_key_tmp0); | ||||
| 4804 | for (int i = 1; i < rounds[k]; i++) { | ||||
| 4805 | load_key(xmm_key_tmp0, key, (0x10 * i), xmm_key_shuf_mask); | ||||
| 4806 | __masm-> aesenc(xmm_result0, xmm_key_tmp0); | ||||
| 4807 | } | ||||
| 4808 | load_key(xmm_key_tmp0, key, (rounds[k] * 0x10), xmm_key_shuf_mask); | ||||
| 4809 | __masm-> aesenclast(xmm_result0, xmm_key_tmp0); | ||||
| 4810 | __masm-> cmpptr(len_reg, AESBlockSize); | ||||
| 4811 | __masm-> jcc(Assembler::less, L_processTail_insr[k]); | ||||
| 4812 | __masm-> movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); | ||||
| 4813 | __masm-> pxor(xmm_result0, xmm_from0); | ||||
| 4814 | __masm-> movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0); | ||||
| 4815 | __masm-> addptr(pos, AESBlockSize); | ||||
| 4816 | __masm-> subptr(len_reg, AESBlockSize); | ||||
| 4817 | __masm-> jmp(L_singleBlockLoopTop[k]); | ||||
| 4818 | __masm-> BIND(L_processTail_insr[k])bind(L_processTail_insr[k]); masm-> block_comment("L_processTail_insr[k]" ":"); // Process the tail part of the input array | ||||
| 4819 | __masm-> addptr(pos, len_reg); // 1. Insert bytes from src array into xmm_from0 register | ||||
| 4820 | __masm-> testptr(len_reg, 8); | ||||
| 4821 | __masm-> jcc(Assembler::zero, L_processTail_4_insr[k]); | ||||
| 4822 | __masm-> subptr(pos,8); | ||||
| 4823 | __masm-> pinsrq(xmm_from0, Address(from, pos), 0); | ||||
| 4824 | __masm-> BIND(L_processTail_4_insr[k])bind(L_processTail_4_insr[k]); masm-> block_comment("L_processTail_4_insr[k]" ":"); | ||||
| 4825 | __masm-> testptr(len_reg, 4); | ||||
| 4826 | __masm-> jcc(Assembler::zero, L_processTail_2_insr[k]); | ||||
| 4827 | __masm-> subptr(pos,4); | ||||
| 4828 | __masm-> pslldq(xmm_from0, 4); | ||||
| 4829 | __masm-> pinsrd(xmm_from0, Address(from, pos), 0); | ||||
| 4830 | __masm-> BIND(L_processTail_2_insr[k])bind(L_processTail_2_insr[k]); masm-> block_comment("L_processTail_2_insr[k]" ":"); | ||||
| 4831 | __masm-> testptr(len_reg, 2); | ||||
| 4832 | __masm-> jcc(Assembler::zero, L_processTail_1_insr[k]); | ||||
| 4833 | __masm-> subptr(pos, 2); | ||||
| 4834 | __masm-> pslldq(xmm_from0, 2); | ||||
| 4835 | __masm-> pinsrw(xmm_from0, Address(from, pos), 0); | ||||
| 4836 | __masm-> BIND(L_processTail_1_insr[k])bind(L_processTail_1_insr[k]); masm-> block_comment("L_processTail_1_insr[k]" ":"); | ||||
| 4837 | __masm-> testptr(len_reg, 1); | ||||
| 4838 | __masm-> jcc(Assembler::zero, L_processTail_exit_insr[k]); | ||||
| 4839 | __masm-> subptr(pos, 1); | ||||
| 4840 | __masm-> pslldq(xmm_from0, 1); | ||||
| 4841 | __masm-> pinsrb(xmm_from0, Address(from, pos), 0); | ||||
| 4842 | __masm-> BIND(L_processTail_exit_insr[k])bind(L_processTail_exit_insr[k]); masm-> block_comment("L_processTail_exit_insr[k]" ":"); | ||||
| 4843 | |||||
| 4844 | __masm-> movdqu(Address(saved_encCounter_start, 0), xmm_result0); // 2. Perform pxor of the encrypted counter and plaintext Bytes. | ||||
| 4845 | __masm-> pxor(xmm_result0, xmm_from0); // Also the encrypted counter is saved for next invocation. | ||||
| 4846 | |||||
| 4847 | __masm-> testptr(len_reg, 8); | ||||
| 4848 | __masm-> jcc(Assembler::zero, L_processTail_4_extr[k]); // 3. Extract bytes from xmm_result0 into the dest. array | ||||
| 4849 | __masm-> pextrq(Address(to, pos), xmm_result0, 0); | ||||
| 4850 | __masm-> psrldq(xmm_result0, 8); | ||||
| 4851 | __masm-> addptr(pos, 8); | ||||
| 4852 | __masm-> BIND(L_processTail_4_extr[k])bind(L_processTail_4_extr[k]); masm-> block_comment("L_processTail_4_extr[k]" ":"); | ||||
| 4853 | __masm-> testptr(len_reg, 4); | ||||
| 4854 | __masm-> jcc(Assembler::zero, L_processTail_2_extr[k]); | ||||
| 4855 | __masm-> pextrd(Address(to, pos), xmm_result0, 0); | ||||
| 4856 | __masm-> psrldq(xmm_result0, 4); | ||||
| 4857 | __masm-> addptr(pos, 4); | ||||
| 4858 | __masm-> BIND(L_processTail_2_extr[k])bind(L_processTail_2_extr[k]); masm-> block_comment("L_processTail_2_extr[k]" ":"); | ||||
| 4859 | __masm-> testptr(len_reg, 2); | ||||
| 4860 | __masm-> jcc(Assembler::zero, L_processTail_1_extr[k]); | ||||
| 4861 | __masm-> pextrw(Address(to, pos), xmm_result0, 0); | ||||
| 4862 | __masm-> psrldq(xmm_result0, 2); | ||||
| 4863 | __masm-> addptr(pos, 2); | ||||
| 4864 | __masm-> BIND(L_processTail_1_extr[k])bind(L_processTail_1_extr[k]); masm-> block_comment("L_processTail_1_extr[k]" ":"); | ||||
| 4865 | __masm-> testptr(len_reg, 1); | ||||
| 4866 | __masm-> jcc(Assembler::zero, L_processTail_exit_extr[k]); | ||||
| 4867 | __masm-> pextrb(Address(to, pos), xmm_result0, 0); | ||||
| 4868 | |||||
| 4869 | __masm-> BIND(L_processTail_exit_extr[k])bind(L_processTail_exit_extr[k]); masm-> block_comment("L_processTail_exit_extr[k]" ":"); | ||||
| 4870 | __masm-> movl(Address(used_addr, 0), len_reg); | ||||
| 4871 | __masm-> jmp(L_exit); | ||||
| 4872 | |||||
| 4873 | } | ||||
| 4874 | |||||
| 4875 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 4876 | __masm-> pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back. | ||||
| 4877 | __masm-> movdqu(Address(counter, 0), xmm_curr_counter); //save counter back | ||||
| 4878 | __masm-> pop(rbx); // pop the saved RBX. | ||||
| 4879 | #ifdef _WIN64 | ||||
| 4880 | __masm-> movl(rax, len_mem); | ||||
| 4881 | __masm-> movptr(r13, Address(rsp, saved_r13_offset * wordSize)); | ||||
| 4882 | __masm-> movptr(r14, Address(rsp, saved_r14_offset * wordSize)); | ||||
| 4883 | __masm-> addptr(rsp, 2 * wordSize); | ||||
| 4884 | #else | ||||
| 4885 | __masm-> pop(rax); // return 'len' | ||||
| 4886 | #endif | ||||
| 4887 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 4888 | __masm-> ret(0); | ||||
| 4889 | return start; | ||||
| 4890 | } | ||||
| 4891 | |||||
| 4892 | void roundDec(XMMRegister xmm_reg) { | ||||
| 4893 | __masm-> vaesdec(xmm1, xmm1, xmm_reg, Assembler::AVX_512bit); | ||||
| 4894 | __masm-> vaesdec(xmm2, xmm2, xmm_reg, Assembler::AVX_512bit); | ||||
| 4895 | __masm-> vaesdec(xmm3, xmm3, xmm_reg, Assembler::AVX_512bit); | ||||
| 4896 | __masm-> vaesdec(xmm4, xmm4, xmm_reg, Assembler::AVX_512bit); | ||||
| 4897 | __masm-> vaesdec(xmm5, xmm5, xmm_reg, Assembler::AVX_512bit); | ||||
| 4898 | __masm-> vaesdec(xmm6, xmm6, xmm_reg, Assembler::AVX_512bit); | ||||
| 4899 | __masm-> vaesdec(xmm7, xmm7, xmm_reg, Assembler::AVX_512bit); | ||||
| 4900 | __masm-> vaesdec(xmm8, xmm8, xmm_reg, Assembler::AVX_512bit); | ||||
| 4901 | } | ||||
| 4902 | |||||
| 4903 | void roundDeclast(XMMRegister xmm_reg) { | ||||
| 4904 | __masm-> vaesdeclast(xmm1, xmm1, xmm_reg, Assembler::AVX_512bit); | ||||
| 4905 | __masm-> vaesdeclast(xmm2, xmm2, xmm_reg, Assembler::AVX_512bit); | ||||
| 4906 | __masm-> vaesdeclast(xmm3, xmm3, xmm_reg, Assembler::AVX_512bit); | ||||
| 4907 | __masm-> vaesdeclast(xmm4, xmm4, xmm_reg, Assembler::AVX_512bit); | ||||
| 4908 | __masm-> vaesdeclast(xmm5, xmm5, xmm_reg, Assembler::AVX_512bit); | ||||
| 4909 | __masm-> vaesdeclast(xmm6, xmm6, xmm_reg, Assembler::AVX_512bit); | ||||
| 4910 | __masm-> vaesdeclast(xmm7, xmm7, xmm_reg, Assembler::AVX_512bit); | ||||
| 4911 | __masm-> vaesdeclast(xmm8, xmm8, xmm_reg, Assembler::AVX_512bit); | ||||
| 4912 | } | ||||
| 4913 | |||||
| 4914 | void ev_load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask = NULL__null) { | ||||
| 4915 | __masm-> movdqu(xmmdst, Address(key, offset)); | ||||
| 4916 | if (xmm_shuf_mask != NULL__null) { | ||||
| 4917 | __masm-> pshufb(xmmdst, xmm_shuf_mask); | ||||
| 4918 | } else { | ||||
| 4919 | __masm-> pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 4920 | } | ||||
| 4921 | __masm-> evshufi64x2(xmmdst, xmmdst, xmmdst, 0x0, Assembler::AVX_512bit); | ||||
| 4922 | |||||
| 4923 | } | ||||
| 4924 | |||||
| 4925 | address generate_cipherBlockChaining_decryptVectorAESCrypt() { | ||||
| 4926 | assert(VM_Version::supports_avx512_vaes(), "need AES instructions and misaligned SSE support")do { if (!(VM_Version::supports_avx512_vaes())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 4926, "assert(" "VM_Version::supports_avx512_vaes()" ") failed" , "need AES instructions and misaligned SSE support"); ::breakpoint (); } } while (0); | ||||
| 4927 | __masm-> align(CodeEntryAlignment); | ||||
| 4928 | StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt"); | ||||
| 4929 | address start = __masm-> pc(); | ||||
| 4930 | |||||
| 4931 | const Register from = c_rarg0; // source array address | ||||
| 4932 | const Register to = c_rarg1; // destination array address | ||||
| 4933 | const Register key = c_rarg2; // key array address | ||||
| 4934 | const Register rvec = c_rarg3; // r byte array initialized from initvector array address | ||||
| 4935 | // and left with the results of the last encryption block | ||||
| 4936 | #ifndef _WIN64 | ||||
| 4937 | const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) | ||||
| 4938 | #else | ||||
| 4939 | const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 4940 | const Register len_reg = r11; // pick the volatile windows register | ||||
| 4941 | #endif | ||||
| 4942 | |||||
| 4943 | Label Loop, Loop1, L_128, L_256, L_192, KEY_192, KEY_256, Loop2, Lcbc_dec_rem_loop, | ||||
| 4944 | Lcbc_dec_rem_last, Lcbc_dec_ret, Lcbc_dec_rem, Lcbc_exit; | ||||
| 4945 | |||||
| 4946 | __masm-> enter(); | ||||
| 4947 | |||||
| 4948 | #ifdef _WIN64 | ||||
| 4949 | // on win64, fill len_reg from stack position | ||||
| 4950 | __masm-> movl(len_reg, len_mem); | ||||
| 4951 | #else | ||||
| 4952 | __masm-> push(len_reg); // Save | ||||
| 4953 | #endif | ||||
| 4954 | __masm-> push(rbx); | ||||
| 4955 | __masm-> vzeroupper(); | ||||
| 4956 | |||||
| 4957 | // Temporary variable declaration for swapping key bytes | ||||
| 4958 | const XMMRegister xmm_key_shuf_mask = xmm1; | ||||
| 4959 | __masm-> movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr())); | ||||
| 4960 | |||||
| 4961 | // Calculate number of rounds from key size: 44 for 10-rounds, 52 for 12-rounds, 60 for 14-rounds | ||||
| 4962 | const Register rounds = rbx; | ||||
| 4963 | __masm-> movl(rounds, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); | ||||
| 4964 | |||||
| 4965 | const XMMRegister IV = xmm0; | ||||
| 4966 | // Load IV and broadcast value to 512-bits | ||||
| 4967 | __masm-> evbroadcasti64x2(IV, Address(rvec, 0), Assembler::AVX_512bit); | ||||
| 4968 | |||||
| 4969 | // Temporary variables for storing round keys | ||||
| 4970 | const XMMRegister RK0 = xmm30; | ||||
| 4971 | const XMMRegister RK1 = xmm9; | ||||
| 4972 | const XMMRegister RK2 = xmm18; | ||||
| 4973 | const XMMRegister RK3 = xmm19; | ||||
| 4974 | const XMMRegister RK4 = xmm20; | ||||
| 4975 | const XMMRegister RK5 = xmm21; | ||||
| 4976 | const XMMRegister RK6 = xmm22; | ||||
| 4977 | const XMMRegister RK7 = xmm23; | ||||
| 4978 | const XMMRegister RK8 = xmm24; | ||||
| 4979 | const XMMRegister RK9 = xmm25; | ||||
| 4980 | const XMMRegister RK10 = xmm26; | ||||
| 4981 | |||||
| 4982 | // Load and shuffle key | ||||
| 4983 | // the java expanded key ordering is rotated one position from what we want | ||||
| 4984 | // so we start from 1*16 here and hit 0*16 last | ||||
| 4985 | ev_load_key(RK1, key, 1 * 16, xmm_key_shuf_mask); | ||||
| 4986 | ev_load_key(RK2, key, 2 * 16, xmm_key_shuf_mask); | ||||
| 4987 | ev_load_key(RK3, key, 3 * 16, xmm_key_shuf_mask); | ||||
| 4988 | ev_load_key(RK4, key, 4 * 16, xmm_key_shuf_mask); | ||||
| 4989 | ev_load_key(RK5, key, 5 * 16, xmm_key_shuf_mask); | ||||
| 4990 | ev_load_key(RK6, key, 6 * 16, xmm_key_shuf_mask); | ||||
| 4991 | ev_load_key(RK7, key, 7 * 16, xmm_key_shuf_mask); | ||||
| 4992 | ev_load_key(RK8, key, 8 * 16, xmm_key_shuf_mask); | ||||
| 4993 | ev_load_key(RK9, key, 9 * 16, xmm_key_shuf_mask); | ||||
| 4994 | ev_load_key(RK10, key, 10 * 16, xmm_key_shuf_mask); | ||||
| 4995 | ev_load_key(RK0, key, 0*16, xmm_key_shuf_mask); | ||||
| 4996 | |||||
| 4997 | // Variables for storing source cipher text | ||||
| 4998 | const XMMRegister S0 = xmm10; | ||||
| 4999 | const XMMRegister S1 = xmm11; | ||||
| 5000 | const XMMRegister S2 = xmm12; | ||||
| 5001 | const XMMRegister S3 = xmm13; | ||||
| 5002 | const XMMRegister S4 = xmm14; | ||||
| 5003 | const XMMRegister S5 = xmm15; | ||||
| 5004 | const XMMRegister S6 = xmm16; | ||||
| 5005 | const XMMRegister S7 = xmm17; | ||||
| 5006 | |||||
| 5007 | // Variables for storing decrypted text | ||||
| 5008 | const XMMRegister B0 = xmm1; | ||||
| 5009 | const XMMRegister B1 = xmm2; | ||||
| 5010 | const XMMRegister B2 = xmm3; | ||||
| 5011 | const XMMRegister B3 = xmm4; | ||||
| 5012 | const XMMRegister B4 = xmm5; | ||||
| 5013 | const XMMRegister B5 = xmm6; | ||||
| 5014 | const XMMRegister B6 = xmm7; | ||||
| 5015 | const XMMRegister B7 = xmm8; | ||||
| 5016 | |||||
| 5017 | __masm-> cmpl(rounds, 44); | ||||
| 5018 | __masm-> jcc(Assembler::greater, KEY_192); | ||||
| 5019 | __masm-> jmp(Loop); | ||||
| 5020 | |||||
| 5021 | __masm-> BIND(KEY_192)bind(KEY_192); masm-> block_comment("KEY_192" ":"); | ||||
| 5022 | const XMMRegister RK11 = xmm27; | ||||
| 5023 | const XMMRegister RK12 = xmm28; | ||||
| 5024 | ev_load_key(RK11, key, 11*16, xmm_key_shuf_mask); | ||||
| 5025 | ev_load_key(RK12, key, 12*16, xmm_key_shuf_mask); | ||||
| 5026 | |||||
| 5027 | __masm-> cmpl(rounds, 52); | ||||
| 5028 | __masm-> jcc(Assembler::greater, KEY_256); | ||||
| 5029 | __masm-> jmp(Loop); | ||||
| 5030 | |||||
| 5031 | __masm-> BIND(KEY_256)bind(KEY_256); masm-> block_comment("KEY_256" ":"); | ||||
| 5032 | const XMMRegister RK13 = xmm29; | ||||
| 5033 | const XMMRegister RK14 = xmm31; | ||||
| 5034 | ev_load_key(RK13, key, 13*16, xmm_key_shuf_mask); | ||||
| 5035 | ev_load_key(RK14, key, 14*16, xmm_key_shuf_mask); | ||||
| 5036 | |||||
| 5037 | __masm-> BIND(Loop)bind(Loop); masm-> block_comment("Loop" ":"); | ||||
| 5038 | __masm-> cmpl(len_reg, 512); | ||||
| 5039 | __masm-> jcc(Assembler::below, Lcbc_dec_rem); | ||||
| 5040 | __masm-> BIND(Loop1)bind(Loop1); masm-> block_comment("Loop1" ":"); | ||||
| 5041 | __masm-> subl(len_reg, 512); | ||||
| 5042 | __masm-> evmovdquq(S0, Address(from, 0 * 64), Assembler::AVX_512bit); | ||||
| 5043 | __masm-> evmovdquq(S1, Address(from, 1 * 64), Assembler::AVX_512bit); | ||||
| 5044 | __masm-> evmovdquq(S2, Address(from, 2 * 64), Assembler::AVX_512bit); | ||||
| 5045 | __masm-> evmovdquq(S3, Address(from, 3 * 64), Assembler::AVX_512bit); | ||||
| 5046 | __masm-> evmovdquq(S4, Address(from, 4 * 64), Assembler::AVX_512bit); | ||||
| 5047 | __masm-> evmovdquq(S5, Address(from, 5 * 64), Assembler::AVX_512bit); | ||||
| 5048 | __masm-> evmovdquq(S6, Address(from, 6 * 64), Assembler::AVX_512bit); | ||||
| 5049 | __masm-> evmovdquq(S7, Address(from, 7 * 64), Assembler::AVX_512bit); | ||||
| 5050 | __masm-> leaq(from, Address(from, 8 * 64)); | ||||
| 5051 | |||||
| 5052 | __masm-> evpxorq(B0, S0, RK1, Assembler::AVX_512bit); | ||||
| 5053 | __masm-> evpxorq(B1, S1, RK1, Assembler::AVX_512bit); | ||||
| 5054 | __masm-> evpxorq(B2, S2, RK1, Assembler::AVX_512bit); | ||||
| 5055 | __masm-> evpxorq(B3, S3, RK1, Assembler::AVX_512bit); | ||||
| 5056 | __masm-> evpxorq(B4, S4, RK1, Assembler::AVX_512bit); | ||||
| 5057 | __masm-> evpxorq(B5, S5, RK1, Assembler::AVX_512bit); | ||||
| 5058 | __masm-> evpxorq(B6, S6, RK1, Assembler::AVX_512bit); | ||||
| 5059 | __masm-> evpxorq(B7, S7, RK1, Assembler::AVX_512bit); | ||||
| 5060 | |||||
| 5061 | __masm-> evalignq(IV, S0, IV, 0x06); | ||||
| 5062 | __masm-> evalignq(S0, S1, S0, 0x06); | ||||
| 5063 | __masm-> evalignq(S1, S2, S1, 0x06); | ||||
| 5064 | __masm-> evalignq(S2, S3, S2, 0x06); | ||||
| 5065 | __masm-> evalignq(S3, S4, S3, 0x06); | ||||
| 5066 | __masm-> evalignq(S4, S5, S4, 0x06); | ||||
| 5067 | __masm-> evalignq(S5, S6, S5, 0x06); | ||||
| 5068 | __masm-> evalignq(S6, S7, S6, 0x06); | ||||
| 5069 | |||||
| 5070 | roundDec(RK2); | ||||
| 5071 | roundDec(RK3); | ||||
| 5072 | roundDec(RK4); | ||||
| 5073 | roundDec(RK5); | ||||
| 5074 | roundDec(RK6); | ||||
| 5075 | roundDec(RK7); | ||||
| 5076 | roundDec(RK8); | ||||
| 5077 | roundDec(RK9); | ||||
| 5078 | roundDec(RK10); | ||||
| 5079 | |||||
| 5080 | __masm-> cmpl(rounds, 44); | ||||
| 5081 | __masm-> jcc(Assembler::belowEqual, L_128); | ||||
| 5082 | roundDec(RK11); | ||||
| 5083 | roundDec(RK12); | ||||
| 5084 | |||||
| 5085 | __masm-> cmpl(rounds, 52); | ||||
| 5086 | __masm-> jcc(Assembler::belowEqual, L_192); | ||||
| 5087 | roundDec(RK13); | ||||
| 5088 | roundDec(RK14); | ||||
| 5089 | |||||
| 5090 | __masm-> BIND(L_256)bind(L_256); masm-> block_comment("L_256" ":"); | ||||
| 5091 | roundDeclast(RK0); | ||||
| 5092 | __masm-> jmp(Loop2); | ||||
| 5093 | |||||
| 5094 | __masm-> BIND(L_128)bind(L_128); masm-> block_comment("L_128" ":"); | ||||
| 5095 | roundDeclast(RK0); | ||||
| 5096 | __masm-> jmp(Loop2); | ||||
| 5097 | |||||
| 5098 | __masm-> BIND(L_192)bind(L_192); masm-> block_comment("L_192" ":"); | ||||
| 5099 | roundDeclast(RK0); | ||||
| 5100 | |||||
| 5101 | __masm-> BIND(Loop2)bind(Loop2); masm-> block_comment("Loop2" ":"); | ||||
| 5102 | __masm-> evpxorq(B0, B0, IV, Assembler::AVX_512bit); | ||||
| 5103 | __masm-> evpxorq(B1, B1, S0, Assembler::AVX_512bit); | ||||
| 5104 | __masm-> evpxorq(B2, B2, S1, Assembler::AVX_512bit); | ||||
| 5105 | __masm-> evpxorq(B3, B3, S2, Assembler::AVX_512bit); | ||||
| 5106 | __masm-> evpxorq(B4, B4, S3, Assembler::AVX_512bit); | ||||
| 5107 | __masm-> evpxorq(B5, B5, S4, Assembler::AVX_512bit); | ||||
| 5108 | __masm-> evpxorq(B6, B6, S5, Assembler::AVX_512bit); | ||||
| 5109 | __masm-> evpxorq(B7, B7, S6, Assembler::AVX_512bit); | ||||
| 5110 | __masm-> evmovdquq(IV, S7, Assembler::AVX_512bit); | ||||
| 5111 | |||||
| 5112 | __masm-> evmovdquq(Address(to, 0 * 64), B0, Assembler::AVX_512bit); | ||||
| 5113 | __masm-> evmovdquq(Address(to, 1 * 64), B1, Assembler::AVX_512bit); | ||||
| 5114 | __masm-> evmovdquq(Address(to, 2 * 64), B2, Assembler::AVX_512bit); | ||||
| 5115 | __masm-> evmovdquq(Address(to, 3 * 64), B3, Assembler::AVX_512bit); | ||||
| 5116 | __masm-> evmovdquq(Address(to, 4 * 64), B4, Assembler::AVX_512bit); | ||||
| 5117 | __masm-> evmovdquq(Address(to, 5 * 64), B5, Assembler::AVX_512bit); | ||||
| 5118 | __masm-> evmovdquq(Address(to, 6 * 64), B6, Assembler::AVX_512bit); | ||||
| 5119 | __masm-> evmovdquq(Address(to, 7 * 64), B7, Assembler::AVX_512bit); | ||||
| 5120 | __masm-> leaq(to, Address(to, 8 * 64)); | ||||
| 5121 | __masm-> jmp(Loop); | ||||
| 5122 | |||||
| 5123 | __masm-> BIND(Lcbc_dec_rem)bind(Lcbc_dec_rem); masm-> block_comment("Lcbc_dec_rem" ":" ); | ||||
| 5124 | __masm-> evshufi64x2(IV, IV, IV, 0x03, Assembler::AVX_512bit); | ||||
| 5125 | |||||
| 5126 | __masm-> BIND(Lcbc_dec_rem_loop)bind(Lcbc_dec_rem_loop); masm-> block_comment("Lcbc_dec_rem_loop" ":"); | ||||
| 5127 | __masm-> subl(len_reg, 16); | ||||
| 5128 | __masm-> jcc(Assembler::carrySet, Lcbc_dec_ret); | ||||
| 5129 | |||||
| 5130 | __masm-> movdqu(S0, Address(from, 0)); | ||||
| 5131 | __masm-> evpxorq(B0, S0, RK1, Assembler::AVX_512bit); | ||||
| 5132 | __masm-> vaesdec(B0, B0, RK2, Assembler::AVX_512bit); | ||||
| 5133 | __masm-> vaesdec(B0, B0, RK3, Assembler::AVX_512bit); | ||||
| 5134 | __masm-> vaesdec(B0, B0, RK4, Assembler::AVX_512bit); | ||||
| 5135 | __masm-> vaesdec(B0, B0, RK5, Assembler::AVX_512bit); | ||||
| 5136 | __masm-> vaesdec(B0, B0, RK6, Assembler::AVX_512bit); | ||||
| 5137 | __masm-> vaesdec(B0, B0, RK7, Assembler::AVX_512bit); | ||||
| 5138 | __masm-> vaesdec(B0, B0, RK8, Assembler::AVX_512bit); | ||||
| 5139 | __masm-> vaesdec(B0, B0, RK9, Assembler::AVX_512bit); | ||||
| 5140 | __masm-> vaesdec(B0, B0, RK10, Assembler::AVX_512bit); | ||||
| 5141 | __masm-> cmpl(rounds, 44); | ||||
| 5142 | __masm-> jcc(Assembler::belowEqual, Lcbc_dec_rem_last); | ||||
| 5143 | |||||
| 5144 | __masm-> vaesdec(B0, B0, RK11, Assembler::AVX_512bit); | ||||
| 5145 | __masm-> vaesdec(B0, B0, RK12, Assembler::AVX_512bit); | ||||
| 5146 | __masm-> cmpl(rounds, 52); | ||||
| 5147 | __masm-> jcc(Assembler::belowEqual, Lcbc_dec_rem_last); | ||||
| 5148 | |||||
| 5149 | __masm-> vaesdec(B0, B0, RK13, Assembler::AVX_512bit); | ||||
| 5150 | __masm-> vaesdec(B0, B0, RK14, Assembler::AVX_512bit); | ||||
| 5151 | |||||
| 5152 | __masm-> BIND(Lcbc_dec_rem_last)bind(Lcbc_dec_rem_last); masm-> block_comment("Lcbc_dec_rem_last" ":"); | ||||
| 5153 | __masm-> vaesdeclast(B0, B0, RK0, Assembler::AVX_512bit); | ||||
| 5154 | |||||
| 5155 | __masm-> evpxorq(B0, B0, IV, Assembler::AVX_512bit); | ||||
| 5156 | __masm-> evmovdquq(IV, S0, Assembler::AVX_512bit); | ||||
| 5157 | __masm-> movdqu(Address(to, 0), B0); | ||||
| 5158 | __masm-> leaq(from, Address(from, 16)); | ||||
| 5159 | __masm-> leaq(to, Address(to, 16)); | ||||
| 5160 | __masm-> jmp(Lcbc_dec_rem_loop); | ||||
| 5161 | |||||
| 5162 | __masm-> BIND(Lcbc_dec_ret)bind(Lcbc_dec_ret); masm-> block_comment("Lcbc_dec_ret" ":" ); | ||||
| 5163 | __masm-> movdqu(Address(rvec, 0), IV); | ||||
| 5164 | |||||
| 5165 | // Zero out the round keys | ||||
| 5166 | __masm-> evpxorq(RK0, RK0, RK0, Assembler::AVX_512bit); | ||||
| 5167 | __masm-> evpxorq(RK1, RK1, RK1, Assembler::AVX_512bit); | ||||
| 5168 | __masm-> evpxorq(RK2, RK2, RK2, Assembler::AVX_512bit); | ||||
| 5169 | __masm-> evpxorq(RK3, RK3, RK3, Assembler::AVX_512bit); | ||||
| 5170 | __masm-> evpxorq(RK4, RK4, RK4, Assembler::AVX_512bit); | ||||
| 5171 | __masm-> evpxorq(RK5, RK5, RK5, Assembler::AVX_512bit); | ||||
| 5172 | __masm-> evpxorq(RK6, RK6, RK6, Assembler::AVX_512bit); | ||||
| 5173 | __masm-> evpxorq(RK7, RK7, RK7, Assembler::AVX_512bit); | ||||
| 5174 | __masm-> evpxorq(RK8, RK8, RK8, Assembler::AVX_512bit); | ||||
| 5175 | __masm-> evpxorq(RK9, RK9, RK9, Assembler::AVX_512bit); | ||||
| 5176 | __masm-> evpxorq(RK10, RK10, RK10, Assembler::AVX_512bit); | ||||
| 5177 | __masm-> cmpl(rounds, 44); | ||||
| 5178 | __masm-> jcc(Assembler::belowEqual, Lcbc_exit); | ||||
| 5179 | __masm-> evpxorq(RK11, RK11, RK11, Assembler::AVX_512bit); | ||||
| 5180 | __masm-> evpxorq(RK12, RK12, RK12, Assembler::AVX_512bit); | ||||
| 5181 | __masm-> cmpl(rounds, 52); | ||||
| 5182 | __masm-> jcc(Assembler::belowEqual, Lcbc_exit); | ||||
| 5183 | __masm-> evpxorq(RK13, RK13, RK13, Assembler::AVX_512bit); | ||||
| 5184 | __masm-> evpxorq(RK14, RK14, RK14, Assembler::AVX_512bit); | ||||
| 5185 | |||||
| 5186 | __masm-> BIND(Lcbc_exit)bind(Lcbc_exit); masm-> block_comment("Lcbc_exit" ":"); | ||||
| 5187 | __masm-> pop(rbx); | ||||
| 5188 | #ifdef _WIN64 | ||||
| 5189 | __masm-> movl(rax, len_mem); | ||||
| 5190 | #else | ||||
| 5191 | __masm-> pop(rax); // return length | ||||
| 5192 | #endif | ||||
| 5193 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 5194 | __masm-> ret(0); | ||||
| 5195 | return start; | ||||
| 5196 | } | ||||
| 5197 | |||||
| 5198 | // Polynomial x^128+x^127+x^126+x^121+1 | ||||
| 5199 | address ghash_polynomial_addr() { | ||||
| 5200 | __masm-> align(CodeEntryAlignment); | ||||
| 5201 | StubCodeMark mark(this, "StubRoutines", "_ghash_poly_addr"); | ||||
| 5202 | address start = __masm-> pc(); | ||||
| 5203 | __masm-> emit_data64(0x0000000000000001, relocInfo::none); | ||||
| 5204 | __masm-> emit_data64(0xc200000000000000, relocInfo::none); | ||||
| 5205 | return start; | ||||
| 5206 | } | ||||
| 5207 | |||||
| 5208 | address ghash_shufflemask_addr() { | ||||
| 5209 | __masm-> align(CodeEntryAlignment); | ||||
| 5210 | StubCodeMark mark(this, "StubRoutines", "_ghash_shuffmask_addr"); | ||||
| 5211 | address start = __masm-> pc(); | ||||
| 5212 | __masm-> emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none); | ||||
| 5213 | __masm-> emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none); | ||||
| 5214 | return start; | ||||
| 5215 | } | ||||
| 5216 | |||||
| 5217 | // Ghash single and multi block operations using AVX instructions | ||||
| 5218 | address generate_avx_ghash_processBlocks() { | ||||
| 5219 | __masm-> align(CodeEntryAlignment); | ||||
| 5220 | |||||
| 5221 | StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks"); | ||||
| 5222 | address start = __masm-> pc(); | ||||
| 5223 | |||||
| 5224 | // arguments | ||||
| 5225 | const Register state = c_rarg0; | ||||
| 5226 | const Register htbl = c_rarg1; | ||||
| 5227 | const Register data = c_rarg2; | ||||
| 5228 | const Register blocks = c_rarg3; | ||||
| 5229 | __masm-> enter(); | ||||
| 5230 | // Save state before entering routine | ||||
| 5231 | __masm-> avx_ghash(state, htbl, data, blocks); | ||||
| 5232 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 5233 | __masm-> ret(0); | ||||
| 5234 | return start; | ||||
| 5235 | } | ||||
| 5236 | |||||
| 5237 | // byte swap x86 long | ||||
| 5238 | address generate_ghash_long_swap_mask() { | ||||
| 5239 | __masm-> align(CodeEntryAlignment); | ||||
| 5240 | StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask"); | ||||
| 5241 | address start = __masm-> pc(); | ||||
| 5242 | __masm-> emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none ); | ||||
| 5243 | __masm-> emit_data64(0x0706050403020100, relocInfo::none ); | ||||
| 5244 | return start; | ||||
| 5245 | } | ||||
| 5246 | |||||
| 5247 | // byte swap x86 byte array | ||||
| 5248 | address generate_ghash_byte_swap_mask() { | ||||
| 5249 | __masm-> align(CodeEntryAlignment); | ||||
| 5250 | StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask"); | ||||
| 5251 | address start = __masm-> pc(); | ||||
| 5252 | __masm-> emit_data64(0x08090a0b0c0d0e0f, relocInfo::none ); | ||||
| 5253 | __masm-> emit_data64(0x0001020304050607, relocInfo::none ); | ||||
| 5254 | return start; | ||||
| 5255 | } | ||||
| 5256 | |||||
| 5257 | /* Single and multi-block ghash operations */ | ||||
| 5258 | address generate_ghash_processBlocks() { | ||||
| 5259 | __masm-> align(CodeEntryAlignment); | ||||
| 5260 | Label L_ghash_loop, L_exit; | ||||
| 5261 | StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks"); | ||||
| 5262 | address start = __masm-> pc(); | ||||
| 5263 | |||||
| 5264 | const Register state = c_rarg0; | ||||
| 5265 | const Register subkeyH = c_rarg1; | ||||
| 5266 | const Register data = c_rarg2; | ||||
| 5267 | const Register blocks = c_rarg3; | ||||
| 5268 | |||||
| 5269 | const XMMRegister xmm_temp0 = xmm0; | ||||
| 5270 | const XMMRegister xmm_temp1 = xmm1; | ||||
| 5271 | const XMMRegister xmm_temp2 = xmm2; | ||||
| 5272 | const XMMRegister xmm_temp3 = xmm3; | ||||
| 5273 | const XMMRegister xmm_temp4 = xmm4; | ||||
| 5274 | const XMMRegister xmm_temp5 = xmm5; | ||||
| 5275 | const XMMRegister xmm_temp6 = xmm6; | ||||
| 5276 | const XMMRegister xmm_temp7 = xmm7; | ||||
| 5277 | const XMMRegister xmm_temp8 = xmm8; | ||||
| 5278 | const XMMRegister xmm_temp9 = xmm9; | ||||
| 5279 | const XMMRegister xmm_temp10 = xmm10; | ||||
| 5280 | |||||
| 5281 | __masm-> enter(); | ||||
| 5282 | |||||
| 5283 | __masm-> movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr())); | ||||
| 5284 | |||||
| 5285 | __masm-> movdqu(xmm_temp0, Address(state, 0)); | ||||
| 5286 | __masm-> pshufb(xmm_temp0, xmm_temp10); | ||||
| 5287 | |||||
| 5288 | |||||
| 5289 | __masm-> BIND(L_ghash_loop)bind(L_ghash_loop); masm-> block_comment("L_ghash_loop" ":" ); | ||||
| 5290 | __masm-> movdqu(xmm_temp2, Address(data, 0)); | ||||
| 5291 | __masm-> pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr())); | ||||
| 5292 | |||||
| 5293 | __masm-> movdqu(xmm_temp1, Address(subkeyH, 0)); | ||||
| 5294 | __masm-> pshufb(xmm_temp1, xmm_temp10); | ||||
| 5295 | |||||
| 5296 | __masm-> pxor(xmm_temp0, xmm_temp2); | ||||
| 5297 | |||||
| 5298 | // | ||||
| 5299 | // Multiply with the hash key | ||||
| 5300 | // | ||||
| 5301 | __masm-> movdqu(xmm_temp3, xmm_temp0); | ||||
| 5302 | __masm-> pclmulqdq(xmm_temp3, xmm_temp1, 0); // xmm3 holds a0*b0 | ||||
| 5303 | __masm-> movdqu(xmm_temp4, xmm_temp0); | ||||
| 5304 | __masm-> pclmulqdq(xmm_temp4, xmm_temp1, 16); // xmm4 holds a0*b1 | ||||
| 5305 | |||||
| 5306 | __masm-> movdqu(xmm_temp5, xmm_temp0); | ||||
| 5307 | __masm-> pclmulqdq(xmm_temp5, xmm_temp1, 1); // xmm5 holds a1*b0 | ||||
| 5308 | __masm-> movdqu(xmm_temp6, xmm_temp0); | ||||
| 5309 | __masm-> pclmulqdq(xmm_temp6, xmm_temp1, 17); // xmm6 holds a1*b1 | ||||
| 5310 | |||||
| 5311 | __masm-> pxor(xmm_temp4, xmm_temp5); // xmm4 holds a0*b1 + a1*b0 | ||||
| 5312 | |||||
| 5313 | __masm-> movdqu(xmm_temp5, xmm_temp4); // move the contents of xmm4 to xmm5 | ||||
| 5314 | __masm-> psrldq(xmm_temp4, 8); // shift by xmm4 64 bits to the right | ||||
| 5315 | __masm-> pslldq(xmm_temp5, 8); // shift by xmm5 64 bits to the left | ||||
| 5316 | __masm-> pxor(xmm_temp3, xmm_temp5); | ||||
| 5317 | __masm-> pxor(xmm_temp6, xmm_temp4); // Register pair <xmm6:xmm3> holds the result | ||||
| 5318 | // of the carry-less multiplication of | ||||
| 5319 | // xmm0 by xmm1. | ||||
| 5320 | |||||
| 5321 | // We shift the result of the multiplication by one bit position | ||||
| 5322 | // to the left to cope for the fact that the bits are reversed. | ||||
| 5323 | __masm-> movdqu(xmm_temp7, xmm_temp3); | ||||
| 5324 | __masm-> movdqu(xmm_temp8, xmm_temp6); | ||||
| 5325 | __masm-> pslld(xmm_temp3, 1); | ||||
| 5326 | __masm-> pslld(xmm_temp6, 1); | ||||
| 5327 | __masm-> psrld(xmm_temp7, 31); | ||||
| 5328 | __masm-> psrld(xmm_temp8, 31); | ||||
| 5329 | __masm-> movdqu(xmm_temp9, xmm_temp7); | ||||
| 5330 | __masm-> pslldq(xmm_temp8, 4); | ||||
| 5331 | __masm-> pslldq(xmm_temp7, 4); | ||||
| 5332 | __masm-> psrldq(xmm_temp9, 12); | ||||
| 5333 | __masm-> por(xmm_temp3, xmm_temp7); | ||||
| 5334 | __masm-> por(xmm_temp6, xmm_temp8); | ||||
| 5335 | __masm-> por(xmm_temp6, xmm_temp9); | ||||
| 5336 | |||||
| 5337 | // | ||||
| 5338 | // First phase of the reduction | ||||
| 5339 | // | ||||
| 5340 | // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts | ||||
| 5341 | // independently. | ||||
| 5342 | __masm-> movdqu(xmm_temp7, xmm_temp3); | ||||
| 5343 | __masm-> movdqu(xmm_temp8, xmm_temp3); | ||||
| 5344 | __masm-> movdqu(xmm_temp9, xmm_temp3); | ||||
| 5345 | __masm-> pslld(xmm_temp7, 31); // packed right shift shifting << 31 | ||||
| 5346 | __masm-> pslld(xmm_temp8, 30); // packed right shift shifting << 30 | ||||
| 5347 | __masm-> pslld(xmm_temp9, 25); // packed right shift shifting << 25 | ||||
| 5348 | __masm-> pxor(xmm_temp7, xmm_temp8); // xor the shifted versions | ||||
| 5349 | __masm-> pxor(xmm_temp7, xmm_temp9); | ||||
| 5350 | __masm-> movdqu(xmm_temp8, xmm_temp7); | ||||
| 5351 | __masm-> pslldq(xmm_temp7, 12); | ||||
| 5352 | __masm-> psrldq(xmm_temp8, 4); | ||||
| 5353 | __masm-> pxor(xmm_temp3, xmm_temp7); // first phase of the reduction complete | ||||
| 5354 | |||||
| 5355 | // | ||||
| 5356 | // Second phase of the reduction | ||||
| 5357 | // | ||||
| 5358 | // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these | ||||
| 5359 | // shift operations. | ||||
| 5360 | __masm-> movdqu(xmm_temp2, xmm_temp3); | ||||
| 5361 | __masm-> movdqu(xmm_temp4, xmm_temp3); | ||||
| 5362 | __masm-> movdqu(xmm_temp5, xmm_temp3); | ||||
| 5363 | __masm-> psrld(xmm_temp2, 1); // packed left shifting >> 1 | ||||
| 5364 | __masm-> psrld(xmm_temp4, 2); // packed left shifting >> 2 | ||||
| 5365 | __masm-> psrld(xmm_temp5, 7); // packed left shifting >> 7 | ||||
| 5366 | __masm-> pxor(xmm_temp2, xmm_temp4); // xor the shifted versions | ||||
| 5367 | __masm-> pxor(xmm_temp2, xmm_temp5); | ||||
| 5368 | __masm-> pxor(xmm_temp2, xmm_temp8); | ||||
| 5369 | __masm-> pxor(xmm_temp3, xmm_temp2); | ||||
| 5370 | __masm-> pxor(xmm_temp6, xmm_temp3); // the result is in xmm6 | ||||
| 5371 | |||||
| 5372 | __masm-> decrement(blocks); | ||||
| 5373 | __masm-> jcc(Assembler::zero, L_exit); | ||||
| 5374 | __masm-> movdqu(xmm_temp0, xmm_temp6); | ||||
| 5375 | __masm-> addptr(data, 16); | ||||
| 5376 | __masm-> jmp(L_ghash_loop); | ||||
| 5377 | |||||
| 5378 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 5379 | __masm-> pshufb(xmm_temp6, xmm_temp10); // Byte swap 16-byte result | ||||
| 5380 | __masm-> movdqu(Address(state, 0), xmm_temp6); // store the result | ||||
| 5381 | __masm-> leave(); | ||||
| 5382 | __masm-> ret(0); | ||||
| 5383 | return start; | ||||
| 5384 | } | ||||
| 5385 | |||||
| 5386 | address base64_shuffle_addr() | ||||
| 5387 | { | ||||
| 5388 | __masm-> align64(); | ||||
| 5389 | StubCodeMark mark(this, "StubRoutines", "shuffle_base64"); | ||||
| 5390 | address start = __masm-> pc(); | ||||
| 5391 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5392, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5392 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5392, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5393 | __masm-> emit_data64(0x0405030401020001, relocInfo::none); | ||||
| 5394 | __masm-> emit_data64(0x0a0b090a07080607, relocInfo::none); | ||||
| 5395 | __masm-> emit_data64(0x10110f100d0e0c0d, relocInfo::none); | ||||
| 5396 | __masm-> emit_data64(0x1617151613141213, relocInfo::none); | ||||
| 5397 | __masm-> emit_data64(0x1c1d1b1c191a1819, relocInfo::none); | ||||
| 5398 | __masm-> emit_data64(0x222321221f201e1f, relocInfo::none); | ||||
| 5399 | __masm-> emit_data64(0x2829272825262425, relocInfo::none); | ||||
| 5400 | __masm-> emit_data64(0x2e2f2d2e2b2c2a2b, relocInfo::none); | ||||
| 5401 | return start; | ||||
| 5402 | } | ||||
| 5403 | |||||
| 5404 | address base64_avx2_shuffle_addr() | ||||
| 5405 | { | ||||
| 5406 | __masm-> align32(); | ||||
| 5407 | StubCodeMark mark(this, "StubRoutines", "avx2_shuffle_base64"); | ||||
| 5408 | address start = __masm-> pc(); | ||||
| 5409 | __masm-> emit_data64(0x0809070805060405, relocInfo::none); | ||||
| 5410 | __masm-> emit_data64(0x0e0f0d0e0b0c0a0b, relocInfo::none); | ||||
| 5411 | __masm-> emit_data64(0x0405030401020001, relocInfo::none); | ||||
| 5412 | __masm-> emit_data64(0x0a0b090a07080607, relocInfo::none); | ||||
| 5413 | return start; | ||||
| 5414 | } | ||||
| 5415 | |||||
| 5416 | address base64_avx2_input_mask_addr() | ||||
| 5417 | { | ||||
| 5418 | __masm-> align32(); | ||||
| 5419 | StubCodeMark mark(this, "StubRoutines", "avx2_input_mask_base64"); | ||||
| 5420 | address start = __masm-> pc(); | ||||
| 5421 | __masm-> emit_data64(0x8000000000000000, relocInfo::none); | ||||
| 5422 | __masm-> emit_data64(0x8000000080000000, relocInfo::none); | ||||
| 5423 | __masm-> emit_data64(0x8000000080000000, relocInfo::none); | ||||
| 5424 | __masm-> emit_data64(0x8000000080000000, relocInfo::none); | ||||
| 5425 | return start; | ||||
| 5426 | } | ||||
| 5427 | |||||
| 5428 | address base64_avx2_lut_addr() | ||||
| 5429 | { | ||||
| 5430 | __masm-> align32(); | ||||
| 5431 | StubCodeMark mark(this, "StubRoutines", "avx2_lut_base64"); | ||||
| 5432 | address start = __masm-> pc(); | ||||
| 5433 | __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); | ||||
| 5434 | __masm-> emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); | ||||
| 5435 | __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); | ||||
| 5436 | __masm-> emit_data64(0x0000f0edfcfcfcfc, relocInfo::none); | ||||
| 5437 | |||||
| 5438 | // URL LUT | ||||
| 5439 | __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); | ||||
| 5440 | __masm-> emit_data64(0x000020effcfcfcfc, relocInfo::none); | ||||
| 5441 | __masm-> emit_data64(0xfcfcfcfcfcfc4741, relocInfo::none); | ||||
| 5442 | __masm-> emit_data64(0x000020effcfcfcfc, relocInfo::none); | ||||
| 5443 | return start; | ||||
| 5444 | } | ||||
| 5445 | |||||
| 5446 | address base64_encoding_table_addr() | ||||
| 5447 | { | ||||
| 5448 | __masm-> align64(); | ||||
| 5449 | StubCodeMark mark(this, "StubRoutines", "encoding_table_base64"); | ||||
| 5450 | address start = __masm-> pc(); | ||||
| 5451 | assert(((unsigned long long)start & 0x3f) == 0, "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5451, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5452 | __masm-> emit_data64(0x4847464544434241, relocInfo::none); | ||||
| 5453 | __masm-> emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); | ||||
| 5454 | __masm-> emit_data64(0x5857565554535251, relocInfo::none); | ||||
| 5455 | __masm-> emit_data64(0x6665646362615a59, relocInfo::none); | ||||
| 5456 | __masm-> emit_data64(0x6e6d6c6b6a696867, relocInfo::none); | ||||
| 5457 | __masm-> emit_data64(0x767574737271706f, relocInfo::none); | ||||
| 5458 | __masm-> emit_data64(0x333231307a797877, relocInfo::none); | ||||
| 5459 | __masm-> emit_data64(0x2f2b393837363534, relocInfo::none); | ||||
| 5460 | |||||
| 5461 | // URL table | ||||
| 5462 | __masm-> emit_data64(0x4847464544434241, relocInfo::none); | ||||
| 5463 | __masm-> emit_data64(0x504f4e4d4c4b4a49, relocInfo::none); | ||||
| 5464 | __masm-> emit_data64(0x5857565554535251, relocInfo::none); | ||||
| 5465 | __masm-> emit_data64(0x6665646362615a59, relocInfo::none); | ||||
| 5466 | __masm-> emit_data64(0x6e6d6c6b6a696867, relocInfo::none); | ||||
| 5467 | __masm-> emit_data64(0x767574737271706f, relocInfo::none); | ||||
| 5468 | __masm-> emit_data64(0x333231307a797877, relocInfo::none); | ||||
| 5469 | __masm-> emit_data64(0x5f2d393837363534, relocInfo::none); | ||||
| 5470 | return start; | ||||
| 5471 | } | ||||
| 5472 | |||||
| 5473 | // Code for generating Base64 encoding. | ||||
| 5474 | // Intrinsic function prototype in Base64.java: | ||||
| 5475 | // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, | ||||
| 5476 | // boolean isURL) { | ||||
| 5477 | address generate_base64_encodeBlock() | ||||
| 5478 | { | ||||
| 5479 | __masm-> align(CodeEntryAlignment); | ||||
| 5480 | StubCodeMark mark(this, "StubRoutines", "implEncode"); | ||||
| 5481 | address start = __masm-> pc(); | ||||
| 5482 | __masm-> enter(); | ||||
| 5483 | |||||
| 5484 | // Save callee-saved registers before using them | ||||
| 5485 | __masm-> push(r12); | ||||
| 5486 | __masm-> push(r13); | ||||
| 5487 | __masm-> push(r14); | ||||
| 5488 | __masm-> push(r15); | ||||
| 5489 | |||||
| 5490 | // arguments | ||||
| 5491 | const Register source = c_rarg0; // Source Array | ||||
| 5492 | const Register start_offset = c_rarg1; // start offset | ||||
| 5493 | const Register end_offset = c_rarg2; // end offset | ||||
| 5494 | const Register dest = c_rarg3; // destination array | ||||
| 5495 | |||||
| 5496 | #ifndef _WIN64 | ||||
| 5497 | const Register dp = c_rarg4; // Position for writing to dest array | ||||
| 5498 | const Register isURL = c_rarg5; // Base64 or URL character set | ||||
| 5499 | #else | ||||
| 5500 | const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 5501 | const Address isURL_mem(rbp, 7 * wordSize); | ||||
| 5502 | const Register isURL = r10; // pick the volatile windows register | ||||
| 5503 | const Register dp = r12; | ||||
| 5504 | __masm-> movl(dp, dp_mem); | ||||
| 5505 | __masm-> movl(isURL, isURL_mem); | ||||
| 5506 | #endif | ||||
| 5507 | |||||
| 5508 | const Register length = r14; | ||||
| 5509 | const Register encode_table = r13; | ||||
| 5510 | Label L_process3, L_exit, L_processdata, L_vbmiLoop, L_not512, L_32byteLoop; | ||||
| 5511 | |||||
| 5512 | // calculate length from offsets | ||||
| 5513 | __masm-> movl(length, end_offset); | ||||
| 5514 | __masm-> subl(length, start_offset); | ||||
| 5515 | __masm-> cmpl(length, 0); | ||||
| 5516 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 5517 | |||||
| 5518 | // Code for 512-bit VBMI encoding. Encodes 48 input bytes into 64 | ||||
| 5519 | // output bytes. We read 64 input bytes and ignore the last 16, so be | ||||
| 5520 | // sure not to read past the end of the input buffer. | ||||
| 5521 | if (VM_Version::supports_avx512_vbmi()) { | ||||
| 5522 | __masm-> cmpl(length, 64); // Do not overrun input buffer. | ||||
| 5523 | __masm-> jcc(Assembler::below, L_not512); | ||||
| 5524 | |||||
| 5525 | __masm-> shll(isURL, 6); // index into decode table based on isURL | ||||
| 5526 | __masm-> lea(encode_table, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); | ||||
| 5527 | __masm-> addptr(encode_table, isURL); | ||||
| 5528 | __masm-> shrl(isURL, 6); // restore isURL | ||||
| 5529 | |||||
| 5530 | __masm-> mov64(rax, 0x3036242a1016040aull); // Shifts | ||||
| 5531 | __masm-> evmovdquq(xmm3, ExternalAddress(StubRoutines::x86::base64_shuffle_addr()), Assembler::AVX_512bit, r15); | ||||
| 5532 | __masm-> evmovdquq(xmm2, Address(encode_table, 0), Assembler::AVX_512bit); | ||||
| 5533 | __masm-> evpbroadcastq(xmm1, rax, Assembler::AVX_512bit); | ||||
| 5534 | |||||
| 5535 | __masm-> align32(); | ||||
| 5536 | __masm-> BIND(L_vbmiLoop)bind(L_vbmiLoop); masm-> block_comment("L_vbmiLoop" ":"); | ||||
| 5537 | |||||
| 5538 | __masm-> vpermb(xmm0, xmm3, Address(source, start_offset), Assembler::AVX_512bit); | ||||
| 5539 | __masm-> subl(length, 48); | ||||
| 5540 | |||||
| 5541 | // Put the input bytes into the proper lanes for writing, then | ||||
| 5542 | // encode them. | ||||
| 5543 | __masm-> evpmultishiftqb(xmm0, xmm1, xmm0, Assembler::AVX_512bit); | ||||
| 5544 | __masm-> vpermb(xmm0, xmm0, xmm2, Assembler::AVX_512bit); | ||||
| 5545 | |||||
| 5546 | // Write to destination | ||||
| 5547 | __masm-> evmovdquq(Address(dest, dp), xmm0, Assembler::AVX_512bit); | ||||
| 5548 | |||||
| 5549 | __masm-> addptr(dest, 64); | ||||
| 5550 | __masm-> addptr(source, 48); | ||||
| 5551 | __masm-> cmpl(length, 64); | ||||
| 5552 | __masm-> jcc(Assembler::aboveEqual, L_vbmiLoop); | ||||
| 5553 | |||||
| 5554 | __masm-> vzeroupper(); | ||||
| 5555 | } | ||||
| 5556 | |||||
| 5557 | __masm-> BIND(L_not512)bind(L_not512); masm-> block_comment("L_not512" ":"); | ||||
| 5558 | if (VM_Version::supports_avx2() | ||||
| 5559 | && VM_Version::supports_avx512vlbw()) { | ||||
| 5560 | /* | ||||
| 5561 | ** This AVX2 encoder is based off the paper at: | ||||
| 5562 | ** https://dl.acm.org/doi/10.1145/3132709 | ||||
| 5563 | ** | ||||
| 5564 | ** We use AVX2 SIMD instructions to encode 24 bytes into 32 | ||||
| 5565 | ** output bytes. | ||||
| 5566 | ** | ||||
| 5567 | */ | ||||
| 5568 | // Lengths under 32 bytes are done with scalar routine | ||||
| 5569 | __masm-> cmpl(length, 31); | ||||
| 5570 | __masm-> jcc(Assembler::belowEqual, L_process3); | ||||
| 5571 | |||||
| 5572 | // Set up supporting constant table data | ||||
| 5573 | __masm-> vmovdqu(xmm9, ExternalAddress(StubRoutines::x86::base64_avx2_shuffle_addr()), rax); | ||||
| 5574 | // 6-bit mask for 2nd and 4th (and multiples) 6-bit values | ||||
| 5575 | __masm-> movl(rax, 0x0fc0fc00); | ||||
| 5576 | __masm-> vmovdqu(xmm1, ExternalAddress(StubRoutines::x86::base64_avx2_input_mask_addr()), rax); | ||||
| 5577 | __masm-> evpbroadcastd(xmm8, rax, Assembler::AVX_256bit); | ||||
| 5578 | |||||
| 5579 | // Multiplication constant for "shifting" right by 6 and 10 | ||||
| 5580 | // bits | ||||
| 5581 | __masm-> movl(rax, 0x04000040); | ||||
| 5582 | |||||
| 5583 | __masm-> subl(length, 24); | ||||
| 5584 | __masm-> evpbroadcastd(xmm7, rax, Assembler::AVX_256bit); | ||||
| 5585 | |||||
| 5586 | // For the first load, we mask off reading of the first 4 | ||||
| 5587 | // bytes into the register. This is so we can get 4 3-byte | ||||
| 5588 | // chunks into each lane of the register, avoiding having to | ||||
| 5589 | // handle end conditions. We then shuffle these bytes into a | ||||
| 5590 | // specific order so that manipulation is easier. | ||||
| 5591 | // | ||||
| 5592 | // The initial read loads the XMM register like this: | ||||
| 5593 | // | ||||
| 5594 | // Lower 128-bit lane: | ||||
| 5595 | // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ | ||||
| 5596 | // | XX | XX | XX | XX | A0 | A1 | A2 | B0 | B1 | B2 | C0 | C1 | ||||
| 5597 | // | C2 | D0 | D1 | D2 | | ||||
| 5598 | // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ | ||||
| 5599 | // | ||||
| 5600 | // Upper 128-bit lane: | ||||
| 5601 | // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ | ||||
| 5602 | // | E0 | E1 | E2 | F0 | F1 | F2 | G0 | G1 | G2 | H0 | H1 | H2 | ||||
| 5603 | // | XX | XX | XX | XX | | ||||
| 5604 | // +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+ | ||||
| 5605 | // | ||||
| 5606 | // Where A0 is the first input byte, B0 is the fourth, etc. | ||||
| 5607 | // The alphabetical significance denotes the 3 bytes to be | ||||
| 5608 | // consumed and encoded into 4 bytes. | ||||
| 5609 | // | ||||
| 5610 | // We then shuffle the register so each 32-bit word contains | ||||
| 5611 | // the sequence: | ||||
| 5612 | // A1 A0 A2 A1, B1, B0, B2, B1, etc. | ||||
| 5613 | // Each of these byte sequences are then manipulated into 4 | ||||
| 5614 | // 6-bit values ready for encoding. | ||||
| 5615 | // | ||||
| 5616 | // If we focus on one set of 3-byte chunks, changing the | ||||
| 5617 | // nomenclature such that A0 => a, A1 => b, and A2 => c, we | ||||
| 5618 | // shuffle such that each 24-bit chunk contains: | ||||
| 5619 | // | ||||
| 5620 | // b7 b6 b5 b4 b3 b2 b1 b0 | a7 a6 a5 a4 a3 a2 a1 a0 | c7 c6 | ||||
| 5621 | // c5 c4 c3 c2 c1 c0 | b7 b6 b5 b4 b3 b2 b1 b0 | ||||
| 5622 | // Explain this step. | ||||
| 5623 | // b3 b2 b1 b0 c5 c4 c3 c2 | c1 c0 d5 d4 d3 d2 d1 d0 | a5 a4 | ||||
| 5624 | // a3 a2 a1 a0 b5 b4 | b3 b2 b1 b0 c5 c4 c3 c2 | ||||
| 5625 | // | ||||
| 5626 | // W first and off all but bits 4-9 and 16-21 (c5..c0 and | ||||
| 5627 | // a5..a0) and shift them using a vector multiplication | ||||
| 5628 | // operation (vpmulhuw) which effectively shifts c right by 6 | ||||
| 5629 | // bits and a right by 10 bits. We similarly mask bits 10-15 | ||||
| 5630 | // (d5..d0) and 22-27 (b5..b0) and shift them left by 8 and 4 | ||||
| 5631 | // bits respecively. This is done using vpmullw. We end up | ||||
| 5632 | // with 4 6-bit values, thus splitting the 3 input bytes, | ||||
| 5633 | // ready for encoding: | ||||
| 5634 | // 0 0 d5..d0 0 0 c5..c0 0 0 b5..b0 0 0 a5..a0 | ||||
| 5635 | // | ||||
| 5636 | // For translation, we recognize that there are 5 distinct | ||||
| 5637 | // ranges of legal Base64 characters as below: | ||||
| 5638 | // | ||||
| 5639 | // +-------------+-------------+------------+ | ||||
| 5640 | // | 6-bit value | ASCII range | offset | | ||||
| 5641 | // +-------------+-------------+------------+ | ||||
| 5642 | // | 0..25 | A..Z | 65 | | ||||
| 5643 | // | 26..51 | a..z | 71 | | ||||
| 5644 | // | 52..61 | 0..9 | -4 | | ||||
| 5645 | // | 62 | + or - | -19 or -17 | | ||||
| 5646 | // | 63 | / or _ | -16 or 32 | | ||||
| 5647 | // +-------------+-------------+------------+ | ||||
| 5648 | // | ||||
| 5649 | // We note that vpshufb does a parallel lookup in a | ||||
| 5650 | // destination register using the lower 4 bits of bytes from a | ||||
| 5651 | // source register. If we use a saturated subtraction and | ||||
| 5652 | // subtract 51 from each 6-bit value, bytes from [0,51] | ||||
| 5653 | // saturate to 0, and [52,63] map to a range of [1,12]. We | ||||
| 5654 | // distinguish the [0,25] and [26,51] ranges by assigning a | ||||
| 5655 | // value of 13 for all 6-bit values less than 26. We end up | ||||
| 5656 | // with: | ||||
| 5657 | // | ||||
| 5658 | // +-------------+-------------+------------+ | ||||
| 5659 | // | 6-bit value | Reduced | offset | | ||||
| 5660 | // +-------------+-------------+------------+ | ||||
| 5661 | // | 0..25 | 13 | 65 | | ||||
| 5662 | // | 26..51 | 0 | 71 | | ||||
| 5663 | // | 52..61 | 0..9 | -4 | | ||||
| 5664 | // | 62 | 11 | -19 or -17 | | ||||
| 5665 | // | 63 | 12 | -16 or 32 | | ||||
| 5666 | // +-------------+-------------+------------+ | ||||
| 5667 | // | ||||
| 5668 | // We then use a final vpshufb to add the appropriate offset, | ||||
| 5669 | // translating the bytes. | ||||
| 5670 | // | ||||
| 5671 | // Load input bytes - only 28 bytes. Mask the first load to | ||||
| 5672 | // not load into the full register. | ||||
| 5673 | __masm-> vpmaskmovd(xmm1, xmm1, Address(source, start_offset, Address::times_1, -4), Assembler::AVX_256bit); | ||||
| 5674 | |||||
| 5675 | // Move 3-byte chunks of input (12 bytes) into 16 bytes, | ||||
| 5676 | // ordering by: | ||||
| 5677 | // 1, 0, 2, 1; 4, 3, 5, 4; etc. This groups 6-bit chunks | ||||
| 5678 | // for easy masking | ||||
| 5679 | __masm-> vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); | ||||
| 5680 | |||||
| 5681 | __masm-> addl(start_offset, 24); | ||||
| 5682 | |||||
| 5683 | // Load masking register for first and third (and multiples) | ||||
| 5684 | // 6-bit values. | ||||
| 5685 | __masm-> movl(rax, 0x003f03f0); | ||||
| 5686 | __masm-> evpbroadcastd(xmm6, rax, Assembler::AVX_256bit); | ||||
| 5687 | // Multiplication constant for "shifting" left by 4 and 8 bits | ||||
| 5688 | __masm-> movl(rax, 0x01000010); | ||||
| 5689 | __masm-> evpbroadcastd(xmm5, rax, Assembler::AVX_256bit); | ||||
| 5690 | |||||
| 5691 | // Isolate 6-bit chunks of interest | ||||
| 5692 | __masm-> vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); | ||||
| 5693 | |||||
| 5694 | // Load constants for encoding | ||||
| 5695 | __masm-> movl(rax, 0x19191919); | ||||
| 5696 | __masm-> evpbroadcastd(xmm3, rax, Assembler::AVX_256bit); | ||||
| 5697 | __masm-> movl(rax, 0x33333333); | ||||
| 5698 | __masm-> evpbroadcastd(xmm4, rax, Assembler::AVX_256bit); | ||||
| 5699 | |||||
| 5700 | // Shift output bytes 0 and 2 into proper lanes | ||||
| 5701 | __masm-> vpmulhuw(xmm2, xmm0, xmm7, Assembler::AVX_256bit); | ||||
| 5702 | |||||
| 5703 | // Mask and shift output bytes 1 and 3 into proper lanes and | ||||
| 5704 | // combine | ||||
| 5705 | __masm-> vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); | ||||
| 5706 | __masm-> vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); | ||||
| 5707 | __masm-> vpor(xmm0, xmm0, xmm2, Assembler::AVX_256bit); | ||||
| 5708 | |||||
| 5709 | // Find out which are 0..25. This indicates which input | ||||
| 5710 | // values fall in the range of 'A'-'Z', which require an | ||||
| 5711 | // additional offset (see comments above) | ||||
| 5712 | __masm-> vpcmpgtb(xmm2, xmm0, xmm3, Assembler::AVX_256bit); | ||||
| 5713 | __masm-> vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); | ||||
| 5714 | __masm-> vpsubb(xmm1, xmm1, xmm2, Assembler::AVX_256bit); | ||||
| 5715 | |||||
| 5716 | // Load the proper lookup table | ||||
| 5717 | __masm-> lea(r11, ExternalAddress(StubRoutines::x86::base64_avx2_lut_addr())); | ||||
| 5718 | __masm-> movl(r15, isURL); | ||||
| 5719 | __masm-> shll(r15, 5); | ||||
| 5720 | __masm-> vmovdqu(xmm2, Address(r11, r15)); | ||||
| 5721 | |||||
| 5722 | // Shuffle the offsets based on the range calculation done | ||||
| 5723 | // above. This allows us to add the correct offset to the | ||||
| 5724 | // 6-bit value corresponding to the range documented above. | ||||
| 5725 | __masm-> vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); | ||||
| 5726 | __masm-> vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); | ||||
| 5727 | |||||
| 5728 | // Store the encoded bytes | ||||
| 5729 | __masm-> vmovdqu(Address(dest, dp), xmm0); | ||||
| 5730 | __masm-> addl(dp, 32); | ||||
| 5731 | |||||
| 5732 | __masm-> cmpl(length, 31); | ||||
| 5733 | __masm-> jcc(Assembler::belowEqual, L_process3); | ||||
| 5734 | |||||
| 5735 | __masm-> align32(); | ||||
| 5736 | __masm-> BIND(L_32byteLoop)bind(L_32byteLoop); masm-> block_comment("L_32byteLoop" ":" ); | ||||
| 5737 | |||||
| 5738 | // Get next 32 bytes | ||||
| 5739 | __masm-> vmovdqu(xmm1, Address(source, start_offset, Address::times_1, -4)); | ||||
| 5740 | |||||
| 5741 | __masm-> subl(length, 24); | ||||
| 5742 | __masm-> addl(start_offset, 24); | ||||
| 5743 | |||||
| 5744 | // This logic is identical to the above, with only constant | ||||
| 5745 | // register loads removed. Shuffle the input, mask off 6-bit | ||||
| 5746 | // chunks, shift them into place, then add the offset to | ||||
| 5747 | // encode. | ||||
| 5748 | __masm-> vpshufb(xmm1, xmm1, xmm9, Assembler::AVX_256bit); | ||||
| 5749 | |||||
| 5750 | __masm-> vpand(xmm0, xmm8, xmm1, Assembler::AVX_256bit); | ||||
| 5751 | __masm-> vpmulhuw(xmm10, xmm0, xmm7, Assembler::AVX_256bit); | ||||
| 5752 | __masm-> vpand(xmm0, xmm6, xmm1, Assembler::AVX_256bit); | ||||
| 5753 | __masm-> vpmullw(xmm0, xmm5, xmm0, Assembler::AVX_256bit); | ||||
| 5754 | __masm-> vpor(xmm0, xmm0, xmm10, Assembler::AVX_256bit); | ||||
| 5755 | __masm-> vpcmpgtb(xmm10, xmm0, xmm3, Assembler::AVX_256bit); | ||||
| 5756 | __masm-> vpsubusb(xmm1, xmm0, xmm4, Assembler::AVX_256bit); | ||||
| 5757 | __masm-> vpsubb(xmm1, xmm1, xmm10, Assembler::AVX_256bit); | ||||
| 5758 | __masm-> vpshufb(xmm1, xmm2, xmm1, Assembler::AVX_256bit); | ||||
| 5759 | __masm-> vpaddb(xmm0, xmm1, xmm0, Assembler::AVX_256bit); | ||||
| 5760 | |||||
| 5761 | // Store the encoded bytes | ||||
| 5762 | __masm-> vmovdqu(Address(dest, dp), xmm0); | ||||
| 5763 | __masm-> addl(dp, 32); | ||||
| 5764 | |||||
| 5765 | __masm-> cmpl(length, 31); | ||||
| 5766 | __masm-> jcc(Assembler::above, L_32byteLoop); | ||||
| 5767 | |||||
| 5768 | __masm-> BIND(L_process3)bind(L_process3); masm-> block_comment("L_process3" ":"); | ||||
| 5769 | __masm-> vzeroupper(); | ||||
| 5770 | } else { | ||||
| 5771 | __masm-> BIND(L_process3)bind(L_process3); masm-> block_comment("L_process3" ":"); | ||||
| 5772 | } | ||||
| 5773 | |||||
| 5774 | __masm-> cmpl(length, 3); | ||||
| 5775 | __masm-> jcc(Assembler::below, L_exit); | ||||
| 5776 | |||||
| 5777 | // Load the encoding table based on isURL | ||||
| 5778 | __masm-> lea(r11, ExternalAddress(StubRoutines::x86::base64_encoding_table_addr())); | ||||
| 5779 | __masm-> movl(r15, isURL); | ||||
| 5780 | __masm-> shll(r15, 6); | ||||
| 5781 | __masm-> addptr(r11, r15); | ||||
| 5782 | |||||
| 5783 | __masm-> BIND(L_processdata)bind(L_processdata); masm-> block_comment("L_processdata" ":" ); | ||||
| 5784 | |||||
| 5785 | // Load 3 bytes | ||||
| 5786 | __masm-> load_unsigned_byte(r15, Address(source, start_offset)); | ||||
| 5787 | __masm-> load_unsigned_byte(r10, Address(source, start_offset, Address::times_1, 1)); | ||||
| 5788 | __masm-> load_unsigned_byte(r13, Address(source, start_offset, Address::times_1, 2)); | ||||
| 5789 | |||||
| 5790 | // Build a 32-bit word with bytes 1, 2, 0, 1 | ||||
| 5791 | __masm-> movl(rax, r10); | ||||
| 5792 | __masm-> shll(r10, 24); | ||||
| 5793 | __masm-> orl(rax, r10); | ||||
| 5794 | |||||
| 5795 | __masm-> subl(length, 3); | ||||
| 5796 | |||||
| 5797 | __masm-> shll(r15, 8); | ||||
| 5798 | __masm-> shll(r13, 16); | ||||
| 5799 | __masm-> orl(rax, r15); | ||||
| 5800 | |||||
| 5801 | __masm-> addl(start_offset, 3); | ||||
| 5802 | |||||
| 5803 | __masm-> orl(rax, r13); | ||||
| 5804 | // At this point, rax contains | byte1 | byte2 | byte0 | byte1 | ||||
| 5805 | // r13 has byte2 << 16 - need low-order 6 bits to translate. | ||||
| 5806 | // This translated byte is the fourth output byte. | ||||
| 5807 | __masm-> shrl(r13, 16); | ||||
| 5808 | __masm-> andl(r13, 0x3f); | ||||
| 5809 | |||||
| 5810 | // The high-order 6 bits of r15 (byte0) is translated. | ||||
| 5811 | // The translated byte is the first output byte. | ||||
| 5812 | __masm-> shrl(r15, 10); | ||||
| 5813 | |||||
| 5814 | __masm-> load_unsigned_byte(r13, Address(r11, r13)); | ||||
| 5815 | __masm-> load_unsigned_byte(r15, Address(r11, r15)); | ||||
| 5816 | |||||
| 5817 | __masm-> movb(Address(dest, dp, Address::times_1, 3), r13); | ||||
| 5818 | |||||
| 5819 | // Extract high-order 4 bits of byte1 and low-order 2 bits of byte0. | ||||
| 5820 | // This translated byte is the second output byte. | ||||
| 5821 | __masm-> shrl(rax, 4); | ||||
| 5822 | __masm-> movl(r10, rax); | ||||
| 5823 | __masm-> andl(rax, 0x3f); | ||||
| 5824 | |||||
| 5825 | __masm-> movb(Address(dest, dp, Address::times_1, 0), r15); | ||||
| 5826 | |||||
| 5827 | __masm-> load_unsigned_byte(rax, Address(r11, rax)); | ||||
| 5828 | |||||
| 5829 | // Extract low-order 2 bits of byte1 and high-order 4 bits of byte2. | ||||
| 5830 | // This translated byte is the third output byte. | ||||
| 5831 | __masm-> shrl(r10, 18); | ||||
| 5832 | __masm-> andl(r10, 0x3f); | ||||
| 5833 | |||||
| 5834 | __masm-> load_unsigned_byte(r10, Address(r11, r10)); | ||||
| 5835 | |||||
| 5836 | __masm-> movb(Address(dest, dp, Address::times_1, 1), rax); | ||||
| 5837 | __masm-> movb(Address(dest, dp, Address::times_1, 2), r10); | ||||
| 5838 | |||||
| 5839 | __masm-> addl(dp, 4); | ||||
| 5840 | __masm-> cmpl(length, 3); | ||||
| 5841 | __masm-> jcc(Assembler::aboveEqual, L_processdata); | ||||
| 5842 | |||||
| 5843 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 5844 | __masm-> pop(r15); | ||||
| 5845 | __masm-> pop(r14); | ||||
| 5846 | __masm-> pop(r13); | ||||
| 5847 | __masm-> pop(r12); | ||||
| 5848 | __masm-> leave(); | ||||
| 5849 | __masm-> ret(0); | ||||
| 5850 | return start; | ||||
| 5851 | } | ||||
| 5852 | |||||
| 5853 | // base64 AVX512vbmi tables | ||||
| 5854 | address base64_vbmi_lookup_lo_addr() { | ||||
| 5855 | __masm-> align64(); | ||||
| 5856 | StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64"); | ||||
| 5857 | address start = __masm-> pc(); | ||||
| 5858 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5859, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5859 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5859, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5860 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5861 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5862 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5863 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5864 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5865 | __masm-> emit_data64(0x3f8080803e808080, relocInfo::none); | ||||
| 5866 | __masm-> emit_data64(0x3b3a393837363534, relocInfo::none); | ||||
| 5867 | __masm-> emit_data64(0x8080808080803d3c, relocInfo::none); | ||||
| 5868 | return start; | ||||
| 5869 | } | ||||
| 5870 | |||||
| 5871 | address base64_vbmi_lookup_hi_addr() { | ||||
| 5872 | __masm-> align64(); | ||||
| 5873 | StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64"); | ||||
| 5874 | address start = __masm-> pc(); | ||||
| 5875 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5876, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5876 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5876, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5877 | __masm-> emit_data64(0x0605040302010080, relocInfo::none); | ||||
| 5878 | __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none); | ||||
| 5879 | __masm-> emit_data64(0x161514131211100f, relocInfo::none); | ||||
| 5880 | __masm-> emit_data64(0x8080808080191817, relocInfo::none); | ||||
| 5881 | __masm-> emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); | ||||
| 5882 | __masm-> emit_data64(0x2827262524232221, relocInfo::none); | ||||
| 5883 | __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); | ||||
| 5884 | __masm-> emit_data64(0x8080808080333231, relocInfo::none); | ||||
| 5885 | return start; | ||||
| 5886 | } | ||||
| 5887 | address base64_vbmi_lookup_lo_url_addr() { | ||||
| 5888 | __masm-> align64(); | ||||
| 5889 | StubCodeMark mark(this, "StubRoutines", "lookup_lo_base64url"); | ||||
| 5890 | address start = __masm-> pc(); | ||||
| 5891 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5892, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5892 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5892, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5893 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5894 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5895 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5896 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5897 | __masm-> emit_data64(0x8080808080808080, relocInfo::none); | ||||
| 5898 | __masm-> emit_data64(0x80803e8080808080, relocInfo::none); | ||||
| 5899 | __masm-> emit_data64(0x3b3a393837363534, relocInfo::none); | ||||
| 5900 | __masm-> emit_data64(0x8080808080803d3c, relocInfo::none); | ||||
| 5901 | return start; | ||||
| 5902 | } | ||||
| 5903 | |||||
| 5904 | address base64_vbmi_lookup_hi_url_addr() { | ||||
| 5905 | __masm-> align64(); | ||||
| 5906 | StubCodeMark mark(this, "StubRoutines", "lookup_hi_base64url"); | ||||
| 5907 | address start = __masm-> pc(); | ||||
| 5908 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5909, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5909 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5909, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5910 | __masm-> emit_data64(0x0605040302010080, relocInfo::none); | ||||
| 5911 | __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none); | ||||
| 5912 | __masm-> emit_data64(0x161514131211100f, relocInfo::none); | ||||
| 5913 | __masm-> emit_data64(0x3f80808080191817, relocInfo::none); | ||||
| 5914 | __masm-> emit_data64(0x201f1e1d1c1b1a80, relocInfo::none); | ||||
| 5915 | __masm-> emit_data64(0x2827262524232221, relocInfo::none); | ||||
| 5916 | __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); | ||||
| 5917 | __masm-> emit_data64(0x8080808080333231, relocInfo::none); | ||||
| 5918 | return start; | ||||
| 5919 | } | ||||
| 5920 | |||||
| 5921 | address base64_vbmi_pack_vec_addr() { | ||||
| 5922 | __masm-> align64(); | ||||
| 5923 | StubCodeMark mark(this, "StubRoutines", "pack_vec_base64"); | ||||
| 5924 | address start = __masm-> pc(); | ||||
| 5925 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5926, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5926 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5926, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5927 | __masm-> emit_data64(0x090a040506000102, relocInfo::none); | ||||
| 5928 | __masm-> emit_data64(0x161011120c0d0e08, relocInfo::none); | ||||
| 5929 | __masm-> emit_data64(0x1c1d1e18191a1415, relocInfo::none); | ||||
| 5930 | __masm-> emit_data64(0x292a242526202122, relocInfo::none); | ||||
| 5931 | __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none); | ||||
| 5932 | __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none); | ||||
| 5933 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 5934 | __masm-> emit_data64(0x0000000000000000, relocInfo::none); | ||||
| 5935 | return start; | ||||
| 5936 | } | ||||
| 5937 | |||||
| 5938 | address base64_vbmi_join_0_1_addr() { | ||||
| 5939 | __masm-> align64(); | ||||
| 5940 | StubCodeMark mark(this, "StubRoutines", "join_0_1_base64"); | ||||
| 5941 | address start = __masm-> pc(); | ||||
| 5942 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5943, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5943 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5943, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5944 | __masm-> emit_data64(0x090a040506000102, relocInfo::none); | ||||
| 5945 | __masm-> emit_data64(0x161011120c0d0e08, relocInfo::none); | ||||
| 5946 | __masm-> emit_data64(0x1c1d1e18191a1415, relocInfo::none); | ||||
| 5947 | __masm-> emit_data64(0x292a242526202122, relocInfo::none); | ||||
| 5948 | __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none); | ||||
| 5949 | __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none); | ||||
| 5950 | __masm-> emit_data64(0x494a444546404142, relocInfo::none); | ||||
| 5951 | __masm-> emit_data64(0x565051524c4d4e48, relocInfo::none); | ||||
| 5952 | return start; | ||||
| 5953 | } | ||||
| 5954 | |||||
| 5955 | address base64_vbmi_join_1_2_addr() { | ||||
| 5956 | __masm-> align64(); | ||||
| 5957 | StubCodeMark mark(this, "StubRoutines", "join_1_2_base64"); | ||||
| 5958 | address start = __masm-> pc(); | ||||
| 5959 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5960, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5960 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5960, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5961 | __masm-> emit_data64(0x1c1d1e18191a1415, relocInfo::none); | ||||
| 5962 | __masm-> emit_data64(0x292a242526202122, relocInfo::none); | ||||
| 5963 | __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none); | ||||
| 5964 | __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none); | ||||
| 5965 | __masm-> emit_data64(0x494a444546404142, relocInfo::none); | ||||
| 5966 | __masm-> emit_data64(0x565051524c4d4e48, relocInfo::none); | ||||
| 5967 | __masm-> emit_data64(0x5c5d5e58595a5455, relocInfo::none); | ||||
| 5968 | __masm-> emit_data64(0x696a646566606162, relocInfo::none); | ||||
| 5969 | return start; | ||||
| 5970 | } | ||||
| 5971 | |||||
| 5972 | address base64_vbmi_join_2_3_addr() { | ||||
| 5973 | __masm-> align64(); | ||||
| 5974 | StubCodeMark mark(this, "StubRoutines", "join_2_3_base64"); | ||||
| 5975 | address start = __masm-> pc(); | ||||
| 5976 | assert(((unsigned long long)start & 0x3f) == 0,do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5977, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0) | ||||
| 5977 | "Alignment problem (0x%08llx)", (unsigned long long)start)do { if (!(((unsigned long long)start & 0x3f) == 0)) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 5977, "assert(" "((unsigned long long)start & 0x3f) == 0" ") failed", "Alignment problem (0x%08llx)", (unsigned long long )start); ::breakpoint(); } } while (0); | ||||
| 5978 | __masm-> emit_data64(0x363031322c2d2e28, relocInfo::none); | ||||
| 5979 | __masm-> emit_data64(0x3c3d3e38393a3435, relocInfo::none); | ||||
| 5980 | __masm-> emit_data64(0x494a444546404142, relocInfo::none); | ||||
| 5981 | __masm-> emit_data64(0x565051524c4d4e48, relocInfo::none); | ||||
| 5982 | __masm-> emit_data64(0x5c5d5e58595a5455, relocInfo::none); | ||||
| 5983 | __masm-> emit_data64(0x696a646566606162, relocInfo::none); | ||||
| 5984 | __masm-> emit_data64(0x767071726c6d6e68, relocInfo::none); | ||||
| 5985 | __masm-> emit_data64(0x7c7d7e78797a7475, relocInfo::none); | ||||
| 5986 | return start; | ||||
| 5987 | } | ||||
| 5988 | |||||
| 5989 | address base64_decoding_table_addr() { | ||||
| 5990 | StubCodeMark mark(this, "StubRoutines", "decoding_table_base64"); | ||||
| 5991 | address start = __masm-> pc(); | ||||
| 5992 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 5993 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 5994 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 5995 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 5996 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 5997 | __masm-> emit_data64(0x3fffffff3effffff, relocInfo::none); | ||||
| 5998 | __masm-> emit_data64(0x3b3a393837363534, relocInfo::none); | ||||
| 5999 | __masm-> emit_data64(0xffffffffffff3d3c, relocInfo::none); | ||||
| 6000 | __masm-> emit_data64(0x06050403020100ff, relocInfo::none); | ||||
| 6001 | __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none); | ||||
| 6002 | __masm-> emit_data64(0x161514131211100f, relocInfo::none); | ||||
| 6003 | __masm-> emit_data64(0xffffffffff191817, relocInfo::none); | ||||
| 6004 | __masm-> emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); | ||||
| 6005 | __masm-> emit_data64(0x2827262524232221, relocInfo::none); | ||||
| 6006 | __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); | ||||
| 6007 | __masm-> emit_data64(0xffffffffff333231, relocInfo::none); | ||||
| 6008 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6009 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6010 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6011 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6012 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6013 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6014 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6015 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6016 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6017 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6018 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6019 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6020 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6021 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6022 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6023 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6024 | |||||
| 6025 | // URL table | ||||
| 6026 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6027 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6028 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6029 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6030 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6031 | __masm-> emit_data64(0xffff3effffffffff, relocInfo::none); | ||||
| 6032 | __masm-> emit_data64(0x3b3a393837363534, relocInfo::none); | ||||
| 6033 | __masm-> emit_data64(0xffffffffffff3d3c, relocInfo::none); | ||||
| 6034 | __masm-> emit_data64(0x06050403020100ff, relocInfo::none); | ||||
| 6035 | __masm-> emit_data64(0x0e0d0c0b0a090807, relocInfo::none); | ||||
| 6036 | __masm-> emit_data64(0x161514131211100f, relocInfo::none); | ||||
| 6037 | __masm-> emit_data64(0x3fffffffff191817, relocInfo::none); | ||||
| 6038 | __masm-> emit_data64(0x201f1e1d1c1b1aff, relocInfo::none); | ||||
| 6039 | __masm-> emit_data64(0x2827262524232221, relocInfo::none); | ||||
| 6040 | __masm-> emit_data64(0x302f2e2d2c2b2a29, relocInfo::none); | ||||
| 6041 | __masm-> emit_data64(0xffffffffff333231, relocInfo::none); | ||||
| 6042 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6043 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6044 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6045 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6046 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6047 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6048 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6049 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6050 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6051 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6052 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6053 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6054 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6055 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6056 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6057 | __masm-> emit_data64(0xffffffffffffffff, relocInfo::none); | ||||
| 6058 | return start; | ||||
| 6059 | } | ||||
| 6060 | |||||
| 6061 | |||||
| 6062 | // Code for generating Base64 decoding. | ||||
| 6063 | // | ||||
| 6064 | // Based on the article (and associated code) from https://arxiv.org/abs/1910.05109. | ||||
| 6065 | // | ||||
| 6066 | // Intrinsic function prototype in Base64.java: | ||||
| 6067 | // private void decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, isMIME) { | ||||
| 6068 | address generate_base64_decodeBlock() { | ||||
| 6069 | __masm-> align(CodeEntryAlignment); | ||||
| 6070 | StubCodeMark mark(this, "StubRoutines", "implDecode"); | ||||
| 6071 | address start = __masm-> pc(); | ||||
| 6072 | __masm-> enter(); | ||||
| 6073 | |||||
| 6074 | // Save callee-saved registers before using them | ||||
| 6075 | __masm-> push(r12); | ||||
| 6076 | __masm-> push(r13); | ||||
| 6077 | __masm-> push(r14); | ||||
| 6078 | __masm-> push(r15); | ||||
| 6079 | __masm-> push(rbx); | ||||
| 6080 | |||||
| 6081 | // arguments | ||||
| 6082 | const Register source = c_rarg0; // Source Array | ||||
| 6083 | const Register start_offset = c_rarg1; // start offset | ||||
| 6084 | const Register end_offset = c_rarg2; // end offset | ||||
| 6085 | const Register dest = c_rarg3; // destination array | ||||
| 6086 | const Register isMIME = rbx; | ||||
| 6087 | |||||
| 6088 | #ifndef _WIN64 | ||||
| 6089 | const Register dp = c_rarg4; // Position for writing to dest array | ||||
| 6090 | const Register isURL = c_rarg5;// Base64 or URL character set | ||||
| 6091 | __masm-> movl(isMIME, Address(rbp, 2 * wordSize)); | ||||
| 6092 | #else | ||||
| 6093 | const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64 | ||||
| 6094 | const Address isURL_mem(rbp, 7 * wordSize); | ||||
| 6095 | const Register isURL = r10; // pick the volatile windows register | ||||
| 6096 | const Register dp = r12; | ||||
| 6097 | __masm-> movl(dp, dp_mem); | ||||
| 6098 | __masm-> movl(isURL, isURL_mem); | ||||
| 6099 | __masm-> movl(isMIME, Address(rbp, 8 * wordSize)); | ||||
| 6100 | #endif | ||||
| 6101 | |||||
| 6102 | const XMMRegister lookup_lo = xmm5; | ||||
| 6103 | const XMMRegister lookup_hi = xmm6; | ||||
| 6104 | const XMMRegister errorvec = xmm7; | ||||
| 6105 | const XMMRegister pack16_op = xmm9; | ||||
| 6106 | const XMMRegister pack32_op = xmm8; | ||||
| 6107 | const XMMRegister input0 = xmm3; | ||||
| 6108 | const XMMRegister input1 = xmm20; | ||||
| 6109 | const XMMRegister input2 = xmm21; | ||||
| 6110 | const XMMRegister input3 = xmm19; | ||||
| 6111 | const XMMRegister join01 = xmm12; | ||||
| 6112 | const XMMRegister join12 = xmm11; | ||||
| 6113 | const XMMRegister join23 = xmm10; | ||||
| 6114 | const XMMRegister translated0 = xmm2; | ||||
| 6115 | const XMMRegister translated1 = xmm1; | ||||
| 6116 | const XMMRegister translated2 = xmm0; | ||||
| 6117 | const XMMRegister translated3 = xmm4; | ||||
| 6118 | |||||
| 6119 | const XMMRegister merged0 = xmm2; | ||||
| 6120 | const XMMRegister merged1 = xmm1; | ||||
| 6121 | const XMMRegister merged2 = xmm0; | ||||
| 6122 | const XMMRegister merged3 = xmm4; | ||||
| 6123 | const XMMRegister merge_ab_bc0 = xmm2; | ||||
| 6124 | const XMMRegister merge_ab_bc1 = xmm1; | ||||
| 6125 | const XMMRegister merge_ab_bc2 = xmm0; | ||||
| 6126 | const XMMRegister merge_ab_bc3 = xmm4; | ||||
| 6127 | |||||
| 6128 | const XMMRegister pack24bits = xmm4; | ||||
| 6129 | |||||
| 6130 | const Register length = r14; | ||||
| 6131 | const Register output_size = r13; | ||||
| 6132 | const Register output_mask = r15; | ||||
| 6133 | const KRegister input_mask = k1; | ||||
| 6134 | |||||
| 6135 | const XMMRegister input_initial_valid_b64 = xmm0; | ||||
| 6136 | const XMMRegister tmp = xmm10; | ||||
| 6137 | const XMMRegister mask = xmm0; | ||||
| 6138 | const XMMRegister invalid_b64 = xmm1; | ||||
| 6139 | |||||
| 6140 | Label L_process256, L_process64, L_process64Loop, L_exit, L_processdata, L_loadURL; | ||||
| 6141 | Label L_continue, L_finalBit, L_padding, L_donePadding, L_bruteForce; | ||||
| 6142 | Label L_forceLoop, L_bottomLoop, L_checkMIME, L_exit_no_vzero; | ||||
| 6143 | |||||
| 6144 | // calculate length from offsets | ||||
| 6145 | __masm-> movl(length, end_offset); | ||||
| 6146 | __masm-> subl(length, start_offset); | ||||
| 6147 | __masm-> push(dest); // Save for return value calc | ||||
| 6148 | |||||
| 6149 | // If AVX512 VBMI not supported, just compile non-AVX code | ||||
| 6150 | if(VM_Version::supports_avx512_vbmi() && | ||||
| 6151 | VM_Version::supports_avx512bw()) { | ||||
| 6152 | __masm-> cmpl(length, 128); // 128-bytes is break-even for AVX-512 | ||||
| 6153 | __masm-> jcc(Assembler::lessEqual, L_bruteForce); | ||||
| 6154 | |||||
| 6155 | __masm-> cmpl(isMIME, 0); | ||||
| 6156 | __masm-> jcc(Assembler::notEqual, L_bruteForce); | ||||
| 6157 | |||||
| 6158 | // Load lookup tables based on isURL | ||||
| 6159 | __masm-> cmpl(isURL, 0); | ||||
| 6160 | __masm-> jcc(Assembler::notZero, L_loadURL); | ||||
| 6161 | |||||
| 6162 | __masm-> evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_addr()), Assembler::AVX_512bit, r13); | ||||
| 6163 | __masm-> evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_addr()), Assembler::AVX_512bit, r13); | ||||
| 6164 | |||||
| 6165 | __masm-> BIND(L_continue)bind(L_continue); masm-> block_comment("L_continue" ":"); | ||||
| 6166 | |||||
| 6167 | __masm-> movl(r15, 0x01400140); | ||||
| 6168 | __masm-> evpbroadcastd(pack16_op, r15, Assembler::AVX_512bit); | ||||
| 6169 | |||||
| 6170 | __masm-> movl(r15, 0x00011000); | ||||
| 6171 | __masm-> evpbroadcastd(pack32_op, r15, Assembler::AVX_512bit); | ||||
| 6172 | |||||
| 6173 | __masm-> cmpl(length, 0xff); | ||||
| 6174 | __masm-> jcc(Assembler::lessEqual, L_process64); | ||||
| 6175 | |||||
| 6176 | // load masks required for decoding data | ||||
| 6177 | __masm-> BIND(L_processdata)bind(L_processdata); masm-> block_comment("L_processdata" ":" ); | ||||
| 6178 | __masm-> evmovdquq(join01, ExternalAddress(StubRoutines::x86::base64_vbmi_join_0_1_addr()), Assembler::AVX_512bit,r13); | ||||
| 6179 | __masm-> evmovdquq(join12, ExternalAddress(StubRoutines::x86::base64_vbmi_join_1_2_addr()), Assembler::AVX_512bit, r13); | ||||
| 6180 | __masm-> evmovdquq(join23, ExternalAddress(StubRoutines::x86::base64_vbmi_join_2_3_addr()), Assembler::AVX_512bit, r13); | ||||
| 6181 | |||||
| 6182 | __masm-> align32(); | ||||
| 6183 | __masm-> BIND(L_process256)bind(L_process256); masm-> block_comment("L_process256" ":" ); | ||||
| 6184 | // Grab input data | ||||
| 6185 | __masm-> evmovdquq(input0, Address(source, start_offset, Address::times_1, 0x00), Assembler::AVX_512bit); | ||||
| 6186 | __masm-> evmovdquq(input1, Address(source, start_offset, Address::times_1, 0x40), Assembler::AVX_512bit); | ||||
| 6187 | __masm-> evmovdquq(input2, Address(source, start_offset, Address::times_1, 0x80), Assembler::AVX_512bit); | ||||
| 6188 | __masm-> evmovdquq(input3, Address(source, start_offset, Address::times_1, 0xc0), Assembler::AVX_512bit); | ||||
| 6189 | |||||
| 6190 | // Copy the low part of the lookup table into the destination of the permutation | ||||
| 6191 | __masm-> evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); | ||||
| 6192 | __masm-> evmovdquq(translated1, lookup_lo, Assembler::AVX_512bit); | ||||
| 6193 | __masm-> evmovdquq(translated2, lookup_lo, Assembler::AVX_512bit); | ||||
| 6194 | __masm-> evmovdquq(translated3, lookup_lo, Assembler::AVX_512bit); | ||||
| 6195 | |||||
| 6196 | // Translate the base64 input into "decoded" bytes | ||||
| 6197 | __masm-> evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); | ||||
| 6198 | __masm-> evpermt2b(translated1, input1, lookup_hi, Assembler::AVX_512bit); | ||||
| 6199 | __masm-> evpermt2b(translated2, input2, lookup_hi, Assembler::AVX_512bit); | ||||
| 6200 | __masm-> evpermt2b(translated3, input3, lookup_hi, Assembler::AVX_512bit); | ||||
| 6201 | |||||
| 6202 | // OR all of the translations together to check for errors (high-order bit of byte set) | ||||
| 6203 | __masm-> vpternlogd(input0, 0xfe, input1, input2, Assembler::AVX_512bit); | ||||
| 6204 | |||||
| 6205 | __masm-> vpternlogd(input3, 0xfe, translated0, translated1, Assembler::AVX_512bit); | ||||
| 6206 | __masm-> vpternlogd(input0, 0xfe, translated2, translated3, Assembler::AVX_512bit); | ||||
| 6207 | __masm-> vpor(errorvec, input3, input0, Assembler::AVX_512bit); | ||||
| 6208 | |||||
| 6209 | // Check if there was an error - if so, try 64-byte chunks | ||||
| 6210 | __masm-> evpmovb2m(k3, errorvec, Assembler::AVX_512bit); | ||||
| 6211 | __masm-> kortestql(k3, k3); | ||||
| 6212 | __masm-> jcc(Assembler::notZero, L_process64); | ||||
| 6213 | |||||
| 6214 | // The merging and shuffling happens here | ||||
| 6215 | // We multiply each byte pair [00dddddd | 00cccccc | 00bbbbbb | 00aaaaaa] | ||||
| 6216 | // Multiply [00cccccc] by 2^6 added to [00dddddd] to get [0000cccc | ccdddddd] | ||||
| 6217 | // The pack16_op is a vector of 0x01400140, so multiply D by 1 and C by 0x40 | ||||
| 6218 | __masm-> vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); | ||||
| 6219 | __masm-> vpmaddubsw(merge_ab_bc1, translated1, pack16_op, Assembler::AVX_512bit); | ||||
| 6220 | __masm-> vpmaddubsw(merge_ab_bc2, translated2, pack16_op, Assembler::AVX_512bit); | ||||
| 6221 | __masm-> vpmaddubsw(merge_ab_bc3, translated3, pack16_op, Assembler::AVX_512bit); | ||||
| 6222 | |||||
| 6223 | // Now do the same with packed 16-bit values. | ||||
| 6224 | // We start with [0000cccc | ccdddddd | 0000aaaa | aabbbbbb] | ||||
| 6225 | // pack32_op is 0x00011000 (2^12, 1), so this multiplies [0000aaaa | aabbbbbb] by 2^12 | ||||
| 6226 | // and adds [0000cccc | ccdddddd] to yield [00000000 | aaaaaabb | bbbbcccc | ccdddddd] | ||||
| 6227 | __masm-> vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); | ||||
| 6228 | __masm-> vpmaddwd(merged1, merge_ab_bc1, pack32_op, Assembler::AVX_512bit); | ||||
| 6229 | __masm-> vpmaddwd(merged2, merge_ab_bc2, pack32_op, Assembler::AVX_512bit); | ||||
| 6230 | __masm-> vpmaddwd(merged3, merge_ab_bc3, pack32_op, Assembler::AVX_512bit); | ||||
| 6231 | |||||
| 6232 | // The join vectors specify which byte from which vector goes into the outputs | ||||
| 6233 | // One of every 4 bytes in the extended vector is zero, so we pack them into their | ||||
| 6234 | // final positions in the register for storing (256 bytes in, 192 bytes out) | ||||
| 6235 | __masm-> evpermt2b(merged0, join01, merged1, Assembler::AVX_512bit); | ||||
| 6236 | __masm-> evpermt2b(merged1, join12, merged2, Assembler::AVX_512bit); | ||||
| 6237 | __masm-> evpermt2b(merged2, join23, merged3, Assembler::AVX_512bit); | ||||
| 6238 | |||||
| 6239 | // Store result | ||||
| 6240 | __masm-> evmovdquq(Address(dest, dp, Address::times_1, 0x00), merged0, Assembler::AVX_512bit); | ||||
| 6241 | __masm-> evmovdquq(Address(dest, dp, Address::times_1, 0x40), merged1, Assembler::AVX_512bit); | ||||
| 6242 | __masm-> evmovdquq(Address(dest, dp, Address::times_1, 0x80), merged2, Assembler::AVX_512bit); | ||||
| 6243 | |||||
| 6244 | __masm-> addptr(source, 0x100); | ||||
| 6245 | __masm-> addptr(dest, 0xc0); | ||||
| 6246 | __masm-> subl(length, 0x100); | ||||
| 6247 | __masm-> cmpl(length, 64 * 4); | ||||
| 6248 | __masm-> jcc(Assembler::greaterEqual, L_process256); | ||||
| 6249 | |||||
| 6250 | // At this point, we've decoded 64 * 4 * n bytes. | ||||
| 6251 | // The remaining length will be <= 64 * 4 - 1. | ||||
| 6252 | // UNLESS there was an error decoding the first 256-byte chunk. In this | ||||
| 6253 | // case, the length will be arbitrarily long. | ||||
| 6254 | // | ||||
| 6255 | // Note that this will be the path for MIME-encoded strings. | ||||
| 6256 | |||||
| 6257 | __masm-> BIND(L_process64)bind(L_process64); masm-> block_comment("L_process64" ":"); | ||||
| 6258 | |||||
| 6259 | __masm-> evmovdquq(pack24bits, ExternalAddress(StubRoutines::x86::base64_vbmi_pack_vec_addr()), Assembler::AVX_512bit, r13); | ||||
| 6260 | |||||
| 6261 | __masm-> cmpl(length, 63); | ||||
| 6262 | __masm-> jcc(Assembler::lessEqual, L_finalBit); | ||||
| 6263 | |||||
| 6264 | __masm-> mov64(rax, 0x0000ffffffffffff); | ||||
| 6265 | __masm-> kmovql(k2, rax); | ||||
| 6266 | |||||
| 6267 | __masm-> align32(); | ||||
| 6268 | __masm-> BIND(L_process64Loop)bind(L_process64Loop); masm-> block_comment("L_process64Loop" ":"); | ||||
| 6269 | |||||
| 6270 | // Handle first 64-byte block | ||||
| 6271 | |||||
| 6272 | __masm-> evmovdquq(input0, Address(source, start_offset), Assembler::AVX_512bit); | ||||
| 6273 | __masm-> evmovdquq(translated0, lookup_lo, Assembler::AVX_512bit); | ||||
| 6274 | __masm-> evpermt2b(translated0, input0, lookup_hi, Assembler::AVX_512bit); | ||||
| 6275 | |||||
| 6276 | __masm-> vpor(errorvec, translated0, input0, Assembler::AVX_512bit); | ||||
| 6277 | |||||
| 6278 | // Check for error and bomb out before updating dest | ||||
| 6279 | __masm-> evpmovb2m(k3, errorvec, Assembler::AVX_512bit); | ||||
| 6280 | __masm-> kortestql(k3, k3); | ||||
| 6281 | __masm-> jcc(Assembler::notZero, L_exit); | ||||
| 6282 | |||||
| 6283 | // Pack output register, selecting correct byte ordering | ||||
| 6284 | __masm-> vpmaddubsw(merge_ab_bc0, translated0, pack16_op, Assembler::AVX_512bit); | ||||
| 6285 | __masm-> vpmaddwd(merged0, merge_ab_bc0, pack32_op, Assembler::AVX_512bit); | ||||
| 6286 | __masm-> vpermb(merged0, pack24bits, merged0, Assembler::AVX_512bit); | ||||
| 6287 | |||||
| 6288 | __masm-> evmovdqub(Address(dest, dp), k2, merged0, true, Assembler::AVX_512bit); | ||||
| 6289 | |||||
| 6290 | __masm-> subl(length, 64); | ||||
| 6291 | __masm-> addptr(source, 64); | ||||
| 6292 | __masm-> addptr(dest, 48); | ||||
| 6293 | |||||
| 6294 | __masm-> cmpl(length, 64); | ||||
| 6295 | __masm-> jcc(Assembler::greaterEqual, L_process64Loop); | ||||
| 6296 | |||||
| 6297 | __masm-> cmpl(length, 0); | ||||
| 6298 | __masm-> jcc(Assembler::lessEqual, L_exit); | ||||
| 6299 | |||||
| 6300 | __masm-> BIND(L_finalBit)bind(L_finalBit); masm-> block_comment("L_finalBit" ":"); | ||||
| 6301 | // Now have 1 to 63 bytes left to decode | ||||
| 6302 | |||||
| 6303 | // I was going to let Java take care of the final fragment | ||||
| 6304 | // however it will repeatedly call this routine for every 4 bytes | ||||
| 6305 | // of input data, so handle the rest here. | ||||
| 6306 | __masm-> movq(rax, -1); | ||||
| 6307 | __masm-> bzhiq(rax, rax, length); // Input mask in rax | ||||
| 6308 | |||||
| 6309 | __masm-> movl(output_size, length); | ||||
| 6310 | __masm-> shrl(output_size, 2); // Find (len / 4) * 3 (output length) | ||||
| 6311 | __masm-> lea(output_size, Address(output_size, output_size, Address::times_2, 0)); | ||||
| 6312 | // output_size in r13 | ||||
| 6313 | |||||
| 6314 | // Strip pad characters, if any, and adjust length and mask | ||||
| 6315 | __masm-> cmpb(Address(source, length, Address::times_1, -1), '='); | ||||
| 6316 | __masm-> jcc(Assembler::equal, L_padding); | ||||
| 6317 | |||||
| 6318 | __masm-> BIND(L_donePadding)bind(L_donePadding); masm-> block_comment("L_donePadding" ":" ); | ||||
| 6319 | |||||
| 6320 | // Output size is (64 - output_size), output mask is (all 1s >> output_size). | ||||
| 6321 | __masm-> kmovql(input_mask, rax); | ||||
| 6322 | __masm-> movq(output_mask, -1); | ||||
| 6323 | __masm-> bzhiq(output_mask, output_mask, output_size); | ||||
| 6324 | |||||
| 6325 | // Load initial input with all valid base64 characters. Will be used | ||||
| 6326 | // in merging source bytes to avoid masking when determining if an error occurred. | ||||
| 6327 | __masm-> movl(rax, 0x61616161); | ||||
| 6328 | __masm-> evpbroadcastd(input_initial_valid_b64, rax, Assembler::AVX_512bit); | ||||
| 6329 | |||||
| 6330 | // A register containing all invalid base64 decoded values | ||||
| 6331 | __masm-> movl(rax, 0x80808080); | ||||
| 6332 | __masm-> evpbroadcastd(invalid_b64, rax, Assembler::AVX_512bit); | ||||
| 6333 | |||||
| 6334 | // input_mask is in k1 | ||||
| 6335 | // output_size is in r13 | ||||
| 6336 | // output_mask is in r15 | ||||
| 6337 | // zmm0 - free | ||||
| 6338 | // zmm1 - 0x00011000 | ||||
| 6339 | // zmm2 - 0x01400140 | ||||
| 6340 | // zmm3 - errorvec | ||||
| 6341 | // zmm4 - pack vector | ||||
| 6342 | // zmm5 - lookup_lo | ||||
| 6343 | // zmm6 - lookup_hi | ||||
| 6344 | // zmm7 - errorvec | ||||
| 6345 | // zmm8 - 0x61616161 | ||||
| 6346 | // zmm9 - 0x80808080 | ||||
| 6347 | |||||
| 6348 | // Load only the bytes from source, merging into our "fully-valid" register | ||||
| 6349 | __masm-> evmovdqub(input_initial_valid_b64, input_mask, Address(source, start_offset, Address::times_1, 0x0), true, Assembler::AVX_512bit); | ||||
| 6350 | |||||
| 6351 | // Decode all bytes within our merged input | ||||
| 6352 | __masm-> evmovdquq(tmp, lookup_lo, Assembler::AVX_512bit); | ||||
| 6353 | __masm-> evpermt2b(tmp, input_initial_valid_b64, lookup_hi, Assembler::AVX_512bit); | ||||
| 6354 | __masm-> vporq(mask, tmp, input_initial_valid_b64, Assembler::AVX_512bit); | ||||
| 6355 | |||||
| 6356 | // Check for error. Compare (decoded | initial) to all invalid. | ||||
| 6357 | // If any bytes have their high-order bit set, then we have an error. | ||||
| 6358 | __masm-> evptestmb(k2, mask, invalid_b64, Assembler::AVX_512bit); | ||||
| 6359 | __masm-> kortestql(k2, k2); | ||||
| 6360 | |||||
| 6361 | // If we have an error, use the brute force loop to decode what we can (4-byte chunks). | ||||
| 6362 | __masm-> jcc(Assembler::notZero, L_bruteForce); | ||||
| 6363 | |||||
| 6364 | // Shuffle output bytes | ||||
| 6365 | __masm-> vpmaddubsw(tmp, tmp, pack16_op, Assembler::AVX_512bit); | ||||
| 6366 | __masm-> vpmaddwd(tmp, tmp, pack32_op, Assembler::AVX_512bit); | ||||
| 6367 | |||||
| 6368 | __masm-> vpermb(tmp, pack24bits, tmp, Assembler::AVX_512bit); | ||||
| 6369 | __masm-> kmovql(k1, output_mask); | ||||
| 6370 | __masm-> evmovdqub(Address(dest, dp), k1, tmp, true, Assembler::AVX_512bit); | ||||
| 6371 | |||||
| 6372 | __masm-> addptr(dest, output_size); | ||||
| 6373 | |||||
| 6374 | __masm-> BIND(L_exit)bind(L_exit); masm-> block_comment("L_exit" ":"); | ||||
| 6375 | __masm-> vzeroupper(); | ||||
| 6376 | __masm-> pop(rax); // Get original dest value | ||||
| 6377 | __masm-> subptr(dest, rax); // Number of bytes converted | ||||
| 6378 | __masm-> movptr(rax, dest); | ||||
| 6379 | __masm-> pop(rbx); | ||||
| 6380 | __masm-> pop(r15); | ||||
| 6381 | __masm-> pop(r14); | ||||
| 6382 | __masm-> pop(r13); | ||||
| 6383 | __masm-> pop(r12); | ||||
| 6384 | __masm-> leave(); | ||||
| 6385 | __masm-> ret(0); | ||||
| 6386 | |||||
| 6387 | __masm-> BIND(L_loadURL)bind(L_loadURL); masm-> block_comment("L_loadURL" ":"); | ||||
| 6388 | __masm-> evmovdquq(lookup_lo, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_lo_url_addr()), Assembler::AVX_512bit, r13); | ||||
| 6389 | __masm-> evmovdquq(lookup_hi, ExternalAddress(StubRoutines::x86::base64_vbmi_lookup_hi_url_addr()), Assembler::AVX_512bit, r13); | ||||
| 6390 | __masm-> jmp(L_continue); | ||||
| 6391 | |||||
| 6392 | __masm-> BIND(L_padding)bind(L_padding); masm-> block_comment("L_padding" ":"); | ||||
| 6393 | __masm-> decrementq(output_size, 1); | ||||
| 6394 | __masm-> shrq(rax, 1); | ||||
| 6395 | |||||
| 6396 | __masm-> cmpb(Address(source, length, Address::times_1, -2), '='); | ||||
| 6397 | __masm-> jcc(Assembler::notEqual, L_donePadding); | ||||
| 6398 | |||||
| 6399 | __masm-> decrementq(output_size, 1); | ||||
| 6400 | __masm-> shrq(rax, 1); | ||||
| 6401 | __masm-> jmp(L_donePadding); | ||||
| 6402 | |||||
| 6403 | __masm-> align32(); | ||||
| 6404 | __masm-> BIND(L_bruteForce)bind(L_bruteForce); masm-> block_comment("L_bruteForce" ":" ); | ||||
| 6405 | } // End of if(avx512_vbmi) | ||||
| 6406 | |||||
| 6407 | // Use non-AVX code to decode 4-byte chunks into 3 bytes of output | ||||
| 6408 | |||||
| 6409 | // Register state (Linux): | ||||
| 6410 | // r12-15 - saved on stack | ||||
| 6411 | // rdi - src | ||||
| 6412 | // rsi - sp | ||||
| 6413 | // rdx - sl | ||||
| 6414 | // rcx - dst | ||||
| 6415 | // r8 - dp | ||||
| 6416 | // r9 - isURL | ||||
| 6417 | |||||
| 6418 | // Register state (Windows): | ||||
| 6419 | // r12-15 - saved on stack | ||||
| 6420 | // rcx - src | ||||
| 6421 | // rdx - sp | ||||
| 6422 | // r8 - sl | ||||
| 6423 | // r9 - dst | ||||
| 6424 | // r12 - dp | ||||
| 6425 | // r10 - isURL | ||||
| 6426 | |||||
| 6427 | // Registers (common): | ||||
| 6428 | // length (r14) - bytes in src | ||||
| 6429 | |||||
| 6430 | const Register decode_table = r11; | ||||
| 6431 | const Register out_byte_count = rbx; | ||||
| 6432 | const Register byte1 = r13; | ||||
| 6433 | const Register byte2 = r15; | ||||
| 6434 | const Register byte3 = WINDOWS_ONLY(r8) NOT_WINDOWS(rdx)rdx; | ||||
| 6435 | const Register byte4 = WINDOWS_ONLY(r10) NOT_WINDOWS(r9)r9; | ||||
| 6436 | |||||
| 6437 | __masm-> shrl(length, 2); // Multiple of 4 bytes only - length is # 4-byte chunks | ||||
| 6438 | __masm-> cmpl(length, 0); | ||||
| 6439 | __masm-> jcc(Assembler::lessEqual, L_exit_no_vzero); | ||||
| 6440 | |||||
| 6441 | __masm-> shll(isURL, 8); // index into decode table based on isURL | ||||
| 6442 | __masm-> lea(decode_table, ExternalAddress(StubRoutines::x86::base64_decoding_table_addr())); | ||||
| 6443 | __masm-> addptr(decode_table, isURL); | ||||
| 6444 | |||||
| 6445 | __masm-> jmp(L_bottomLoop); | ||||
| 6446 | |||||
| 6447 | __masm-> align32(); | ||||
| 6448 | __masm-> BIND(L_forceLoop)bind(L_forceLoop); masm-> block_comment("L_forceLoop" ":"); | ||||
| 6449 | __masm-> shll(byte1, 18); | ||||
| 6450 | __masm-> shll(byte2, 12); | ||||
| 6451 | __masm-> shll(byte3, 6); | ||||
| 6452 | __masm-> orl(byte1, byte2); | ||||
| 6453 | __masm-> orl(byte1, byte3); | ||||
| 6454 | __masm-> orl(byte1, byte4); | ||||
| 6455 | |||||
| 6456 | __masm-> addptr(source, 4); | ||||
| 6457 | |||||
| 6458 | __masm-> movb(Address(dest, dp, Address::times_1, 2), byte1); | ||||
| 6459 | __masm-> shrl(byte1, 8); | ||||
| 6460 | __masm-> movb(Address(dest, dp, Address::times_1, 1), byte1); | ||||
| 6461 | __masm-> shrl(byte1, 8); | ||||
| 6462 | __masm-> movb(Address(dest, dp, Address::times_1, 0), byte1); | ||||
| 6463 | |||||
| 6464 | __masm-> addptr(dest, 3); | ||||
| 6465 | __masm-> decrementl(length, 1); | ||||
| 6466 | __masm-> jcc(Assembler::zero, L_exit_no_vzero); | ||||
| 6467 | |||||
| 6468 | __masm-> BIND(L_bottomLoop)bind(L_bottomLoop); masm-> block_comment("L_bottomLoop" ":" ); | ||||
| 6469 | __masm-> load_unsigned_byte(byte1, Address(source, start_offset, Address::times_1, 0x00)); | ||||
| 6470 | __masm-> load_unsigned_byte(byte2, Address(source, start_offset, Address::times_1, 0x01)); | ||||
| 6471 | __masm-> load_signed_byte(byte1, Address(decode_table, byte1)); | ||||
| 6472 | __masm-> load_signed_byte(byte2, Address(decode_table, byte2)); | ||||
| 6473 | __masm-> load_unsigned_byte(byte3, Address(source, start_offset, Address::times_1, 0x02)); | ||||
| 6474 | __masm-> load_unsigned_byte(byte4, Address(source, start_offset, Address::times_1, 0x03)); | ||||
| 6475 | __masm-> load_signed_byte(byte3, Address(decode_table, byte3)); | ||||
| 6476 | __masm-> load_signed_byte(byte4, Address(decode_table, byte4)); | ||||
| 6477 | |||||
| 6478 | __masm-> mov(rax, byte1); | ||||
| 6479 | __masm-> orl(rax, byte2); | ||||
| 6480 | __masm-> orl(rax, byte3); | ||||
| 6481 | __masm-> orl(rax, byte4); | ||||
| 6482 | __masm-> jcc(Assembler::positive, L_forceLoop); | ||||
| 6483 | |||||
| 6484 | __masm-> BIND(L_exit_no_vzero)bind(L_exit_no_vzero); masm-> block_comment("L_exit_no_vzero" ":"); | ||||
| 6485 | __masm-> pop(rax); // Get original dest value | ||||
| 6486 | __masm-> subptr(dest, rax); // Number of bytes converted | ||||
| 6487 | __masm-> movptr(rax, dest); | ||||
| 6488 | __masm-> pop(rbx); | ||||
| 6489 | __masm-> pop(r15); | ||||
| 6490 | __masm-> pop(r14); | ||||
| 6491 | __masm-> pop(r13); | ||||
| 6492 | __masm-> pop(r12); | ||||
| 6493 | __masm-> leave(); | ||||
| 6494 | __masm-> ret(0); | ||||
| 6495 | |||||
| 6496 | return start; | ||||
| 6497 | } | ||||
| 6498 | |||||
| 6499 | |||||
| 6500 | /** | ||||
| 6501 | * Arguments: | ||||
| 6502 | * | ||||
| 6503 | * Inputs: | ||||
| 6504 | * c_rarg0 - int crc | ||||
| 6505 | * c_rarg1 - byte* buf | ||||
| 6506 | * c_rarg2 - int length | ||||
| 6507 | * | ||||
| 6508 | * Ouput: | ||||
| 6509 | * rax - int crc result | ||||
| 6510 | */ | ||||
| 6511 | address generate_updateBytesCRC32() { | ||||
| 6512 | assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions")do { if (!(UseCRC32Intrinsics)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 6512, "assert(" "UseCRC32Intrinsics" ") failed", "need AVX and CLMUL instructions" ); ::breakpoint(); } } while (0); | ||||
| 6513 | |||||
| 6514 | __masm-> align(CodeEntryAlignment); | ||||
| 6515 | StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32"); | ||||
| 6516 | |||||
| 6517 | address start = __masm-> pc(); | ||||
| 6518 | // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6519 | // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6520 | // rscratch1: r10 | ||||
| 6521 | const Register crc = c_rarg0; // crc | ||||
| 6522 | const Register buf = c_rarg1; // source java byte array address | ||||
| 6523 | const Register len = c_rarg2; // length | ||||
| 6524 | const Register table = c_rarg3; // crc_table address (reuse register) | ||||
| 6525 | const Register tmp1 = r11; | ||||
| 6526 | const Register tmp2 = r10; | ||||
| 6527 | assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax); | ||||
| 6528 | |||||
| 6529 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6530 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6531 | |||||
| 6532 | if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && | ||||
| 6533 | VM_Version::supports_avx512bw() && | ||||
| 6534 | VM_Version::supports_avx512vl()) { | ||||
| 6535 | // The constants used in the CRC32 algorithm requires the 1's compliment of the initial crc value. | ||||
| 6536 | // However, the constant table for CRC32-C assumes the original crc value. Account for this | ||||
| 6537 | // difference before calling and after returning. | ||||
| 6538 | __masm-> lea(table, ExternalAddress(StubRoutines::x86::crc_table_avx512_addr())); | ||||
| 6539 | __masm-> notl(crc); | ||||
| 6540 | __masm-> kernel_crc32_avx512(crc, buf, len, table, tmp1, tmp2); | ||||
| 6541 | __masm-> notl(crc); | ||||
| 6542 | } else { | ||||
| 6543 | __masm-> kernel_crc32(crc, buf, len, table, tmp1); | ||||
| 6544 | } | ||||
| 6545 | |||||
| 6546 | __masm-> movl(rax, crc); | ||||
| 6547 | __masm-> vzeroupper(); | ||||
| 6548 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6549 | __masm-> ret(0); | ||||
| 6550 | |||||
| 6551 | return start; | ||||
| 6552 | } | ||||
| 6553 | |||||
| 6554 | /** | ||||
| 6555 | * Arguments: | ||||
| 6556 | * | ||||
| 6557 | * Inputs: | ||||
| 6558 | * c_rarg0 - int crc | ||||
| 6559 | * c_rarg1 - byte* buf | ||||
| 6560 | * c_rarg2 - long length | ||||
| 6561 | * c_rarg3 - table_start - optional (present only when doing a library_call, | ||||
| 6562 | * not used by x86 algorithm) | ||||
| 6563 | * | ||||
| 6564 | * Ouput: | ||||
| 6565 | * rax - int crc result | ||||
| 6566 | */ | ||||
| 6567 | address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) { | ||||
| 6568 | assert(UseCRC32CIntrinsics, "need SSE4_2")do { if (!(UseCRC32CIntrinsics)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 6568, "assert(" "UseCRC32CIntrinsics" ") failed", "need SSE4_2" ); ::breakpoint(); } } while (0); | ||||
| 6569 | __masm-> align(CodeEntryAlignment); | ||||
| 6570 | StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C"); | ||||
| 6571 | address start = __masm-> pc(); | ||||
| 6572 | //reg.arg int#0 int#1 int#2 int#3 int#4 int#5 float regs | ||||
| 6573 | //Windows RCX RDX R8 R9 none none XMM0..XMM3 | ||||
| 6574 | //Lin / Sol RDI RSI RDX RCX R8 R9 XMM0..XMM7 | ||||
| 6575 | const Register crc = c_rarg0; // crc | ||||
| 6576 | const Register buf = c_rarg1; // source java byte array address | ||||
| 6577 | const Register len = c_rarg2; // length | ||||
| 6578 | const Register a = rax; | ||||
| 6579 | const Register j = r9; | ||||
| 6580 | const Register k = r10; | ||||
| 6581 | const Register l = r11; | ||||
| 6582 | #ifdef _WIN64 | ||||
| 6583 | const Register y = rdi; | ||||
| 6584 | const Register z = rsi; | ||||
| 6585 | #else | ||||
| 6586 | const Register y = rcx; | ||||
| 6587 | const Register z = r8; | ||||
| 6588 | #endif | ||||
| 6589 | assert_different_registers(crc, buf, len, a, j, k, l, y, z); | ||||
| 6590 | |||||
| 6591 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6592 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6593 | if (VM_Version::supports_sse4_1() && VM_Version::supports_avx512_vpclmulqdq() && | ||||
| 6594 | VM_Version::supports_avx512bw() && | ||||
| 6595 | VM_Version::supports_avx512vl()) { | ||||
| 6596 | __masm-> lea(j, ExternalAddress(StubRoutines::x86::crc32c_table_avx512_addr())); | ||||
| 6597 | __masm-> kernel_crc32_avx512(crc, buf, len, j, l, k); | ||||
| 6598 | } else { | ||||
| 6599 | #ifdef _WIN64 | ||||
| 6600 | __masm-> push(y); | ||||
| 6601 | __masm-> push(z); | ||||
| 6602 | #endif | ||||
| 6603 | __masm-> crc32c_ipl_alg2_alt2(crc, buf, len, | ||||
| 6604 | a, j, k, | ||||
| 6605 | l, y, z, | ||||
| 6606 | c_farg0, c_farg1, c_farg2, | ||||
| 6607 | is_pclmulqdq_supported); | ||||
| 6608 | #ifdef _WIN64 | ||||
| 6609 | __masm-> pop(z); | ||||
| 6610 | __masm-> pop(y); | ||||
| 6611 | #endif | ||||
| 6612 | } | ||||
| 6613 | __masm-> movl(rax, crc); | ||||
| 6614 | __masm-> vzeroupper(); | ||||
| 6615 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6616 | __masm-> ret(0); | ||||
| 6617 | |||||
| 6618 | return start; | ||||
| 6619 | } | ||||
| 6620 | |||||
| 6621 | |||||
| 6622 | /*** | ||||
| 6623 | * Arguments: | ||||
| 6624 | * | ||||
| 6625 | * Inputs: | ||||
| 6626 | * c_rarg0 - int adler | ||||
| 6627 | * c_rarg1 - byte* buff | ||||
| 6628 | * c_rarg2 - int len | ||||
| 6629 | * | ||||
| 6630 | * Output: | ||||
| 6631 | * rax - int adler result | ||||
| 6632 | */ | ||||
| 6633 | |||||
| 6634 | address generate_updateBytesAdler32() { | ||||
| 6635 | assert(UseAdler32Intrinsics, "need AVX2")do { if (!(UseAdler32Intrinsics)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 6635, "assert(" "UseAdler32Intrinsics" ") failed", "need AVX2" ); ::breakpoint(); } } while (0); | ||||
| 6636 | |||||
| 6637 | __masm-> align(CodeEntryAlignment); | ||||
| 6638 | StubCodeMark mark(this, "StubRoutines", "updateBytesAdler32"); | ||||
| 6639 | |||||
| 6640 | address start = __masm-> pc(); | ||||
| 6641 | |||||
| 6642 | const Register data = r9; | ||||
| 6643 | const Register size = r10; | ||||
| 6644 | |||||
| 6645 | const XMMRegister yshuf0 = xmm6; | ||||
| 6646 | const XMMRegister yshuf1 = xmm7; | ||||
| 6647 | assert_different_registers(c_rarg0, c_rarg1, c_rarg2, data, size); | ||||
| 6648 | |||||
| 6649 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6650 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6651 | |||||
| 6652 | __masm-> vmovdqu(yshuf0, ExternalAddress((address) StubRoutines::x86::_adler32_shuf0_table), r9); | ||||
| 6653 | __masm-> vmovdqu(yshuf1, ExternalAddress((address) StubRoutines::x86::_adler32_shuf1_table), r9); | ||||
| 6654 | __masm-> movptr(data, c_rarg1); //data | ||||
| 6655 | __masm-> movl(size, c_rarg2); //length | ||||
| 6656 | __masm-> updateBytesAdler32(c_rarg0, data, size, yshuf0, yshuf1, ExternalAddress((address) StubRoutines::x86::_adler32_ascale_table)); | ||||
| 6657 | __masm-> leave(); | ||||
| 6658 | __masm-> ret(0); | ||||
| 6659 | return start; | ||||
| 6660 | } | ||||
| 6661 | |||||
| 6662 | /** | ||||
| 6663 | * Arguments: | ||||
| 6664 | * | ||||
| 6665 | * Input: | ||||
| 6666 | * c_rarg0 - x address | ||||
| 6667 | * c_rarg1 - x length | ||||
| 6668 | * c_rarg2 - y address | ||||
| 6669 | * c_rarg3 - y length | ||||
| 6670 | * not Win64 | ||||
| 6671 | * c_rarg4 - z address | ||||
| 6672 | * c_rarg5 - z length | ||||
| 6673 | * Win64 | ||||
| 6674 | * rsp+40 - z address | ||||
| 6675 | * rsp+48 - z length | ||||
| 6676 | */ | ||||
| 6677 | address generate_multiplyToLen() { | ||||
| 6678 | __masm-> align(CodeEntryAlignment); | ||||
| 6679 | StubCodeMark mark(this, "StubRoutines", "multiplyToLen"); | ||||
| 6680 | |||||
| 6681 | address start = __masm-> pc(); | ||||
| 6682 | // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6683 | // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6684 | const Register x = rdi; | ||||
| 6685 | const Register xlen = rax; | ||||
| 6686 | const Register y = rsi; | ||||
| 6687 | const Register ylen = rcx; | ||||
| 6688 | const Register z = r8; | ||||
| 6689 | const Register zlen = r11; | ||||
| 6690 | |||||
| 6691 | // Next registers will be saved on stack in multiply_to_len(). | ||||
| 6692 | const Register tmp1 = r12; | ||||
| 6693 | const Register tmp2 = r13; | ||||
| 6694 | const Register tmp3 = r14; | ||||
| 6695 | const Register tmp4 = r15; | ||||
| 6696 | const Register tmp5 = rbx; | ||||
| 6697 | |||||
| 6698 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6699 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6700 | |||||
| 6701 | #ifndef _WIN64 | ||||
| 6702 | __masm-> movptr(zlen, r9); // Save r9 in r11 - zlen | ||||
| 6703 | #endif | ||||
| 6704 | setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx | ||||
| 6705 | // ylen => rcx, z => r8, zlen => r11 | ||||
| 6706 | // r9 and r10 may be used to save non-volatile registers | ||||
| 6707 | #ifdef _WIN64 | ||||
| 6708 | // last 2 arguments (#4, #5) are on stack on Win64 | ||||
| 6709 | __masm-> movptr(z, Address(rsp, 6 * wordSize)); | ||||
| 6710 | __masm-> movptr(zlen, Address(rsp, 7 * wordSize)); | ||||
| 6711 | #endif | ||||
| 6712 | |||||
| 6713 | __masm-> movptr(xlen, rsi); | ||||
| 6714 | __masm-> movptr(y, rdx); | ||||
| 6715 | __masm-> multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5); | ||||
| 6716 | |||||
| 6717 | restore_arg_regs(); | ||||
| 6718 | |||||
| 6719 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6720 | __masm-> ret(0); | ||||
| 6721 | |||||
| 6722 | return start; | ||||
| 6723 | } | ||||
| 6724 | |||||
| 6725 | /** | ||||
| 6726 | * Arguments: | ||||
| 6727 | * | ||||
| 6728 | * Input: | ||||
| 6729 | * c_rarg0 - obja address | ||||
| 6730 | * c_rarg1 - objb address | ||||
| 6731 | * c_rarg3 - length length | ||||
| 6732 | * c_rarg4 - scale log2_array_indxscale | ||||
| 6733 | * | ||||
| 6734 | * Output: | ||||
| 6735 | * rax - int >= mismatched index, < 0 bitwise complement of tail | ||||
| 6736 | */ | ||||
| 6737 | address generate_vectorizedMismatch() { | ||||
| 6738 | __masm-> align(CodeEntryAlignment); | ||||
| 6739 | StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch"); | ||||
| 6740 | address start = __masm-> pc(); | ||||
| 6741 | |||||
| 6742 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6743 | __masm-> enter(); | ||||
| 6744 | |||||
| 6745 | #ifdef _WIN64 // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6746 | const Register scale = c_rarg0; //rcx, will exchange with r9 | ||||
| 6747 | const Register objb = c_rarg1; //rdx | ||||
| 6748 | const Register length = c_rarg2; //r8 | ||||
| 6749 | const Register obja = c_rarg3; //r9 | ||||
| 6750 | __masm-> xchgq(obja, scale); //now obja and scale contains the correct contents | ||||
| 6751 | |||||
| 6752 | const Register tmp1 = r10; | ||||
| 6753 | const Register tmp2 = r11; | ||||
| 6754 | #endif | ||||
| 6755 | #ifndef _WIN64 // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6756 | const Register obja = c_rarg0; //U:rdi | ||||
| 6757 | const Register objb = c_rarg1; //U:rsi | ||||
| 6758 | const Register length = c_rarg2; //U:rdx | ||||
| 6759 | const Register scale = c_rarg3; //U:rcx | ||||
| 6760 | const Register tmp1 = r8; | ||||
| 6761 | const Register tmp2 = r9; | ||||
| 6762 | #endif | ||||
| 6763 | const Register result = rax; //return value | ||||
| 6764 | const XMMRegister vec0 = xmm0; | ||||
| 6765 | const XMMRegister vec1 = xmm1; | ||||
| 6766 | const XMMRegister vec2 = xmm2; | ||||
| 6767 | |||||
| 6768 | __masm-> vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2); | ||||
| 6769 | |||||
| 6770 | __masm-> vzeroupper(); | ||||
| 6771 | __masm-> leave(); | ||||
| 6772 | __masm-> ret(0); | ||||
| 6773 | |||||
| 6774 | return start; | ||||
| 6775 | } | ||||
| 6776 | |||||
| 6777 | /** | ||||
| 6778 | * Arguments: | ||||
| 6779 | * | ||||
| 6780 | // Input: | ||||
| 6781 | // c_rarg0 - x address | ||||
| 6782 | // c_rarg1 - x length | ||||
| 6783 | // c_rarg2 - z address | ||||
| 6784 | // c_rarg3 - z lenth | ||||
| 6785 | * | ||||
| 6786 | */ | ||||
| 6787 | address generate_squareToLen() { | ||||
| 6788 | |||||
| 6789 | __masm-> align(CodeEntryAlignment); | ||||
| 6790 | StubCodeMark mark(this, "StubRoutines", "squareToLen"); | ||||
| 6791 | |||||
| 6792 | address start = __masm-> pc(); | ||||
| 6793 | // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6794 | // Unix: rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...) | ||||
| 6795 | const Register x = rdi; | ||||
| 6796 | const Register len = rsi; | ||||
| 6797 | const Register z = r8; | ||||
| 6798 | const Register zlen = rcx; | ||||
| 6799 | |||||
| 6800 | const Register tmp1 = r12; | ||||
| 6801 | const Register tmp2 = r13; | ||||
| 6802 | const Register tmp3 = r14; | ||||
| 6803 | const Register tmp4 = r15; | ||||
| 6804 | const Register tmp5 = rbx; | ||||
| 6805 | |||||
| 6806 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6807 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6808 | |||||
| 6809 | setup_arg_regs(4); // x => rdi, len => rsi, z => rdx | ||||
| 6810 | // zlen => rcx | ||||
| 6811 | // r9 and r10 may be used to save non-volatile registers | ||||
| 6812 | __masm-> movptr(r8, rdx); | ||||
| 6813 | __masm-> square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); | ||||
| 6814 | |||||
| 6815 | restore_arg_regs(); | ||||
| 6816 | |||||
| 6817 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6818 | __masm-> ret(0); | ||||
| 6819 | |||||
| 6820 | return start; | ||||
| 6821 | } | ||||
| 6822 | |||||
| 6823 | address generate_method_entry_barrier() { | ||||
| 6824 | __masm-> align(CodeEntryAlignment); | ||||
| 6825 | StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier"); | ||||
| 6826 | |||||
| 6827 | Label deoptimize_label; | ||||
| 6828 | |||||
| 6829 | address start = __masm-> pc(); | ||||
| 6830 | |||||
| 6831 | __masm-> push(-1); // cookie, this is used for writing the new rsp when deoptimizing | ||||
| 6832 | |||||
| 6833 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6834 | __masm-> enter(); // save rbp | ||||
| 6835 | |||||
| 6836 | // save c_rarg0, because we want to use that value. | ||||
| 6837 | // We could do without it but then we depend on the number of slots used by pusha | ||||
| 6838 | __masm-> push(c_rarg0); | ||||
| 6839 | |||||
| 6840 | __masm-> lea(c_rarg0, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for c_rarg0 - this should be the return address | ||||
| 6841 | |||||
| 6842 | __masm-> pusha(); | ||||
| 6843 | |||||
| 6844 | // The method may have floats as arguments, and we must spill them before calling | ||||
| 6845 | // the VM runtime. | ||||
| 6846 | assert(Argument::n_float_register_parameters_j == 8, "Assumption")do { if (!(Argument::n_float_register_parameters_j == 8)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 6846, "assert(" "Argument::n_float_register_parameters_j == 8" ") failed", "Assumption"); ::breakpoint(); } } while (0); | ||||
| 6847 | const int xmm_size = wordSize * 2; | ||||
| 6848 | const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j; | ||||
| 6849 | __masm-> subptr(rsp, xmm_spill_size); | ||||
| 6850 | __masm-> movdqu(Address(rsp, xmm_size * 7), xmm7); | ||||
| 6851 | __masm-> movdqu(Address(rsp, xmm_size * 6), xmm6); | ||||
| 6852 | __masm-> movdqu(Address(rsp, xmm_size * 5), xmm5); | ||||
| 6853 | __masm-> movdqu(Address(rsp, xmm_size * 4), xmm4); | ||||
| 6854 | __masm-> movdqu(Address(rsp, xmm_size * 3), xmm3); | ||||
| 6855 | __masm-> movdqu(Address(rsp, xmm_size * 2), xmm2); | ||||
| 6856 | __masm-> movdqu(Address(rsp, xmm_size * 1), xmm1); | ||||
| 6857 | __masm-> movdqu(Address(rsp, xmm_size * 0), xmm0); | ||||
| 6858 | |||||
| 6859 | __masm-> call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier))((address)((address_word)(static_cast<int (*)(address*)> (BarrierSetNMethod::nmethod_stub_entry_barrier)))), 1); | ||||
| 6860 | |||||
| 6861 | __masm-> movdqu(xmm0, Address(rsp, xmm_size * 0)); | ||||
| 6862 | __masm-> movdqu(xmm1, Address(rsp, xmm_size * 1)); | ||||
| 6863 | __masm-> movdqu(xmm2, Address(rsp, xmm_size * 2)); | ||||
| 6864 | __masm-> movdqu(xmm3, Address(rsp, xmm_size * 3)); | ||||
| 6865 | __masm-> movdqu(xmm4, Address(rsp, xmm_size * 4)); | ||||
| 6866 | __masm-> movdqu(xmm5, Address(rsp, xmm_size * 5)); | ||||
| 6867 | __masm-> movdqu(xmm6, Address(rsp, xmm_size * 6)); | ||||
| 6868 | __masm-> movdqu(xmm7, Address(rsp, xmm_size * 7)); | ||||
| 6869 | __masm-> addptr(rsp, xmm_spill_size); | ||||
| 6870 | |||||
| 6871 | __masm-> cmpl(rax, 1); // 1 means deoptimize | ||||
| 6872 | __masm-> jcc(Assembler::equal, deoptimize_label); | ||||
| 6873 | |||||
| 6874 | __masm-> popa(); | ||||
| 6875 | __masm-> pop(c_rarg0); | ||||
| 6876 | |||||
| 6877 | __masm-> leave(); | ||||
| 6878 | |||||
| 6879 | __masm-> addptr(rsp, 1 * wordSize); // cookie | ||||
| 6880 | __masm-> ret(0); | ||||
| 6881 | |||||
| 6882 | |||||
| 6883 | __masm-> BIND(deoptimize_label)bind(deoptimize_label); masm-> block_comment("deoptimize_label" ":"); | ||||
| 6884 | |||||
| 6885 | __masm-> popa(); | ||||
| 6886 | __masm-> pop(c_rarg0); | ||||
| 6887 | |||||
| 6888 | __masm-> leave(); | ||||
| 6889 | |||||
| 6890 | // this can be taken out, but is good for verification purposes. getting a SIGSEGV | ||||
| 6891 | // here while still having a correct stack is valuable | ||||
| 6892 | __masm-> testptr(rsp, Address(rsp, 0)); | ||||
| 6893 | |||||
| 6894 | __masm-> movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier | ||||
| 6895 | __masm-> jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point | ||||
| 6896 | |||||
| 6897 | return start; | ||||
| 6898 | } | ||||
| 6899 | |||||
| 6900 | /** | ||||
| 6901 | * Arguments: | ||||
| 6902 | * | ||||
| 6903 | * Input: | ||||
| 6904 | * c_rarg0 - out address | ||||
| 6905 | * c_rarg1 - in address | ||||
| 6906 | * c_rarg2 - offset | ||||
| 6907 | * c_rarg3 - len | ||||
| 6908 | * not Win64 | ||||
| 6909 | * c_rarg4 - k | ||||
| 6910 | * Win64 | ||||
| 6911 | * rsp+40 - k | ||||
| 6912 | */ | ||||
| 6913 | address generate_mulAdd() { | ||||
| 6914 | __masm-> align(CodeEntryAlignment); | ||||
| 6915 | StubCodeMark mark(this, "StubRoutines", "mulAdd"); | ||||
| 6916 | |||||
| 6917 | address start = __masm-> pc(); | ||||
| 6918 | // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6919 | // Unix: rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) | ||||
| 6920 | const Register out = rdi; | ||||
| 6921 | const Register in = rsi; | ||||
| 6922 | const Register offset = r11; | ||||
| 6923 | const Register len = rcx; | ||||
| 6924 | const Register k = r8; | ||||
| 6925 | |||||
| 6926 | // Next registers will be saved on stack in mul_add(). | ||||
| 6927 | const Register tmp1 = r12; | ||||
| 6928 | const Register tmp2 = r13; | ||||
| 6929 | const Register tmp3 = r14; | ||||
| 6930 | const Register tmp4 = r15; | ||||
| 6931 | const Register tmp5 = rbx; | ||||
| 6932 | |||||
| 6933 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6934 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6935 | |||||
| 6936 | setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx | ||||
| 6937 | // len => rcx, k => r8 | ||||
| 6938 | // r9 and r10 may be used to save non-volatile registers | ||||
| 6939 | #ifdef _WIN64 | ||||
| 6940 | // last argument is on stack on Win64 | ||||
| 6941 | __masm-> movl(k, Address(rsp, 6 * wordSize)); | ||||
| 6942 | #endif | ||||
| 6943 | __masm-> movptr(r11, rdx); // move offset in rdx to offset(r11) | ||||
| 6944 | __masm-> mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax); | ||||
| 6945 | |||||
| 6946 | restore_arg_regs(); | ||||
| 6947 | |||||
| 6948 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6949 | __masm-> ret(0); | ||||
| 6950 | |||||
| 6951 | return start; | ||||
| 6952 | } | ||||
| 6953 | |||||
| 6954 | address generate_bigIntegerRightShift() { | ||||
| 6955 | __masm-> align(CodeEntryAlignment); | ||||
| 6956 | StubCodeMark mark(this, "StubRoutines", "bigIntegerRightShiftWorker"); | ||||
| 6957 | |||||
| 6958 | address start = __masm-> pc(); | ||||
| 6959 | Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; | ||||
| 6960 | // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. | ||||
| 6961 | const Register newArr = rdi; | ||||
| 6962 | const Register oldArr = rsi; | ||||
| 6963 | const Register newIdx = rdx; | ||||
| 6964 | const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. | ||||
| 6965 | const Register totalNumIter = r8; | ||||
| 6966 | |||||
| 6967 | // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. | ||||
| 6968 | // For everything else, we prefer using r9 and r10 since we do not have to save them before use. | ||||
| 6969 | const Register tmp1 = r11; // Caller save. | ||||
| 6970 | const Register tmp2 = rax; // Caller save. | ||||
| 6971 | const Register tmp3 = WINDOWS_ONLY(r12) NOT_WINDOWS(r9)r9; // Windows: Callee save. Linux: Caller save. | ||||
| 6972 | const Register tmp4 = WINDOWS_ONLY(r13) NOT_WINDOWS(r10)r10; // Windows: Callee save. Linux: Caller save. | ||||
| 6973 | const Register tmp5 = r14; // Callee save. | ||||
| 6974 | const Register tmp6 = r15; | ||||
| 6975 | |||||
| 6976 | const XMMRegister x0 = xmm0; | ||||
| 6977 | const XMMRegister x1 = xmm1; | ||||
| 6978 | const XMMRegister x2 = xmm2; | ||||
| 6979 | |||||
| 6980 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 6981 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 6982 | |||||
| 6983 | #ifdef _WINDOWS | ||||
| 6984 | setup_arg_regs(4); | ||||
| 6985 | // For windows, since last argument is on stack, we need to move it to the appropriate register. | ||||
| 6986 | __masm-> movl(totalNumIter, Address(rsp, 6 * wordSize)); | ||||
| 6987 | // Save callee save registers. | ||||
| 6988 | __masm-> push(tmp3); | ||||
| 6989 | __masm-> push(tmp4); | ||||
| 6990 | #endif | ||||
| 6991 | __masm-> push(tmp5); | ||||
| 6992 | |||||
| 6993 | // Rename temps used throughout the code. | ||||
| 6994 | const Register idx = tmp1; | ||||
| 6995 | const Register nIdx = tmp2; | ||||
| 6996 | |||||
| 6997 | __masm-> xorl(idx, idx); | ||||
| 6998 | |||||
| 6999 | // Start right shift from end of the array. | ||||
| 7000 | // For example, if #iteration = 4 and newIdx = 1 | ||||
| 7001 | // then dest[4] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) | ||||
| 7002 | // if #iteration = 4 and newIdx = 0 | ||||
| 7003 | // then dest[3] = src[4] >> shiftCount | src[3] <<< (shiftCount - 32) | ||||
| 7004 | __masm-> movl(idx, totalNumIter); | ||||
| 7005 | __masm-> movl(nIdx, idx); | ||||
| 7006 | __masm-> addl(nIdx, newIdx); | ||||
| 7007 | |||||
| 7008 | // If vectorization is enabled, check if the number of iterations is at least 64 | ||||
| 7009 | // If not, then go to ShifTwo processing 2 iterations | ||||
| 7010 | if (VM_Version::supports_avx512_vbmi2()) { | ||||
| 7011 | __masm-> cmpptr(totalNumIter, (AVX3Threshold/64)); | ||||
| 7012 | __masm-> jcc(Assembler::less, ShiftTwo); | ||||
| 7013 | |||||
| 7014 | if (AVX3Threshold < 16 * 64) { | ||||
| 7015 | __masm-> cmpl(totalNumIter, 16); | ||||
| 7016 | __masm-> jcc(Assembler::less, ShiftTwo); | ||||
| 7017 | } | ||||
| 7018 | __masm-> evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); | ||||
| 7019 | __masm-> subl(idx, 16); | ||||
| 7020 | __masm-> subl(nIdx, 16); | ||||
| 7021 | __masm-> BIND(Shift512Loop)bind(Shift512Loop); masm-> block_comment("Shift512Loop" ":" ); | ||||
| 7022 | __masm-> evmovdqul(x2, Address(oldArr, idx, Address::times_4, 4), Assembler::AVX_512bit); | ||||
| 7023 | __masm-> evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); | ||||
| 7024 | __masm-> vpshrdvd(x2, x1, x0, Assembler::AVX_512bit); | ||||
| 7025 | __masm-> evmovdqul(Address(newArr, nIdx, Address::times_4), x2, Assembler::AVX_512bit); | ||||
| 7026 | __masm-> subl(nIdx, 16); | ||||
| 7027 | __masm-> subl(idx, 16); | ||||
| 7028 | __masm-> jcc(Assembler::greaterEqual, Shift512Loop); | ||||
| 7029 | __masm-> addl(idx, 16); | ||||
| 7030 | __masm-> addl(nIdx, 16); | ||||
| 7031 | } | ||||
| 7032 | __masm-> BIND(ShiftTwo)bind(ShiftTwo); masm-> block_comment("ShiftTwo" ":"); | ||||
| 7033 | __masm-> cmpl(idx, 2); | ||||
| 7034 | __masm-> jcc(Assembler::less, ShiftOne); | ||||
| 7035 | __masm-> subl(idx, 2); | ||||
| 7036 | __masm-> subl(nIdx, 2); | ||||
| 7037 | __masm-> BIND(ShiftTwoLoop)bind(ShiftTwoLoop); masm-> block_comment("ShiftTwoLoop" ":" ); | ||||
| 7038 | __masm-> movl(tmp5, Address(oldArr, idx, Address::times_4, 8)); | ||||
| 7039 | __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); | ||||
| 7040 | __masm-> movl(tmp3, Address(oldArr, idx, Address::times_4)); | ||||
| 7041 | __masm-> shrdl(tmp5, tmp4); | ||||
| 7042 | __masm-> shrdl(tmp4, tmp3); | ||||
| 7043 | __masm-> movl(Address(newArr, nIdx, Address::times_4, 4), tmp5); | ||||
| 7044 | __masm-> movl(Address(newArr, nIdx, Address::times_4), tmp4); | ||||
| 7045 | __masm-> subl(nIdx, 2); | ||||
| 7046 | __masm-> subl(idx, 2); | ||||
| 7047 | __masm-> jcc(Assembler::greaterEqual, ShiftTwoLoop); | ||||
| 7048 | __masm-> addl(idx, 2); | ||||
| 7049 | __masm-> addl(nIdx, 2); | ||||
| 7050 | |||||
| 7051 | // Do the last iteration | ||||
| 7052 | __masm-> BIND(ShiftOne)bind(ShiftOne); masm-> block_comment("ShiftOne" ":"); | ||||
| 7053 | __masm-> cmpl(idx, 1); | ||||
| 7054 | __masm-> jcc(Assembler::less, Exit); | ||||
| 7055 | __masm-> subl(idx, 1); | ||||
| 7056 | __masm-> subl(nIdx, 1); | ||||
| 7057 | __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 4)); | ||||
| 7058 | __masm-> movl(tmp3, Address(oldArr, idx, Address::times_4)); | ||||
| 7059 | __masm-> shrdl(tmp4, tmp3); | ||||
| 7060 | __masm-> movl(Address(newArr, nIdx, Address::times_4), tmp4); | ||||
| 7061 | __masm-> BIND(Exit)bind(Exit); masm-> block_comment("Exit" ":"); | ||||
| 7062 | // Restore callee save registers. | ||||
| 7063 | __masm-> pop(tmp5); | ||||
| 7064 | #ifdef _WINDOWS | ||||
| 7065 | __masm-> pop(tmp4); | ||||
| 7066 | __masm-> pop(tmp3); | ||||
| 7067 | restore_arg_regs(); | ||||
| 7068 | #endif | ||||
| 7069 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7070 | __masm-> ret(0); | ||||
| 7071 | return start; | ||||
| 7072 | } | ||||
| 7073 | |||||
| 7074 | /** | ||||
| 7075 | * Arguments: | ||||
| 7076 | * | ||||
| 7077 | * Input: | ||||
| 7078 | * c_rarg0 - newArr address | ||||
| 7079 | * c_rarg1 - oldArr address | ||||
| 7080 | * c_rarg2 - newIdx | ||||
| 7081 | * c_rarg3 - shiftCount | ||||
| 7082 | * not Win64 | ||||
| 7083 | * c_rarg4 - numIter | ||||
| 7084 | * Win64 | ||||
| 7085 | * rsp40 - numIter | ||||
| 7086 | */ | ||||
| 7087 | address generate_bigIntegerLeftShift() { | ||||
| 7088 | __masm-> align(CodeEntryAlignment); | ||||
| 7089 | StubCodeMark mark(this, "StubRoutines", "bigIntegerLeftShiftWorker"); | ||||
| 7090 | address start = __masm-> pc(); | ||||
| 7091 | Label Shift512Loop, ShiftTwo, ShiftTwoLoop, ShiftOne, Exit; | ||||
| 7092 | // For Unix, the arguments are as follows: rdi, rsi, rdx, rcx, r8. | ||||
| 7093 | const Register newArr = rdi; | ||||
| 7094 | const Register oldArr = rsi; | ||||
| 7095 | const Register newIdx = rdx; | ||||
| 7096 | const Register shiftCount = rcx; // It was intentional to have shiftCount in rcx since it is used implicitly for shift. | ||||
| 7097 | const Register totalNumIter = r8; | ||||
| 7098 | // For windows, we use r9 and r10 as temps to save rdi and rsi. Thus we cannot allocate them for our temps. | ||||
| 7099 | // For everything else, we prefer using r9 and r10 since we do not have to save them before use. | ||||
| 7100 | const Register tmp1 = r11; // Caller save. | ||||
| 7101 | const Register tmp2 = rax; // Caller save. | ||||
| 7102 | const Register tmp3 = WINDOWS_ONLY(r12) NOT_WINDOWS(r9)r9; // Windows: Callee save. Linux: Caller save. | ||||
| 7103 | const Register tmp4 = WINDOWS_ONLY(r13) NOT_WINDOWS(r10)r10; // Windows: Callee save. Linux: Caller save. | ||||
| 7104 | const Register tmp5 = r14; // Callee save. | ||||
| 7105 | |||||
| 7106 | const XMMRegister x0 = xmm0; | ||||
| 7107 | const XMMRegister x1 = xmm1; | ||||
| 7108 | const XMMRegister x2 = xmm2; | ||||
| 7109 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7110 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7111 | |||||
| 7112 | #ifdef _WINDOWS | ||||
| 7113 | setup_arg_regs(4); | ||||
| 7114 | // For windows, since last argument is on stack, we need to move it to the appropriate register. | ||||
| 7115 | __masm-> movl(totalNumIter, Address(rsp, 6 * wordSize)); | ||||
| 7116 | // Save callee save registers. | ||||
| 7117 | __masm-> push(tmp3); | ||||
| 7118 | __masm-> push(tmp4); | ||||
| 7119 | #endif | ||||
| 7120 | __masm-> push(tmp5); | ||||
| 7121 | |||||
| 7122 | // Rename temps used throughout the code | ||||
| 7123 | const Register idx = tmp1; | ||||
| 7124 | const Register numIterTmp = tmp2; | ||||
| 7125 | |||||
| 7126 | // Start idx from zero. | ||||
| 7127 | __masm-> xorl(idx, idx); | ||||
| 7128 | // Compute interior pointer for new array. We do this so that we can use same index for both old and new arrays. | ||||
| 7129 | __masm-> lea(newArr, Address(newArr, newIdx, Address::times_4)); | ||||
| 7130 | __masm-> movl(numIterTmp, totalNumIter); | ||||
| 7131 | |||||
| 7132 | // If vectorization is enabled, check if the number of iterations is at least 64 | ||||
| 7133 | // If not, then go to ShiftTwo shifting two numbers at a time | ||||
| 7134 | if (VM_Version::supports_avx512_vbmi2()) { | ||||
| 7135 | __masm-> cmpl(totalNumIter, (AVX3Threshold/64)); | ||||
| 7136 | __masm-> jcc(Assembler::less, ShiftTwo); | ||||
| 7137 | |||||
| 7138 | if (AVX3Threshold < 16 * 64) { | ||||
| 7139 | __masm-> cmpl(totalNumIter, 16); | ||||
| 7140 | __masm-> jcc(Assembler::less, ShiftTwo); | ||||
| 7141 | } | ||||
| 7142 | __masm-> evpbroadcastd(x0, shiftCount, Assembler::AVX_512bit); | ||||
| 7143 | __masm-> subl(numIterTmp, 16); | ||||
| 7144 | __masm-> BIND(Shift512Loop)bind(Shift512Loop); masm-> block_comment("Shift512Loop" ":" ); | ||||
| 7145 | __masm-> evmovdqul(x1, Address(oldArr, idx, Address::times_4), Assembler::AVX_512bit); | ||||
| 7146 | __masm-> evmovdqul(x2, Address(oldArr, idx, Address::times_4, 0x4), Assembler::AVX_512bit); | ||||
| 7147 | __masm-> vpshldvd(x1, x2, x0, Assembler::AVX_512bit); | ||||
| 7148 | __masm-> evmovdqul(Address(newArr, idx, Address::times_4), x1, Assembler::AVX_512bit); | ||||
| 7149 | __masm-> addl(idx, 16); | ||||
| 7150 | __masm-> subl(numIterTmp, 16); | ||||
| 7151 | __masm-> jcc(Assembler::greaterEqual, Shift512Loop); | ||||
| 7152 | __masm-> addl(numIterTmp, 16); | ||||
| 7153 | } | ||||
| 7154 | __masm-> BIND(ShiftTwo)bind(ShiftTwo); masm-> block_comment("ShiftTwo" ":"); | ||||
| 7155 | __masm-> cmpl(totalNumIter, 1); | ||||
| 7156 | __masm-> jcc(Assembler::less, Exit); | ||||
| 7157 | __masm-> movl(tmp3, Address(oldArr, idx, Address::times_4)); | ||||
| 7158 | __masm-> subl(numIterTmp, 2); | ||||
| 7159 | __masm-> jcc(Assembler::less, ShiftOne); | ||||
| 7160 | |||||
| 7161 | __masm-> BIND(ShiftTwoLoop)bind(ShiftTwoLoop); masm-> block_comment("ShiftTwoLoop" ":" ); | ||||
| 7162 | __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); | ||||
| 7163 | __masm-> movl(tmp5, Address(oldArr, idx, Address::times_4, 0x8)); | ||||
| 7164 | __masm-> shldl(tmp3, tmp4); | ||||
| 7165 | __masm-> shldl(tmp4, tmp5); | ||||
| 7166 | __masm-> movl(Address(newArr, idx, Address::times_4), tmp3); | ||||
| 7167 | __masm-> movl(Address(newArr, idx, Address::times_4, 0x4), tmp4); | ||||
| 7168 | __masm-> movl(tmp3, tmp5); | ||||
| 7169 | __masm-> addl(idx, 2); | ||||
| 7170 | __masm-> subl(numIterTmp, 2); | ||||
| 7171 | __masm-> jcc(Assembler::greaterEqual, ShiftTwoLoop); | ||||
| 7172 | |||||
| 7173 | // Do the last iteration | ||||
| 7174 | __masm-> BIND(ShiftOne)bind(ShiftOne); masm-> block_comment("ShiftOne" ":"); | ||||
| 7175 | __masm-> addl(numIterTmp, 2); | ||||
| 7176 | __masm-> cmpl(numIterTmp, 1); | ||||
| 7177 | __masm-> jcc(Assembler::less, Exit); | ||||
| 7178 | __masm-> movl(tmp4, Address(oldArr, idx, Address::times_4, 0x4)); | ||||
| 7179 | __masm-> shldl(tmp3, tmp4); | ||||
| 7180 | __masm-> movl(Address(newArr, idx, Address::times_4), tmp3); | ||||
| 7181 | |||||
| 7182 | __masm-> BIND(Exit)bind(Exit); masm-> block_comment("Exit" ":"); | ||||
| 7183 | // Restore callee save registers. | ||||
| 7184 | __masm-> pop(tmp5); | ||||
| 7185 | #ifdef _WINDOWS | ||||
| 7186 | __masm-> pop(tmp4); | ||||
| 7187 | __masm-> pop(tmp3); | ||||
| 7188 | restore_arg_regs(); | ||||
| 7189 | #endif | ||||
| 7190 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7191 | __masm-> ret(0); | ||||
| 7192 | return start; | ||||
| 7193 | } | ||||
| 7194 | |||||
| 7195 | address generate_libmExp() { | ||||
| 7196 | StubCodeMark mark(this, "StubRoutines", "libmExp"); | ||||
| 7197 | |||||
| 7198 | address start = __masm-> pc(); | ||||
| 7199 | |||||
| 7200 | const XMMRegister x0 = xmm0; | ||||
| 7201 | const XMMRegister x1 = xmm1; | ||||
| 7202 | const XMMRegister x2 = xmm2; | ||||
| 7203 | const XMMRegister x3 = xmm3; | ||||
| 7204 | |||||
| 7205 | const XMMRegister x4 = xmm4; | ||||
| 7206 | const XMMRegister x5 = xmm5; | ||||
| 7207 | const XMMRegister x6 = xmm6; | ||||
| 7208 | const XMMRegister x7 = xmm7; | ||||
| 7209 | |||||
| 7210 | const Register tmp = r11; | ||||
| 7211 | |||||
| 7212 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7213 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7214 | |||||
| 7215 | __masm-> fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp); | ||||
| 7216 | |||||
| 7217 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7218 | __masm-> ret(0); | ||||
| 7219 | |||||
| 7220 | return start; | ||||
| 7221 | |||||
| 7222 | } | ||||
| 7223 | |||||
| 7224 | address generate_libmLog() { | ||||
| 7225 | StubCodeMark mark(this, "StubRoutines", "libmLog"); | ||||
| 7226 | |||||
| 7227 | address start = __masm-> pc(); | ||||
| 7228 | |||||
| 7229 | const XMMRegister x0 = xmm0; | ||||
| 7230 | const XMMRegister x1 = xmm1; | ||||
| 7231 | const XMMRegister x2 = xmm2; | ||||
| 7232 | const XMMRegister x3 = xmm3; | ||||
| 7233 | |||||
| 7234 | const XMMRegister x4 = xmm4; | ||||
| 7235 | const XMMRegister x5 = xmm5; | ||||
| 7236 | const XMMRegister x6 = xmm6; | ||||
| 7237 | const XMMRegister x7 = xmm7; | ||||
| 7238 | |||||
| 7239 | const Register tmp1 = r11; | ||||
| 7240 | const Register tmp2 = r8; | ||||
| 7241 | |||||
| 7242 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7243 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7244 | |||||
| 7245 | __masm-> fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2); | ||||
| 7246 | |||||
| 7247 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7248 | __masm-> ret(0); | ||||
| 7249 | |||||
| 7250 | return start; | ||||
| 7251 | |||||
| 7252 | } | ||||
| 7253 | |||||
| 7254 | address generate_libmLog10() { | ||||
| 7255 | StubCodeMark mark(this, "StubRoutines", "libmLog10"); | ||||
| 7256 | |||||
| 7257 | address start = __masm-> pc(); | ||||
| 7258 | |||||
| 7259 | const XMMRegister x0 = xmm0; | ||||
| 7260 | const XMMRegister x1 = xmm1; | ||||
| 7261 | const XMMRegister x2 = xmm2; | ||||
| 7262 | const XMMRegister x3 = xmm3; | ||||
| 7263 | |||||
| 7264 | const XMMRegister x4 = xmm4; | ||||
| 7265 | const XMMRegister x5 = xmm5; | ||||
| 7266 | const XMMRegister x6 = xmm6; | ||||
| 7267 | const XMMRegister x7 = xmm7; | ||||
| 7268 | |||||
| 7269 | const Register tmp = r11; | ||||
| 7270 | |||||
| 7271 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7272 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7273 | |||||
| 7274 | __masm-> fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp); | ||||
| 7275 | |||||
| 7276 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7277 | __masm-> ret(0); | ||||
| 7278 | |||||
| 7279 | return start; | ||||
| 7280 | |||||
| 7281 | } | ||||
| 7282 | |||||
| 7283 | address generate_libmPow() { | ||||
| 7284 | StubCodeMark mark(this, "StubRoutines", "libmPow"); | ||||
| 7285 | |||||
| 7286 | address start = __masm-> pc(); | ||||
| 7287 | |||||
| 7288 | const XMMRegister x0 = xmm0; | ||||
| 7289 | const XMMRegister x1 = xmm1; | ||||
| 7290 | const XMMRegister x2 = xmm2; | ||||
| 7291 | const XMMRegister x3 = xmm3; | ||||
| 7292 | |||||
| 7293 | const XMMRegister x4 = xmm4; | ||||
| 7294 | const XMMRegister x5 = xmm5; | ||||
| 7295 | const XMMRegister x6 = xmm6; | ||||
| 7296 | const XMMRegister x7 = xmm7; | ||||
| 7297 | |||||
| 7298 | const Register tmp1 = r8; | ||||
| 7299 | const Register tmp2 = r9; | ||||
| 7300 | const Register tmp3 = r10; | ||||
| 7301 | const Register tmp4 = r11; | ||||
| 7302 | |||||
| 7303 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7304 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7305 | |||||
| 7306 | __masm-> fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4); | ||||
| 7307 | |||||
| 7308 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7309 | __masm-> ret(0); | ||||
| 7310 | |||||
| 7311 | return start; | ||||
| 7312 | |||||
| 7313 | } | ||||
| 7314 | |||||
| 7315 | address generate_libmSin() { | ||||
| 7316 | StubCodeMark mark(this, "StubRoutines", "libmSin"); | ||||
| 7317 | |||||
| 7318 | address start = __masm-> pc(); | ||||
| 7319 | |||||
| 7320 | const XMMRegister x0 = xmm0; | ||||
| 7321 | const XMMRegister x1 = xmm1; | ||||
| 7322 | const XMMRegister x2 = xmm2; | ||||
| 7323 | const XMMRegister x3 = xmm3; | ||||
| 7324 | |||||
| 7325 | const XMMRegister x4 = xmm4; | ||||
| 7326 | const XMMRegister x5 = xmm5; | ||||
| 7327 | const XMMRegister x6 = xmm6; | ||||
| 7328 | const XMMRegister x7 = xmm7; | ||||
| 7329 | |||||
| 7330 | const Register tmp1 = r8; | ||||
| 7331 | const Register tmp2 = r9; | ||||
| 7332 | const Register tmp3 = r10; | ||||
| 7333 | const Register tmp4 = r11; | ||||
| 7334 | |||||
| 7335 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7336 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7337 | |||||
| 7338 | #ifdef _WIN64 | ||||
| 7339 | __masm-> push(rsi); | ||||
| 7340 | __masm-> push(rdi); | ||||
| 7341 | #endif | ||||
| 7342 | __masm-> fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rcx, rdx, tmp1, tmp2, tmp3, tmp4); | ||||
| 7343 | |||||
| 7344 | #ifdef _WIN64 | ||||
| 7345 | __masm-> pop(rdi); | ||||
| 7346 | __masm-> pop(rsi); | ||||
| 7347 | #endif | ||||
| 7348 | |||||
| 7349 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7350 | __masm-> ret(0); | ||||
| 7351 | |||||
| 7352 | return start; | ||||
| 7353 | |||||
| 7354 | } | ||||
| 7355 | |||||
| 7356 | address generate_libmCos() { | ||||
| 7357 | StubCodeMark mark(this, "StubRoutines", "libmCos"); | ||||
| 7358 | |||||
| 7359 | address start = __masm-> pc(); | ||||
| 7360 | |||||
| 7361 | const XMMRegister x0 = xmm0; | ||||
| 7362 | const XMMRegister x1 = xmm1; | ||||
| 7363 | const XMMRegister x2 = xmm2; | ||||
| 7364 | const XMMRegister x3 = xmm3; | ||||
| 7365 | |||||
| 7366 | const XMMRegister x4 = xmm4; | ||||
| 7367 | const XMMRegister x5 = xmm5; | ||||
| 7368 | const XMMRegister x6 = xmm6; | ||||
| 7369 | const XMMRegister x7 = xmm7; | ||||
| 7370 | |||||
| 7371 | const Register tmp1 = r8; | ||||
| 7372 | const Register tmp2 = r9; | ||||
| 7373 | const Register tmp3 = r10; | ||||
| 7374 | const Register tmp4 = r11; | ||||
| 7375 | |||||
| 7376 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7377 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7378 | |||||
| 7379 | #ifdef _WIN64 | ||||
| 7380 | __masm-> push(rsi); | ||||
| 7381 | __masm-> push(rdi); | ||||
| 7382 | #endif | ||||
| 7383 | __masm-> fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4); | ||||
| 7384 | |||||
| 7385 | #ifdef _WIN64 | ||||
| 7386 | __masm-> pop(rdi); | ||||
| 7387 | __masm-> pop(rsi); | ||||
| 7388 | #endif | ||||
| 7389 | |||||
| 7390 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7391 | __masm-> ret(0); | ||||
| 7392 | |||||
| 7393 | return start; | ||||
| 7394 | |||||
| 7395 | } | ||||
| 7396 | |||||
| 7397 | address generate_libmTan() { | ||||
| 7398 | StubCodeMark mark(this, "StubRoutines", "libmTan"); | ||||
| 7399 | |||||
| 7400 | address start = __masm-> pc(); | ||||
| 7401 | |||||
| 7402 | const XMMRegister x0 = xmm0; | ||||
| 7403 | const XMMRegister x1 = xmm1; | ||||
| 7404 | const XMMRegister x2 = xmm2; | ||||
| 7405 | const XMMRegister x3 = xmm3; | ||||
| 7406 | |||||
| 7407 | const XMMRegister x4 = xmm4; | ||||
| 7408 | const XMMRegister x5 = xmm5; | ||||
| 7409 | const XMMRegister x6 = xmm6; | ||||
| 7410 | const XMMRegister x7 = xmm7; | ||||
| 7411 | |||||
| 7412 | const Register tmp1 = r8; | ||||
| 7413 | const Register tmp2 = r9; | ||||
| 7414 | const Register tmp3 = r10; | ||||
| 7415 | const Register tmp4 = r11; | ||||
| 7416 | |||||
| 7417 | BLOCK_COMMENT("Entry:")masm-> block_comment("Entry:"); | ||||
| 7418 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7419 | |||||
| 7420 | #ifdef _WIN64 | ||||
| 7421 | __masm-> push(rsi); | ||||
| 7422 | __masm-> push(rdi); | ||||
| 7423 | #endif | ||||
| 7424 | __masm-> fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4); | ||||
| 7425 | |||||
| 7426 | #ifdef _WIN64 | ||||
| 7427 | __masm-> pop(rdi); | ||||
| 7428 | __masm-> pop(rsi); | ||||
| 7429 | #endif | ||||
| 7430 | |||||
| 7431 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7432 | __masm-> ret(0); | ||||
| 7433 | |||||
| 7434 | return start; | ||||
| 7435 | |||||
| 7436 | } | ||||
| 7437 | |||||
| 7438 | #undef __masm-> | ||||
| 7439 | #define __masm-> masm-> | ||||
| 7440 | |||||
| 7441 | // Continuation point for throwing of implicit exceptions that are | ||||
| 7442 | // not handled in the current activation. Fabricates an exception | ||||
| 7443 | // oop and initiates normal exception dispatching in this | ||||
| 7444 | // frame. Since we need to preserve callee-saved values (currently | ||||
| 7445 | // only for C2, but done for C1 as well) we need a callee-saved oop | ||||
| 7446 | // map and therefore have to make these stubs into RuntimeStubs | ||||
| 7447 | // rather than BufferBlobs. If the compiler needs all registers to | ||||
| 7448 | // be preserved between the fault point and the exception handler | ||||
| 7449 | // then it must assume responsibility for that in | ||||
| 7450 | // AbstractCompiler::continuation_for_implicit_null_exception or | ||||
| 7451 | // continuation_for_implicit_division_by_zero_exception. All other | ||||
| 7452 | // implicit exceptions (e.g., NullPointerException or | ||||
| 7453 | // AbstractMethodError on entry) are either at call sites or | ||||
| 7454 | // otherwise assume that stack unwinding will be initiated, so | ||||
| 7455 | // caller saved registers were assumed volatile in the compiler. | ||||
| 7456 | address generate_throw_exception(const char* name, | ||||
| 7457 | address runtime_entry, | ||||
| 7458 | Register arg1 = noreg, | ||||
| 7459 | Register arg2 = noreg) { | ||||
| 7460 | // Information about frame layout at time of blocking runtime call. | ||||
| 7461 | // Note that we only have to preserve callee-saved registers since | ||||
| 7462 | // the compilers are responsible for supplying a continuation point | ||||
| 7463 | // if they expect all registers to be preserved. | ||||
| 7464 | enum layout { | ||||
| 7465 | rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt, | ||||
| 7466 | rbp_off2, | ||||
| 7467 | return_off, | ||||
| 7468 | return_off2, | ||||
| 7469 | framesize // inclusive of return address | ||||
| 7470 | }; | ||||
| 7471 | |||||
| 7472 | int insts_size = 512; | ||||
| 7473 | int locs_size = 64; | ||||
| 7474 | |||||
| 7475 | CodeBuffer code(name, insts_size, locs_size); | ||||
| 7476 | OopMapSet* oop_maps = new OopMapSet(); | ||||
| 7477 | MacroAssembler* masm = new MacroAssembler(&code); | ||||
| 7478 | |||||
| 7479 | address start = __masm-> pc(); | ||||
| 7480 | |||||
| 7481 | // This is an inlined and slightly modified version of call_VM | ||||
| 7482 | // which has the ability to fetch the return PC out of | ||||
| 7483 | // thread-local storage and also sets up last_Java_sp slightly | ||||
| 7484 | // differently than the real call_VM | ||||
| 7485 | |||||
| 7486 | __masm-> enter(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7487 | |||||
| 7488 | assert(is_even(framesize/2), "sp not 16-byte aligned")do { if (!(is_even(framesize/2))) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 7488, "assert(" "is_even(framesize/2)" ") failed", "sp not 16-byte aligned" ); ::breakpoint(); } } while (0); | ||||
| 7489 | |||||
| 7490 | // return address and rbp are already in place | ||||
| 7491 | __masm-> subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog | ||||
| 7492 | |||||
| 7493 | int frame_complete = __masm-> pc() - start; | ||||
| 7494 | |||||
| 7495 | // Set up last_Java_sp and last_Java_fp | ||||
| 7496 | address the_pc = __masm-> pc(); | ||||
| 7497 | __masm-> set_last_Java_frame(rsp, rbp, the_pc); | ||||
| 7498 | __masm-> andptr(rsp, -(StackAlignmentInBytes)); // Align stack | ||||
| 7499 | |||||
| 7500 | // Call runtime | ||||
| 7501 | if (arg1 != noreg) { | ||||
| 7502 | assert(arg2 != c_rarg1, "clobbered")do { if (!(arg2 != c_rarg1)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp" , 7502, "assert(" "arg2 != c_rarg1" ") failed", "clobbered"); ::breakpoint(); } } while (0); | ||||
| 7503 | __masm-> movptr(c_rarg1, arg1); | ||||
| 7504 | } | ||||
| 7505 | if (arg2 != noreg) { | ||||
| 7506 | __masm-> movptr(c_rarg2, arg2); | ||||
| 7507 | } | ||||
| 7508 | __masm-> movptr(c_rarg0, r15_thread); | ||||
| 7509 | BLOCK_COMMENT("call runtime_entry")masm-> block_comment("call runtime_entry"); | ||||
| 7510 | __masm-> call(RuntimeAddress(runtime_entry)); | ||||
| 7511 | |||||
| 7512 | // Generate oop map | ||||
| 7513 | OopMap* map = new OopMap(framesize, 0); | ||||
| 7514 | |||||
| 7515 | oop_maps->add_gc_map(the_pc - start, map); | ||||
| 7516 | |||||
| 7517 | __masm-> reset_last_Java_frame(true); | ||||
| 7518 | |||||
| 7519 | __masm-> leave(); // required for proper stackwalking of RuntimeStub frame | ||||
| 7520 | |||||
| 7521 | // check for pending exceptions | ||||
| 7522 | #ifdef ASSERT1 | ||||
| 7523 | Label L; | ||||
| 7524 | __masm-> cmpptr(Address(r15_thread, Thread::pending_exception_offset()), | ||||
| 7525 | (int32_t) NULL_WORD0L); | ||||
| 7526 | __masm-> jcc(Assembler::notEqual, L); | ||||
| 7527 | __masm-> should_not_reach_here(); | ||||
| 7528 | __masm-> bind(L); | ||||
| 7529 | #endif // ASSERT | ||||
| 7530 | __masm-> jump(RuntimeAddress(StubRoutines::forward_exception_entry())); | ||||
| 7531 | |||||
| 7532 | |||||
| 7533 | // codeBlob framesize is in words (not VMRegImpl::slot_size) | ||||
| 7534 | RuntimeStub* stub = | ||||
| 7535 | RuntimeStub::new_runtime_stub(name, | ||||
| 7536 | &code, | ||||
| 7537 | frame_complete, | ||||
| 7538 | (framesize >> (LogBytesPerWord - LogBytesPerInt)), | ||||
| 7539 | oop_maps, false); | ||||
| 7540 | return stub->entry_point(); | ||||
| 7541 | } | ||||
| 7542 | |||||
| 7543 | void create_control_words() { | ||||
| 7544 | // Round to nearest, 64-bit mode, exceptions masked | ||||
| 7545 | StubRoutines::x86::_mxcsr_std = 0x1F80; | ||||
| 7546 | } | ||||
| 7547 | |||||
| 7548 | // Initialization | ||||
| 7549 | void generate_initial() { | ||||
| 7550 | // Generates all stubs and initializes the entry points | ||||
| 7551 | |||||
| 7552 | // This platform-specific settings are needed by generate_call_stub() | ||||
| 7553 | create_control_words(); | ||||
| 7554 | |||||
| 7555 | // entry points that exist in all platforms Note: This is code | ||||
| 7556 | // that could be shared among different platforms - however the | ||||
| 7557 | // benefit seems to be smaller than the disadvantage of having a | ||||
| 7558 | // much more complicated generator structure. See also comment in | ||||
| 7559 | // stubRoutines.hpp. | ||||
| 7560 | |||||
| 7561 | StubRoutines::_forward_exception_entry = generate_forward_exception(); | ||||
| 7562 | |||||
| 7563 | StubRoutines::_call_stub_entry = | ||||
| 7564 | generate_call_stub(StubRoutines::_call_stub_return_address); | ||||
| 7565 | |||||
| 7566 | // is referenced by megamorphic call | ||||
| 7567 | StubRoutines::_catch_exception_entry = generate_catch_exception(); | ||||
| 7568 | |||||
| 7569 | // atomic calls | ||||
| 7570 | StubRoutines::_fence_entry = generate_orderaccess_fence(); | ||||
| 7571 | |||||
| 7572 | // platform dependent | ||||
| 7573 | StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp(); | ||||
| 7574 | |||||
| 7575 | StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr(); | ||||
| 7576 | |||||
| 7577 | StubRoutines::x86::_f2i_fixup = generate_f2i_fixup(); | ||||
| 7578 | StubRoutines::x86::_f2l_fixup = generate_f2l_fixup(); | ||||
| 7579 | StubRoutines::x86::_d2i_fixup = generate_d2i_fixup(); | ||||
| 7580 | StubRoutines::x86::_d2l_fixup = generate_d2l_fixup(); | ||||
| 7581 | |||||
| 7582 | StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF); | ||||
| 7583 | StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000); | ||||
| 7584 | StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF); | ||||
| 7585 | StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000); | ||||
| 7586 | |||||
| 7587 | // Build this early so it's available for the interpreter. | ||||
| 7588 | StubRoutines::_throw_StackOverflowError_entry = | ||||
| 7589 | generate_throw_exception("StackOverflowError throw_exception", | ||||
| 7590 | CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_StackOverflowError ))) | ||||
| 7591 | SharedRuntime::((address)((address_word)(SharedRuntime:: throw_StackOverflowError ))) | ||||
| 7592 | throw_StackOverflowError)((address)((address_word)(SharedRuntime:: throw_StackOverflowError )))); | ||||
| 7593 | StubRoutines::_throw_delayed_StackOverflowError_entry = | ||||
| 7594 | generate_throw_exception("delayed StackOverflowError throw_exception", | ||||
| 7595 | CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_delayed_StackOverflowError ))) | ||||
| 7596 | SharedRuntime::((address)((address_word)(SharedRuntime:: throw_delayed_StackOverflowError ))) | ||||
| 7597 | throw_delayed_StackOverflowError)((address)((address_word)(SharedRuntime:: throw_delayed_StackOverflowError )))); | ||||
| 7598 | if (UseCRC32Intrinsics) { | ||||
| 7599 | // set table address before stub generation which use it | ||||
| 7600 | StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table; | ||||
| 7601 | StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32(); | ||||
| 7602 | } | ||||
| 7603 | |||||
| 7604 | if (UseCRC32CIntrinsics) { | ||||
| 7605 | bool supports_clmul = VM_Version::supports_clmul(); | ||||
| 7606 | StubRoutines::x86::generate_CRC32C_table(supports_clmul); | ||||
| 7607 | StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table; | ||||
| 7608 | StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul); | ||||
| 7609 | } | ||||
| 7610 | |||||
| 7611 | if (UseAdler32Intrinsics) { | ||||
| 7612 | StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32(); | ||||
| 7613 | } | ||||
| 7614 | |||||
| 7615 | if (UseLibmIntrinsic && InlineIntrinsics) { | ||||
| 7616 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) || | ||||
| 7617 | vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) || | ||||
| 7618 | vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) { | ||||
| 7619 | StubRoutines::x86::_ONEHALF_adr = (address)StubRoutines::x86::_ONEHALF; | ||||
| 7620 | StubRoutines::x86::_P_2_adr = (address)StubRoutines::x86::_P_2; | ||||
| 7621 | StubRoutines::x86::_SC_4_adr = (address)StubRoutines::x86::_SC_4; | ||||
| 7622 | StubRoutines::x86::_Ctable_adr = (address)StubRoutines::x86::_Ctable; | ||||
| 7623 | StubRoutines::x86::_SC_2_adr = (address)StubRoutines::x86::_SC_2; | ||||
| 7624 | StubRoutines::x86::_SC_3_adr = (address)StubRoutines::x86::_SC_3; | ||||
| 7625 | StubRoutines::x86::_SC_1_adr = (address)StubRoutines::x86::_SC_1; | ||||
| 7626 | StubRoutines::x86::_PI_INV_TABLE_adr = (address)StubRoutines::x86::_PI_INV_TABLE; | ||||
| 7627 | StubRoutines::x86::_PI_4_adr = (address)StubRoutines::x86::_PI_4; | ||||
| 7628 | StubRoutines::x86::_PI32INV_adr = (address)StubRoutines::x86::_PI32INV; | ||||
| 7629 | StubRoutines::x86::_SIGN_MASK_adr = (address)StubRoutines::x86::_SIGN_MASK; | ||||
| 7630 | StubRoutines::x86::_P_1_adr = (address)StubRoutines::x86::_P_1; | ||||
| 7631 | StubRoutines::x86::_P_3_adr = (address)StubRoutines::x86::_P_3; | ||||
| 7632 | StubRoutines::x86::_NEG_ZERO_adr = (address)StubRoutines::x86::_NEG_ZERO; | ||||
| 7633 | } | ||||
| 7634 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) { | ||||
| 7635 | StubRoutines::_dexp = generate_libmExp(); | ||||
| 7636 | } | ||||
| 7637 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) { | ||||
| 7638 | StubRoutines::_dlog = generate_libmLog(); | ||||
| 7639 | } | ||||
| 7640 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) { | ||||
| 7641 | StubRoutines::_dlog10 = generate_libmLog10(); | ||||
| 7642 | } | ||||
| 7643 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) { | ||||
| 7644 | StubRoutines::_dpow = generate_libmPow(); | ||||
| 7645 | } | ||||
| 7646 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) { | ||||
| 7647 | StubRoutines::_dsin = generate_libmSin(); | ||||
| 7648 | } | ||||
| 7649 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) { | ||||
| 7650 | StubRoutines::_dcos = generate_libmCos(); | ||||
| 7651 | } | ||||
| 7652 | if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) { | ||||
| 7653 | StubRoutines::_dtan = generate_libmTan(); | ||||
| 7654 | } | ||||
| 7655 | } | ||||
| 7656 | |||||
| 7657 | // Safefetch stubs. | ||||
| 7658 | generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, | ||||
| 7659 | &StubRoutines::_safefetch32_fault_pc, | ||||
| 7660 | &StubRoutines::_safefetch32_continuation_pc); | ||||
| 7661 | generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry, | ||||
| 7662 | &StubRoutines::_safefetchN_fault_pc, | ||||
| 7663 | &StubRoutines::_safefetchN_continuation_pc); | ||||
| 7664 | } | ||||
| 7665 | |||||
| 7666 | void generate_all() { | ||||
| 7667 | // Generates all stubs and initializes the entry points | ||||
| 7668 | |||||
| 7669 | // These entry points require SharedInfo::stack0 to be set up in | ||||
| 7670 | // non-core builds and need to be relocatable, so they each | ||||
| 7671 | // fabricate a RuntimeStub internally. | ||||
| 7672 | StubRoutines::_throw_AbstractMethodError_entry = | ||||
| 7673 | generate_throw_exception("AbstractMethodError throw_exception", | ||||
| 7674 | CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_AbstractMethodError ))) | ||||
| 7675 | SharedRuntime::((address)((address_word)(SharedRuntime:: throw_AbstractMethodError ))) | ||||
| 7676 | throw_AbstractMethodError)((address)((address_word)(SharedRuntime:: throw_AbstractMethodError )))); | ||||
| 7677 | |||||
| 7678 | StubRoutines::_throw_IncompatibleClassChangeError_entry = | ||||
| 7679 | generate_throw_exception("IncompatibleClassChangeError throw_exception", | ||||
| 7680 | CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_IncompatibleClassChangeError ))) | ||||
| 7681 | SharedRuntime::((address)((address_word)(SharedRuntime:: throw_IncompatibleClassChangeError ))) | ||||
| 7682 | throw_IncompatibleClassChangeError)((address)((address_word)(SharedRuntime:: throw_IncompatibleClassChangeError )))); | ||||
| 7683 | |||||
| 7684 | StubRoutines::_throw_NullPointerException_at_call_entry = | ||||
| 7685 | generate_throw_exception("NullPointerException at call throw_exception", | ||||
| 7686 | CAST_FROM_FN_PTR(address,((address)((address_word)(SharedRuntime:: throw_NullPointerException_at_call ))) | ||||
| 7687 | SharedRuntime::((address)((address_word)(SharedRuntime:: throw_NullPointerException_at_call ))) | ||||
| 7688 | throw_NullPointerException_at_call)((address)((address_word)(SharedRuntime:: throw_NullPointerException_at_call )))); | ||||
| 7689 | |||||
| 7690 | // entry points that are platform specific | ||||
| 7691 | StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF); | ||||
| 7692 | StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000); | ||||
| 7693 | StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF); | ||||
| 7694 | StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000); | ||||
| 7695 | StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFFFFFFFFFF); | ||||
| 7696 | StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask("vector_int_mask_cmp_bits", 0x0000000100000001); | ||||
| 7697 | StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff); | ||||
| 7698 | StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask"); | ||||
| 7699 | StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff000000ff); | ||||
| 7700 | StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff0000ffff); | ||||
| 7701 | StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit, | ||||
| 7702 | 0xFFFFFFFF, 0, 0, 0); | ||||
| 7703 | StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit, | ||||
| 7704 | 0xFFFFFFFF, 0xFFFFFFFF, 0, 0); | ||||
| 7705 | StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x0302010003020100); | ||||
| 7706 | StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask"); | ||||
| 7707 | StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x0100010001000100); | ||||
| 7708 | StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask("vector_long_shuffle_mask", 0x0000000100000000); | ||||
| 7709 | StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000); | ||||
| 7710 | StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices"); | ||||
| 7711 | |||||
| 7712 | // support for verify_oop (must happen after universe_init) | ||||
| 7713 | if (VerifyOops) { | ||||
| 7714 | StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); | ||||
| 7715 | } | ||||
| 7716 | |||||
| 7717 | // data cache line writeback | ||||
| 7718 | StubRoutines::_data_cache_writeback = generate_data_cache_writeback(); | ||||
| 7719 | StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync(); | ||||
| 7720 | |||||
| 7721 | // arraycopy stubs used by compilers | ||||
| 7722 | generate_arraycopy_stubs(); | ||||
| 7723 | |||||
| 7724 | // don't bother generating these AES intrinsic stubs unless global flag is set | ||||
| 7725 | if (UseAESIntrinsics) { | ||||
| 7726 | StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others | ||||
| 7727 | StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock(); | ||||
| 7728 | StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock(); | ||||
| 7729 | StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt(); | ||||
| 7730 | if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) { | ||||
| 7731 | StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt(); | ||||
| 7732 | StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt(); | ||||
| 7733 | StubRoutines::_electronicCodeBook_decryptAESCrypt = generate_electronicCodeBook_decryptAESCrypt(); | ||||
| 7734 | StubRoutines::x86::_counter_mask_addr = counter_mask_addr(); | ||||
| 7735 | StubRoutines::x86::_ghash_poly512_addr = ghash_polynomial512_addr(); | ||||
| 7736 | StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask(); | ||||
| 7737 | StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt(); | ||||
| 7738 | } else { | ||||
| 7739 | StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel(); | ||||
| 7740 | } | ||||
| 7741 | } | ||||
| 7742 | |||||
| 7743 | if (UseAESCTRIntrinsics) { | ||||
| 7744 | if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512bw() && VM_Version::supports_avx512vl()) { | ||||
| 7745 | if (StubRoutines::x86::_counter_mask_addr == NULL__null) { | ||||
| 7746 | StubRoutines::x86::_counter_mask_addr = counter_mask_addr(); | ||||
| 7747 | } | ||||
| 7748 | StubRoutines::_counterMode_AESCrypt = generate_counterMode_VectorAESCrypt(); | ||||
| 7749 | } else { | ||||
| 7750 | StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask(); | ||||
| 7751 | StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel(); | ||||
| 7752 | } | ||||
| 7753 | } | ||||
| 7754 | |||||
| 7755 | if (UseMD5Intrinsics) { | ||||
| 7756 | StubRoutines::_md5_implCompress = generate_md5_implCompress(false, "md5_implCompress"); | ||||
| 7757 | StubRoutines::_md5_implCompressMB = generate_md5_implCompress(true, "md5_implCompressMB"); | ||||
| 7758 | } | ||||
| 7759 | if (UseSHA1Intrinsics) { | ||||
| 7760 | StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask(); | ||||
| 7761 | StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask(); | ||||
| 7762 | StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress"); | ||||
| 7763 | StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB"); | ||||
| 7764 | } | ||||
| 7765 | if (UseSHA256Intrinsics) { | ||||
| 7766 | StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256; | ||||
| 7767 | char* dst = (char*)StubRoutines::x86::_k256_W; | ||||
| 7768 | char* src = (char*)StubRoutines::x86::_k256; | ||||
| 7769 | for (int ii = 0; ii < 16; ++ii) { | ||||
| 7770 | memcpy(dst + 32 * ii, src + 16 * ii, 16); | ||||
| 7771 | memcpy(dst + 32 * ii + 16, src + 16 * ii, 16); | ||||
| 7772 | } | ||||
| 7773 | StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W; | ||||
| 7774 | StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask(); | ||||
| 7775 | StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress"); | ||||
| 7776 | StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB"); | ||||
| 7777 | } | ||||
| 7778 | if (UseSHA512Intrinsics) { | ||||
| 7779 | StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W; | ||||
| 7780 | StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512(); | ||||
| 7781 | StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress"); | ||||
| 7782 | StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB"); | ||||
| 7783 | } | ||||
| 7784 | |||||
| 7785 | // Generate GHASH intrinsics code | ||||
| 7786 | if (UseGHASHIntrinsics) { | ||||
| 7787 | if (StubRoutines::x86::_ghash_long_swap_mask_addr == NULL__null) { | ||||
| 7788 | StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask(); | ||||
| 7789 | } | ||||
| 7790 | StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask(); | ||||
| 7791 | if (VM_Version::supports_avx()) { | ||||
| 7792 | StubRoutines::x86::_ghash_shuffmask_addr = ghash_shufflemask_addr(); | ||||
| 7793 | StubRoutines::x86::_ghash_poly_addr = ghash_polynomial_addr(); | ||||
| 7794 | StubRoutines::_ghash_processBlocks = generate_avx_ghash_processBlocks(); | ||||
| 7795 | } else { | ||||
| 7796 | StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(); | ||||
| 7797 | } | ||||
| 7798 | } | ||||
| 7799 | |||||
| 7800 | |||||
| 7801 | if (UseBASE64Intrinsics) { | ||||
| 7802 | if(VM_Version::supports_avx2() && | ||||
| 7803 | VM_Version::supports_avx512bw() && | ||||
| 7804 | VM_Version::supports_avx512vl()) { | ||||
| 7805 | StubRoutines::x86::_avx2_shuffle_base64 = base64_avx2_shuffle_addr(); | ||||
| 7806 | StubRoutines::x86::_avx2_input_mask_base64 = base64_avx2_input_mask_addr(); | ||||
| 7807 | StubRoutines::x86::_avx2_lut_base64 = base64_avx2_lut_addr(); | ||||
| 7808 | } | ||||
| 7809 | StubRoutines::x86::_encoding_table_base64 = base64_encoding_table_addr(); | ||||
| 7810 | if (VM_Version::supports_avx512_vbmi()) { | ||||
| 7811 | StubRoutines::x86::_shuffle_base64 = base64_shuffle_addr(); | ||||
| 7812 | StubRoutines::x86::_lookup_lo_base64 = base64_vbmi_lookup_lo_addr(); | ||||
| 7813 | StubRoutines::x86::_lookup_hi_base64 = base64_vbmi_lookup_hi_addr(); | ||||
| 7814 | StubRoutines::x86::_lookup_lo_base64url = base64_vbmi_lookup_lo_url_addr(); | ||||
| 7815 | StubRoutines::x86::_lookup_hi_base64url = base64_vbmi_lookup_hi_url_addr(); | ||||
| 7816 | StubRoutines::x86::_pack_vec_base64 = base64_vbmi_pack_vec_addr(); | ||||
| 7817 | StubRoutines::x86::_join_0_1_base64 = base64_vbmi_join_0_1_addr(); | ||||
| 7818 | StubRoutines::x86::_join_1_2_base64 = base64_vbmi_join_1_2_addr(); | ||||
| 7819 | StubRoutines::x86::_join_2_3_base64 = base64_vbmi_join_2_3_addr(); | ||||
| 7820 | } | ||||
| 7821 | StubRoutines::x86::_decoding_table_base64 = base64_decoding_table_addr(); | ||||
| 7822 | StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock(); | ||||
| 7823 | StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock(); | ||||
| 7824 | } | ||||
| 7825 | |||||
| 7826 | BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); | ||||
| 7827 | if (bs_nm != NULL__null) { | ||||
| 7828 | StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier(); | ||||
| 7829 | } | ||||
| 7830 | #ifdef COMPILER21 | ||||
| 7831 | if (UseMultiplyToLenIntrinsic) { | ||||
| 7832 | StubRoutines::_multiplyToLen = generate_multiplyToLen(); | ||||
| 7833 | } | ||||
| 7834 | if (UseSquareToLenIntrinsic) { | ||||
| 7835 | StubRoutines::_squareToLen = generate_squareToLen(); | ||||
| 7836 | } | ||||
| 7837 | if (UseMulAddIntrinsic) { | ||||
| 7838 | StubRoutines::_mulAdd = generate_mulAdd(); | ||||
| 7839 | } | ||||
| 7840 | if (VM_Version::supports_avx512_vbmi2()) { | ||||
| 7841 | StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift(); | ||||
| 7842 | StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift(); | ||||
| 7843 | } | ||||
| 7844 | if (UseMontgomeryMultiplyIntrinsic) { | ||||
| 7845 | StubRoutines::_montgomeryMultiply | ||||
| 7846 | = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply)((address)((address_word)(SharedRuntime::montgomery_multiply) )); | ||||
| 7847 | } | ||||
| 7848 | if (UseMontgomerySquareIntrinsic) { | ||||
| 7849 | StubRoutines::_montgomerySquare | ||||
| 7850 | = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square)((address)((address_word)(SharedRuntime::montgomery_square))); | ||||
| 7851 | } | ||||
| 7852 | |||||
| 7853 | // Get svml stub routine addresses | ||||
| 7854 | void *libjsvml = NULL__null; | ||||
| 7855 | char ebuf[1024]; | ||||
| 7856 | char dll_name[JVM_MAXPATHLEN4096 + 1]; | ||||
| 7857 | if (os::dll_locate_lib(dll_name, sizeof(dll_name), Arguments::get_dll_dir(), "jsvml")) { | ||||
| 7858 | libjsvml = os::dll_load(dll_name, ebuf, sizeof ebuf); | ||||
| 7859 | } | ||||
| 7860 | if (libjsvml != NULL__null) { | ||||
| 7861 | // SVML method naming convention | ||||
| 7862 | // All the methods are named as __jsvml_op<T><N>_ha_<VV> | ||||
| 7863 | // Where: | ||||
| 7864 | // ha stands for high accuracy | ||||
| 7865 | // <T> is optional to indicate float/double | ||||
| 7866 | // Set to f for vector float operation | ||||
| 7867 | // Omitted for vector double operation | ||||
| 7868 | // <N> is the number of elements in the vector | ||||
| 7869 | // 1, 2, 4, 8, 16 | ||||
| 7870 | // e.g. 128 bit float vector has 4 float elements | ||||
| 7871 | // <VV> indicates the avx/sse level: | ||||
| 7872 | // z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2 | ||||
| 7873 | // e.g. __jsvml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns | ||||
| 7874 | // __jsvml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns | ||||
| 7875 | |||||
| 7876 | log_info(library)(!(LogImpl<(LogTag::_library), (LogTag::__NO_TAG), (LogTag ::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag:: __NO_TAG)>::is_level(LogLevel::Info))) ? (void)0 : LogImpl <(LogTag::_library), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::write<LogLevel::Info>("Loaded library %s, handle " INTPTR_FORMAT"0x%016" "l" "x", JNI_LIB_PREFIX"lib" "jsvml" JNI_LIB_SUFFIX".so", p2i(libjsvml)); | ||||
| 7877 | if (UseAVX > 2) { | ||||
| 7878 | for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) { | ||||
| 7879 | int vop = VectorSupport::VECTOR_OP_SVML_START + op; | ||||
| 7880 | if ((!VM_Version::supports_avx512dq()) && | ||||
| 7881 | (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) { | ||||
| 7882 | continue; | ||||
| 7883 | } | ||||
| 7884 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf16_ha_z0", VectorSupport::svmlname[op]); | ||||
| 7885 | StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7886 | |||||
| 7887 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%s8_ha_z0", VectorSupport::svmlname[op]); | ||||
| 7888 | StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7889 | } | ||||
| 7890 | } | ||||
| 7891 | const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex"); | ||||
| 7892 | for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) { | ||||
| 7893 | int vop = VectorSupport::VECTOR_OP_SVML_START + op; | ||||
| 7894 | if (vop == VectorSupport::VECTOR_OP_POW) { | ||||
| 7895 | continue; | ||||
| 7896 | } | ||||
| 7897 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); | ||||
| 7898 | StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7899 | |||||
| 7900 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); | ||||
| 7901 | StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7902 | |||||
| 7903 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%sf8_ha_%s", VectorSupport::svmlname[op], avx_sse_str); | ||||
| 7904 | StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7905 | |||||
| 7906 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%s1_ha_%s", VectorSupport::svmlname[op], avx_sse_str); | ||||
| 7907 | StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7908 | |||||
| 7909 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%s2_ha_%s", VectorSupport::svmlname[op], avx_sse_str); | ||||
| 7910 | StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7911 | |||||
| 7912 | snprintf(ebuf, sizeof(ebuf), "__jsvml_%s4_ha_%s", VectorSupport::svmlname[op], avx_sse_str); | ||||
| 7913 | StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libjsvml, ebuf); | ||||
| 7914 | } | ||||
| 7915 | } | ||||
| 7916 | #endif // COMPILER2 | ||||
| 7917 | |||||
| 7918 | if (UseVectorizedMismatchIntrinsic) { | ||||
| 7919 | StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch(); | ||||
| 7920 | } | ||||
| 7921 | } | ||||
| 7922 | |||||
| 7923 | public: | ||||
| 7924 | StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { | ||||
| 7925 | if (all) { | ||||
| 7926 | generate_all(); | ||||
| 7927 | } else { | ||||
| 7928 | generate_initial(); | ||||
| 7929 | } | ||||
| 7930 | } | ||||
| 7931 | }; // end class declaration | ||||
| 7932 | |||||
| 7933 | #define UCM_TABLE_MAX_ENTRIES16 16 | ||||
| 7934 | void StubGenerator_generate(CodeBuffer* code, bool all) { | ||||
| 7935 | if (UnsafeCopyMemory::_table == NULL__null) { | ||||
| 7936 | UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES16); | ||||
| 7937 | } | ||||
| 7938 | StubGenerator g(code, all); | ||||
| 7939 | } |
| 1 | /* |
| 2 | * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef CPU_X86_ASSEMBLER_X86_HPP |
| 26 | #define CPU_X86_ASSEMBLER_X86_HPP |
| 27 | |
| 28 | #include "asm/register.hpp" |
| 29 | #include "utilities/powerOfTwo.hpp" |
| 30 | |
| 31 | // Contains all the definitions needed for x86 assembly code generation. |
| 32 | |
| 33 | // Calling convention |
| 34 | class Argument { |
| 35 | public: |
| 36 | enum { |
| 37 | #ifdef _LP641 |
| 38 | #ifdef _WIN64 |
| 39 | n_int_register_parameters_c = 4, // rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...) |
| 40 | n_float_register_parameters_c = 4, // xmm0 - xmm3 (c_farg0, c_farg1, ... ) |
| 41 | n_int_register_returns_c = 1, // rax |
| 42 | n_float_register_returns_c = 1, // xmm0 |
| 43 | #else |
| 44 | n_int_register_parameters_c = 6, // rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...) |
| 45 | n_float_register_parameters_c = 8, // xmm0 - xmm7 (c_farg0, c_farg1, ... ) |
| 46 | n_int_register_returns_c = 2, // rax, rdx |
| 47 | n_float_register_returns_c = 2, // xmm0, xmm1 |
| 48 | #endif // _WIN64 |
| 49 | n_int_register_parameters_j = 6, // j_rarg0, j_rarg1, ... |
| 50 | n_float_register_parameters_j = 8 // j_farg0, j_farg1, ... |
| 51 | #else |
| 52 | n_register_parameters = 0 // 0 registers used to pass arguments |
| 53 | #endif // _LP64 |
| 54 | }; |
| 55 | }; |
| 56 | |
| 57 | |
| 58 | #ifdef _LP641 |
| 59 | // Symbolically name the register arguments used by the c calling convention. |
| 60 | // Windows is different from linux/solaris. So much for standards... |
| 61 | |
| 62 | #ifdef _WIN64 |
| 63 | |
| 64 | REGISTER_DECLARATION(Register, c_rarg0, rcx)const Register c_rarg0 = ((Register)rcx); |
| 65 | REGISTER_DECLARATION(Register, c_rarg1, rdx)const Register c_rarg1 = ((Register)rdx); |
| 66 | REGISTER_DECLARATION(Register, c_rarg2, r8)const Register c_rarg2 = ((Register)r8); |
| 67 | REGISTER_DECLARATION(Register, c_rarg3, r9)const Register c_rarg3 = ((Register)r9); |
| 68 | |
| 69 | REGISTER_DECLARATION(XMMRegister, c_farg0, xmm0)const XMMRegister c_farg0 = ((XMMRegister)xmm0); |
| 70 | REGISTER_DECLARATION(XMMRegister, c_farg1, xmm1)const XMMRegister c_farg1 = ((XMMRegister)xmm1); |
| 71 | REGISTER_DECLARATION(XMMRegister, c_farg2, xmm2)const XMMRegister c_farg2 = ((XMMRegister)xmm2); |
| 72 | REGISTER_DECLARATION(XMMRegister, c_farg3, xmm3)const XMMRegister c_farg3 = ((XMMRegister)xmm3); |
| 73 | |
| 74 | #else |
| 75 | |
| 76 | REGISTER_DECLARATION(Register, c_rarg0, rdi)const Register c_rarg0 = ((Register)rdi); |
| 77 | REGISTER_DECLARATION(Register, c_rarg1, rsi)const Register c_rarg1 = ((Register)rsi); |
| 78 | REGISTER_DECLARATION(Register, c_rarg2, rdx)const Register c_rarg2 = ((Register)rdx); |
| 79 | REGISTER_DECLARATION(Register, c_rarg3, rcx)const Register c_rarg3 = ((Register)rcx); |
| 80 | REGISTER_DECLARATION(Register, c_rarg4, r8)const Register c_rarg4 = ((Register)r8); |
| 81 | REGISTER_DECLARATION(Register, c_rarg5, r9)const Register c_rarg5 = ((Register)r9); |
| 82 | |
| 83 | REGISTER_DECLARATION(XMMRegister, c_farg0, xmm0)const XMMRegister c_farg0 = ((XMMRegister)xmm0); |
| 84 | REGISTER_DECLARATION(XMMRegister, c_farg1, xmm1)const XMMRegister c_farg1 = ((XMMRegister)xmm1); |
| 85 | REGISTER_DECLARATION(XMMRegister, c_farg2, xmm2)const XMMRegister c_farg2 = ((XMMRegister)xmm2); |
| 86 | REGISTER_DECLARATION(XMMRegister, c_farg3, xmm3)const XMMRegister c_farg3 = ((XMMRegister)xmm3); |
| 87 | REGISTER_DECLARATION(XMMRegister, c_farg4, xmm4)const XMMRegister c_farg4 = ((XMMRegister)xmm4); |
| 88 | REGISTER_DECLARATION(XMMRegister, c_farg5, xmm5)const XMMRegister c_farg5 = ((XMMRegister)xmm5); |
| 89 | REGISTER_DECLARATION(XMMRegister, c_farg6, xmm6)const XMMRegister c_farg6 = ((XMMRegister)xmm6); |
| 90 | REGISTER_DECLARATION(XMMRegister, c_farg7, xmm7)const XMMRegister c_farg7 = ((XMMRegister)xmm7); |
| 91 | |
| 92 | #endif // _WIN64 |
| 93 | |
| 94 | // Symbolically name the register arguments used by the Java calling convention. |
| 95 | // We have control over the convention for java so we can do what we please. |
| 96 | // What pleases us is to offset the java calling convention so that when |
| 97 | // we call a suitable jni method the arguments are lined up and we don't |
| 98 | // have to do little shuffling. A suitable jni method is non-static and a |
| 99 | // small number of arguments (two fewer args on windows) |
| 100 | // |
| 101 | // |-------------------------------------------------------| |
| 102 | // | c_rarg0 c_rarg1 c_rarg2 c_rarg3 c_rarg4 c_rarg5 | |
| 103 | // |-------------------------------------------------------| |
| 104 | // | rcx rdx r8 r9 rdi* rsi* | windows (* not a c_rarg) |
| 105 | // | rdi rsi rdx rcx r8 r9 | solaris/linux |
| 106 | // |-------------------------------------------------------| |
| 107 | // | j_rarg5 j_rarg0 j_rarg1 j_rarg2 j_rarg3 j_rarg4 | |
| 108 | // |-------------------------------------------------------| |
| 109 | |
| 110 | REGISTER_DECLARATION(Register, j_rarg0, c_rarg1)const Register j_rarg0 = ((Register)c_rarg1); |
| 111 | REGISTER_DECLARATION(Register, j_rarg1, c_rarg2)const Register j_rarg1 = ((Register)c_rarg2); |
| 112 | REGISTER_DECLARATION(Register, j_rarg2, c_rarg3)const Register j_rarg2 = ((Register)c_rarg3); |
| 113 | // Windows runs out of register args here |
| 114 | #ifdef _WIN64 |
| 115 | REGISTER_DECLARATION(Register, j_rarg3, rdi)const Register j_rarg3 = ((Register)rdi); |
| 116 | REGISTER_DECLARATION(Register, j_rarg4, rsi)const Register j_rarg4 = ((Register)rsi); |
| 117 | #else |
| 118 | REGISTER_DECLARATION(Register, j_rarg3, c_rarg4)const Register j_rarg3 = ((Register)c_rarg4); |
| 119 | REGISTER_DECLARATION(Register, j_rarg4, c_rarg5)const Register j_rarg4 = ((Register)c_rarg5); |
| 120 | #endif /* _WIN64 */ |
| 121 | REGISTER_DECLARATION(Register, j_rarg5, c_rarg0)const Register j_rarg5 = ((Register)c_rarg0); |
| 122 | |
| 123 | REGISTER_DECLARATION(XMMRegister, j_farg0, xmm0)const XMMRegister j_farg0 = ((XMMRegister)xmm0); |
| 124 | REGISTER_DECLARATION(XMMRegister, j_farg1, xmm1)const XMMRegister j_farg1 = ((XMMRegister)xmm1); |
| 125 | REGISTER_DECLARATION(XMMRegister, j_farg2, xmm2)const XMMRegister j_farg2 = ((XMMRegister)xmm2); |
| 126 | REGISTER_DECLARATION(XMMRegister, j_farg3, xmm3)const XMMRegister j_farg3 = ((XMMRegister)xmm3); |
| 127 | REGISTER_DECLARATION(XMMRegister, j_farg4, xmm4)const XMMRegister j_farg4 = ((XMMRegister)xmm4); |
| 128 | REGISTER_DECLARATION(XMMRegister, j_farg5, xmm5)const XMMRegister j_farg5 = ((XMMRegister)xmm5); |
| 129 | REGISTER_DECLARATION(XMMRegister, j_farg6, xmm6)const XMMRegister j_farg6 = ((XMMRegister)xmm6); |
| 130 | REGISTER_DECLARATION(XMMRegister, j_farg7, xmm7)const XMMRegister j_farg7 = ((XMMRegister)xmm7); |
| 131 | |
| 132 | REGISTER_DECLARATION(Register, rscratch1, r10)const Register rscratch1 = ((Register)r10); // volatile |
| 133 | REGISTER_DECLARATION(Register, rscratch2, r11)const Register rscratch2 = ((Register)r11); // volatile |
| 134 | |
| 135 | REGISTER_DECLARATION(Register, r12_heapbase, r12)const Register r12_heapbase = ((Register)r12); // callee-saved |
| 136 | REGISTER_DECLARATION(Register, r15_thread, r15)const Register r15_thread = ((Register)r15); // callee-saved |
| 137 | |
| 138 | #else |
| 139 | // rscratch1 will apear in 32bit code that is dead but of course must compile |
| 140 | // Using noreg ensures if the dead code is incorrectly live and executed it |
| 141 | // will cause an assertion failure |
| 142 | #define rscratch1 noreg |
| 143 | #define rscratch2 noreg |
| 144 | |
| 145 | #endif // _LP64 |
| 146 | |
| 147 | // JSR 292 |
| 148 | // On x86, the SP does not have to be saved when invoking method handle intrinsics |
| 149 | // or compiled lambda forms. We indicate that by setting rbp_mh_SP_save to noreg. |
| 150 | REGISTER_DECLARATION(Register, rbp_mh_SP_save, noreg)const Register rbp_mh_SP_save = ((Register)noreg); |
| 151 | |
| 152 | // Address is an abstraction used to represent a memory location |
| 153 | // using any of the amd64 addressing modes with one object. |
| 154 | // |
| 155 | // Note: A register location is represented via a Register, not |
| 156 | // via an address for efficiency & simplicity reasons. |
| 157 | |
| 158 | class ArrayAddress; |
| 159 | |
| 160 | class Address { |
| 161 | public: |
| 162 | enum ScaleFactor { |
| 163 | no_scale = -1, |
| 164 | times_1 = 0, |
| 165 | times_2 = 1, |
| 166 | times_4 = 2, |
| 167 | times_8 = 3, |
| 168 | times_ptr = LP64_ONLY(times_8)times_8 NOT_LP64(times_4) |
| 169 | }; |
| 170 | static ScaleFactor times(int size) { |
| 171 | assert(size >= 1 && size <= 8 && is_power_of_2(size), "bad scale size")do { if (!(size >= 1 && size <= 8 && is_power_of_2 (size))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 171, "assert(" "size >= 1 && size <= 8 && is_power_of_2(size)" ") failed", "bad scale size"); ::breakpoint(); } } while (0); |
| 172 | if (size == 8) return times_8; |
| 173 | if (size == 4) return times_4; |
| 174 | if (size == 2) return times_2; |
| 175 | return times_1; |
| 176 | } |
| 177 | static int scale_size(ScaleFactor scale) { |
| 178 | assert(scale != no_scale, "")do { if (!(scale != no_scale)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 178, "assert(" "scale != no_scale" ") failed", ""); ::breakpoint (); } } while (0); |
| 179 | assert(((1 << (int)times_1) == 1 &&do { if (!(((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)" ") failed", ""); ::breakpoint(); } } while (0) |
| 180 | (1 << (int)times_2) == 2 &&do { if (!(((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)" ") failed", ""); ::breakpoint(); } } while (0) |
| 181 | (1 << (int)times_4) == 4 &&do { if (!(((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)" ") failed", ""); ::breakpoint(); } } while (0) |
| 182 | (1 << (int)times_8) == 8), "")do { if (!(((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 182, "assert(" "((1 << (int)times_1) == 1 && (1 << (int)times_2) == 2 && (1 << (int)times_4) == 4 && (1 << (int)times_8) == 8)" ") failed", ""); ::breakpoint(); } } while (0); |
| 183 | return (1 << (int)scale); |
| 184 | } |
| 185 | |
| 186 | private: |
| 187 | Register _base; |
| 188 | Register _index; |
| 189 | XMMRegister _xmmindex; |
| 190 | ScaleFactor _scale; |
| 191 | int _disp; |
| 192 | bool _isxmmindex; |
| 193 | RelocationHolder _rspec; |
| 194 | |
| 195 | // Easily misused constructors make them private |
| 196 | // %%% can we make these go away? |
| 197 | NOT_LP64(Address(address loc, RelocationHolder spec);) |
| 198 | Address(int disp, address loc, relocInfo::relocType rtype); |
| 199 | Address(int disp, address loc, RelocationHolder spec); |
| 200 | |
| 201 | public: |
| 202 | |
| 203 | int disp() { return _disp; } |
| 204 | // creation |
| 205 | Address() |
| 206 | : _base(noreg), |
| 207 | _index(noreg), |
| 208 | _xmmindex(xnoreg), |
| 209 | _scale(no_scale), |
| 210 | _disp(0), |
| 211 | _isxmmindex(false){ |
| 212 | } |
| 213 | |
| 214 | // No default displacement otherwise Register can be implicitly |
| 215 | // converted to 0(Register) which is quite a different animal. |
| 216 | |
| 217 | Address(Register base, int disp) |
| 218 | : _base(base), |
| 219 | _index(noreg), |
| 220 | _xmmindex(xnoreg), |
| 221 | _scale(no_scale), |
| 222 | _disp(disp), |
| 223 | _isxmmindex(false){ |
| 224 | } |
| 225 | |
| 226 | Address(Register base, Register index, ScaleFactor scale, int disp = 0) |
| 227 | : _base (base), |
| 228 | _index(index), |
| 229 | _xmmindex(xnoreg), |
| 230 | _scale(scale), |
| 231 | _disp (disp), |
| 232 | _isxmmindex(false) { |
| 233 | assert(!index->is_valid() == (scale == Address::no_scale),do { if (!(!index->is_valid() == (scale == Address::no_scale ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 234, "assert(" "!index->is_valid() == (scale == Address::no_scale)" ") failed", "inconsistent address"); ::breakpoint(); } } while (0) |
| 234 | "inconsistent address")do { if (!(!index->is_valid() == (scale == Address::no_scale ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 234, "assert(" "!index->is_valid() == (scale == Address::no_scale)" ") failed", "inconsistent address"); ::breakpoint(); } } while (0); |
| 235 | } |
| 236 | |
| 237 | Address(Register base, RegisterOrConstant index, ScaleFactor scale = times_1, int disp = 0) |
| 238 | : _base (base), |
| 239 | _index(index.register_or_noreg()), |
| 240 | _xmmindex(xnoreg), |
| 241 | _scale(scale), |
| 242 | _disp (disp + (index.constant_or_zero() * scale_size(scale))), |
| 243 | _isxmmindex(false){ |
| 244 | if (!index.is_register()) scale = Address::no_scale; |
| 245 | assert(!_index->is_valid() == (scale == Address::no_scale),do { if (!(!_index->is_valid() == (scale == Address::no_scale ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 246, "assert(" "!_index->is_valid() == (scale == Address::no_scale)" ") failed", "inconsistent address"); ::breakpoint(); } } while (0) |
| 246 | "inconsistent address")do { if (!(!_index->is_valid() == (scale == Address::no_scale ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 246, "assert(" "!_index->is_valid() == (scale == Address::no_scale)" ") failed", "inconsistent address"); ::breakpoint(); } } while (0); |
| 247 | } |
| 248 | |
| 249 | Address(Register base, XMMRegister index, ScaleFactor scale, int disp = 0) |
| 250 | : _base (base), |
| 251 | _index(noreg), |
| 252 | _xmmindex(index), |
| 253 | _scale(scale), |
| 254 | _disp(disp), |
| 255 | _isxmmindex(true) { |
| 256 | assert(!index->is_valid() == (scale == Address::no_scale),do { if (!(!index->is_valid() == (scale == Address::no_scale ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 257, "assert(" "!index->is_valid() == (scale == Address::no_scale)" ") failed", "inconsistent address"); ::breakpoint(); } } while (0) |
| 257 | "inconsistent address")do { if (!(!index->is_valid() == (scale == Address::no_scale ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 257, "assert(" "!index->is_valid() == (scale == Address::no_scale)" ") failed", "inconsistent address"); ::breakpoint(); } } while (0); |
| 258 | } |
| 259 | |
| 260 | // The following overloads are used in connection with the |
| 261 | // ByteSize type (see sizes.hpp). They simplify the use of |
| 262 | // ByteSize'd arguments in assembly code. |
| 263 | |
| 264 | Address(Register base, ByteSize disp) |
| 265 | : Address(base, in_bytes(disp)) {} |
| 266 | |
| 267 | Address(Register base, Register index, ScaleFactor scale, ByteSize disp) |
| 268 | : Address(base, index, scale, in_bytes(disp)) {} |
| 269 | |
| 270 | Address(Register base, RegisterOrConstant index, ScaleFactor scale, ByteSize disp) |
| 271 | : Address(base, index, scale, in_bytes(disp)) {} |
| 272 | |
| 273 | Address plus_disp(int disp) const { |
| 274 | Address a = (*this); |
| 275 | a._disp += disp; |
| 276 | return a; |
| 277 | } |
| 278 | Address plus_disp(RegisterOrConstant disp, ScaleFactor scale = times_1) const { |
| 279 | Address a = (*this); |
| 280 | a._disp += disp.constant_or_zero() * scale_size(scale); |
| 281 | if (disp.is_register()) { |
| 282 | assert(!a.index()->is_valid(), "competing indexes")do { if (!(!a.index()->is_valid())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 282, "assert(" "!a.index()->is_valid()" ") failed", "competing indexes" ); ::breakpoint(); } } while (0); |
| 283 | a._index = disp.as_register(); |
| 284 | a._scale = scale; |
| 285 | } |
| 286 | return a; |
| 287 | } |
| 288 | bool is_same_address(Address a) const { |
| 289 | // disregard _rspec |
| 290 | return _base == a._base && _disp == a._disp && _index == a._index && _scale == a._scale; |
| 291 | } |
| 292 | |
| 293 | // accessors |
| 294 | bool uses(Register reg) const { return _base == reg || _index == reg; } |
| 295 | Register base() const { return _base; } |
| 296 | Register index() const { return _index; } |
| 297 | XMMRegister xmmindex() const { return _xmmindex; } |
| 298 | ScaleFactor scale() const { return _scale; } |
| 299 | int disp() const { return _disp; } |
| 300 | bool isxmmindex() const { return _isxmmindex; } |
| 301 | |
| 302 | // Convert the raw encoding form into the form expected by the constructor for |
| 303 | // Address. An index of 4 (rsp) corresponds to having no index, so convert |
| 304 | // that to noreg for the Address constructor. |
| 305 | static Address make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc); |
| 306 | |
| 307 | static Address make_array(ArrayAddress); |
| 308 | |
| 309 | private: |
| 310 | bool base_needs_rex() const { |
| 311 | return _base->is_valid() && _base->encoding() >= 8; |
| 312 | } |
| 313 | |
| 314 | bool index_needs_rex() const { |
| 315 | return _index->is_valid() &&_index->encoding() >= 8; |
| 316 | } |
| 317 | |
| 318 | bool xmmindex_needs_rex() const { |
| 319 | return _xmmindex->is_valid() && _xmmindex->encoding() >= 8; |
| 320 | } |
| 321 | |
| 322 | relocInfo::relocType reloc() const { return _rspec.type(); } |
| 323 | |
| 324 | friend class Assembler; |
| 325 | friend class MacroAssembler; |
| 326 | friend class LIR_Assembler; // base/index/scale/disp |
| 327 | }; |
| 328 | |
| 329 | // |
| 330 | // AddressLiteral has been split out from Address because operands of this type |
| 331 | // need to be treated specially on 32bit vs. 64bit platforms. By splitting it out |
| 332 | // the few instructions that need to deal with address literals are unique and the |
| 333 | // MacroAssembler does not have to implement every instruction in the Assembler |
| 334 | // in order to search for address literals that may need special handling depending |
| 335 | // on the instruction and the platform. As small step on the way to merging i486/amd64 |
| 336 | // directories. |
| 337 | // |
| 338 | class AddressLiteral { |
| 339 | friend class ArrayAddress; |
| 340 | RelocationHolder _rspec; |
| 341 | // Typically we use AddressLiterals we want to use their rval |
| 342 | // However in some situations we want the lval (effect address) of the item. |
| 343 | // We provide a special factory for making those lvals. |
| 344 | bool _is_lval; |
| 345 | |
| 346 | // If the target is far we'll need to load the ea of this to |
| 347 | // a register to reach it. Otherwise if near we can do rip |
| 348 | // relative addressing. |
| 349 | |
| 350 | address _target; |
| 351 | |
| 352 | protected: |
| 353 | // creation |
| 354 | AddressLiteral() |
| 355 | : _is_lval(false), |
| 356 | _target(NULL__null) |
| 357 | {} |
| 358 | |
| 359 | public: |
| 360 | |
| 361 | |
| 362 | AddressLiteral(address target, relocInfo::relocType rtype); |
| 363 | |
| 364 | AddressLiteral(address target, RelocationHolder const& rspec) |
| 365 | : _rspec(rspec), |
| 366 | _is_lval(false), |
| 367 | _target(target) |
| 368 | {} |
| 369 | |
| 370 | AddressLiteral addr() { |
| 371 | AddressLiteral ret = *this; |
| 372 | ret._is_lval = true; |
| 373 | return ret; |
| 374 | } |
| 375 | |
| 376 | |
| 377 | private: |
| 378 | |
| 379 | address target() { return _target; } |
| 380 | bool is_lval() { return _is_lval; } |
| 381 | |
| 382 | relocInfo::relocType reloc() const { return _rspec.type(); } |
| 383 | const RelocationHolder& rspec() const { return _rspec; } |
| 384 | |
| 385 | friend class Assembler; |
| 386 | friend class MacroAssembler; |
| 387 | friend class Address; |
| 388 | friend class LIR_Assembler; |
| 389 | }; |
| 390 | |
| 391 | // Convience classes |
| 392 | class RuntimeAddress: public AddressLiteral { |
| 393 | |
| 394 | public: |
| 395 | |
| 396 | RuntimeAddress(address target) : AddressLiteral(target, relocInfo::runtime_call_type) {} |
| 397 | |
| 398 | }; |
| 399 | |
| 400 | class ExternalAddress: public AddressLiteral { |
| 401 | private: |
| 402 | static relocInfo::relocType reloc_for_target(address target) { |
| 403 | // Sometimes ExternalAddress is used for values which aren't |
| 404 | // exactly addresses, like the card table base. |
| 405 | // external_word_type can't be used for values in the first page |
| 406 | // so just skip the reloc in that case. |
| 407 | return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none; |
| 408 | } |
| 409 | |
| 410 | public: |
| 411 | |
| 412 | ExternalAddress(address target) : AddressLiteral(target, reloc_for_target(target)) {} |
| 413 | |
| 414 | }; |
| 415 | |
| 416 | class InternalAddress: public AddressLiteral { |
| 417 | |
| 418 | public: |
| 419 | |
| 420 | InternalAddress(address target) : AddressLiteral(target, relocInfo::internal_word_type) {} |
| 421 | |
| 422 | }; |
| 423 | |
| 424 | // x86 can do array addressing as a single operation since disp can be an absolute |
| 425 | // address amd64 can't. We create a class that expresses the concept but does extra |
| 426 | // magic on amd64 to get the final result |
| 427 | |
| 428 | class ArrayAddress { |
| 429 | private: |
| 430 | |
| 431 | AddressLiteral _base; |
| 432 | Address _index; |
| 433 | |
| 434 | public: |
| 435 | |
| 436 | ArrayAddress() {}; |
| 437 | ArrayAddress(AddressLiteral base, Address index): _base(base), _index(index) {}; |
| 438 | AddressLiteral base() { return _base; } |
| 439 | Address index() { return _index; } |
| 440 | |
| 441 | }; |
| 442 | |
| 443 | class InstructionAttr; |
| 444 | |
| 445 | // 64-bit refect the fxsave size which is 512 bytes and the new xsave area on EVEX which is another 2176 bytes |
| 446 | // See fxsave and xsave(EVEX enabled) documentation for layout |
| 447 | const int FPUStateSizeInWords = NOT_LP64(27) LP64_ONLY(2688 / wordSize)2688 / wordSize; |
| 448 | |
| 449 | // The Intel x86/Amd64 Assembler: Pure assembler doing NO optimizations on the instruction |
| 450 | // level (e.g. mov rax, 0 is not translated into xor rax, rax!); i.e., what you write |
| 451 | // is what you get. The Assembler is generating code into a CodeBuffer. |
| 452 | |
| 453 | class Assembler : public AbstractAssembler { |
| 454 | friend class AbstractAssembler; // for the non-virtual hack |
| 455 | friend class LIR_Assembler; // as_Address() |
| 456 | friend class StubGenerator; |
| 457 | |
| 458 | public: |
| 459 | enum Condition { // The x86 condition codes used for conditional jumps/moves. |
| 460 | zero = 0x4, |
| 461 | notZero = 0x5, |
| 462 | equal = 0x4, |
| 463 | notEqual = 0x5, |
| 464 | less = 0xc, |
| 465 | lessEqual = 0xe, |
| 466 | greater = 0xf, |
| 467 | greaterEqual = 0xd, |
| 468 | below = 0x2, |
| 469 | belowEqual = 0x6, |
| 470 | above = 0x7, |
| 471 | aboveEqual = 0x3, |
| 472 | overflow = 0x0, |
| 473 | noOverflow = 0x1, |
| 474 | carrySet = 0x2, |
| 475 | carryClear = 0x3, |
| 476 | negative = 0x8, |
| 477 | positive = 0x9, |
| 478 | parity = 0xa, |
| 479 | noParity = 0xb |
| 480 | }; |
| 481 | |
| 482 | enum Prefix { |
| 483 | // segment overrides |
| 484 | CS_segment = 0x2e, |
| 485 | SS_segment = 0x36, |
| 486 | DS_segment = 0x3e, |
| 487 | ES_segment = 0x26, |
| 488 | FS_segment = 0x64, |
| 489 | GS_segment = 0x65, |
| 490 | |
| 491 | REX = 0x40, |
| 492 | |
| 493 | REX_B = 0x41, |
| 494 | REX_X = 0x42, |
| 495 | REX_XB = 0x43, |
| 496 | REX_R = 0x44, |
| 497 | REX_RB = 0x45, |
| 498 | REX_RX = 0x46, |
| 499 | REX_RXB = 0x47, |
| 500 | |
| 501 | REX_W = 0x48, |
| 502 | |
| 503 | REX_WB = 0x49, |
| 504 | REX_WX = 0x4A, |
| 505 | REX_WXB = 0x4B, |
| 506 | REX_WR = 0x4C, |
| 507 | REX_WRB = 0x4D, |
| 508 | REX_WRX = 0x4E, |
| 509 | REX_WRXB = 0x4F, |
| 510 | |
| 511 | VEX_3bytes = 0xC4, |
| 512 | VEX_2bytes = 0xC5, |
| 513 | EVEX_4bytes = 0x62, |
| 514 | Prefix_EMPTY = 0x0 |
| 515 | }; |
| 516 | |
| 517 | enum VexPrefix { |
| 518 | VEX_B = 0x20, |
| 519 | VEX_X = 0x40, |
| 520 | VEX_R = 0x80, |
| 521 | VEX_W = 0x80 |
| 522 | }; |
| 523 | |
| 524 | enum ExexPrefix { |
| 525 | EVEX_F = 0x04, |
| 526 | EVEX_V = 0x08, |
| 527 | EVEX_Rb = 0x10, |
| 528 | EVEX_X = 0x40, |
| 529 | EVEX_Z = 0x80 |
| 530 | }; |
| 531 | |
| 532 | enum VexSimdPrefix { |
| 533 | VEX_SIMD_NONE = 0x0, |
| 534 | VEX_SIMD_66 = 0x1, |
| 535 | VEX_SIMD_F3 = 0x2, |
| 536 | VEX_SIMD_F2 = 0x3 |
| 537 | }; |
| 538 | |
| 539 | enum VexOpcode { |
| 540 | VEX_OPCODE_NONE = 0x0, |
| 541 | VEX_OPCODE_0F = 0x1, |
| 542 | VEX_OPCODE_0F_38 = 0x2, |
| 543 | VEX_OPCODE_0F_3A = 0x3, |
| 544 | VEX_OPCODE_MASK = 0x1F |
| 545 | }; |
| 546 | |
| 547 | enum AvxVectorLen { |
| 548 | AVX_128bit = 0x0, |
| 549 | AVX_256bit = 0x1, |
| 550 | AVX_512bit = 0x2, |
| 551 | AVX_NoVec = 0x4 |
| 552 | }; |
| 553 | |
| 554 | enum EvexTupleType { |
| 555 | EVEX_FV = 0, |
| 556 | EVEX_HV = 4, |
| 557 | EVEX_FVM = 6, |
| 558 | EVEX_T1S = 7, |
| 559 | EVEX_T1F = 11, |
| 560 | EVEX_T2 = 13, |
| 561 | EVEX_T4 = 15, |
| 562 | EVEX_T8 = 17, |
| 563 | EVEX_HVM = 18, |
| 564 | EVEX_QVM = 19, |
| 565 | EVEX_OVM = 20, |
| 566 | EVEX_M128 = 21, |
| 567 | EVEX_DUP = 22, |
| 568 | EVEX_ETUP = 23 |
| 569 | }; |
| 570 | |
| 571 | enum EvexInputSizeInBits { |
| 572 | EVEX_8bit = 0, |
| 573 | EVEX_16bit = 1, |
| 574 | EVEX_32bit = 2, |
| 575 | EVEX_64bit = 3, |
| 576 | EVEX_NObit = 4 |
| 577 | }; |
| 578 | |
| 579 | enum WhichOperand { |
| 580 | // input to locate_operand, and format code for relocations |
| 581 | imm_operand = 0, // embedded 32-bit|64-bit immediate operand |
| 582 | disp32_operand = 1, // embedded 32-bit displacement or address |
| 583 | call32_operand = 2, // embedded 32-bit self-relative displacement |
| 584 | #ifndef _LP641 |
| 585 | _WhichOperand_limit = 3 |
| 586 | #else |
| 587 | narrow_oop_operand = 3, // embedded 32-bit immediate narrow oop |
| 588 | _WhichOperand_limit = 4 |
| 589 | #endif |
| 590 | }; |
| 591 | |
| 592 | // Comparison predicates for integral types & FP types when using SSE |
| 593 | enum ComparisonPredicate { |
| 594 | eq = 0, |
| 595 | lt = 1, |
| 596 | le = 2, |
| 597 | _false = 3, |
| 598 | neq = 4, |
| 599 | nlt = 5, |
| 600 | nle = 6, |
| 601 | _true = 7 |
| 602 | }; |
| 603 | |
| 604 | // Comparison predicates for FP types when using AVX |
| 605 | // O means ordered. U is unordered. When using ordered, any NaN comparison is false. Otherwise, it is true. |
| 606 | // S means signaling. Q means non-signaling. When signaling is true, instruction signals #IA on NaN. |
| 607 | enum ComparisonPredicateFP { |
| 608 | EQ_OQ = 0, |
| 609 | LT_OS = 1, |
| 610 | LE_OS = 2, |
| 611 | UNORD_Q = 3, |
| 612 | NEQ_UQ = 4, |
| 613 | NLT_US = 5, |
| 614 | NLE_US = 6, |
| 615 | ORD_Q = 7, |
| 616 | EQ_UQ = 8, |
| 617 | NGE_US = 9, |
| 618 | NGT_US = 0xA, |
| 619 | FALSE_OQ = 0XB, |
| 620 | NEQ_OQ = 0xC, |
| 621 | GE_OS = 0xD, |
| 622 | GT_OS = 0xE, |
| 623 | TRUE_UQ = 0xF, |
| 624 | EQ_OS = 0x10, |
| 625 | LT_OQ = 0x11, |
| 626 | LE_OQ = 0x12, |
| 627 | UNORD_S = 0x13, |
| 628 | NEQ_US = 0x14, |
| 629 | NLT_UQ = 0x15, |
| 630 | NLE_UQ = 0x16, |
| 631 | ORD_S = 0x17, |
| 632 | EQ_US = 0x18, |
| 633 | NGE_UQ = 0x19, |
| 634 | NGT_UQ = 0x1A, |
| 635 | FALSE_OS = 0x1B, |
| 636 | NEQ_OS = 0x1C, |
| 637 | GE_OQ = 0x1D, |
| 638 | GT_OQ = 0x1E, |
| 639 | TRUE_US =0x1F |
| 640 | }; |
| 641 | |
| 642 | enum Width { |
| 643 | B = 0, |
| 644 | W = 1, |
| 645 | D = 2, |
| 646 | Q = 3 |
| 647 | }; |
| 648 | |
| 649 | //---< calculate length of instruction >--- |
| 650 | // As instruction size can't be found out easily on x86/x64, |
| 651 | // we just use '4' for len and maxlen. |
| 652 | // instruction must start at passed address |
| 653 | static unsigned int instr_len(unsigned char *instr) { return 4; } |
| 654 | |
| 655 | //---< longest instructions >--- |
| 656 | // Max instruction length is not specified in architecture documentation. |
| 657 | // We could use a "safe enough" estimate (15), but just default to |
| 658 | // instruction length guess from above. |
| 659 | static unsigned int instr_maxlen() { return 4; } |
| 660 | |
| 661 | // NOTE: The general philopsophy of the declarations here is that 64bit versions |
| 662 | // of instructions are freely declared without the need for wrapping them an ifdef. |
| 663 | // (Some dangerous instructions are ifdef's out of inappropriate jvm's.) |
| 664 | // In the .cpp file the implementations are wrapped so that they are dropped out |
| 665 | // of the resulting jvm. This is done mostly to keep the footprint of MINIMAL |
| 666 | // to the size it was prior to merging up the 32bit and 64bit assemblers. |
| 667 | // |
| 668 | // This does mean you'll get a linker/runtime error if you use a 64bit only instruction |
| 669 | // in a 32bit vm. This is somewhat unfortunate but keeps the ifdef noise down. |
| 670 | |
| 671 | private: |
| 672 | |
| 673 | bool _legacy_mode_bw; |
| 674 | bool _legacy_mode_dq; |
| 675 | bool _legacy_mode_vl; |
| 676 | bool _legacy_mode_vlbw; |
| 677 | NOT_LP64(bool _is_managed;) |
| 678 | |
| 679 | class InstructionAttr *_attributes; |
| 680 | |
| 681 | // 64bit prefixes |
| 682 | void prefix(Register reg); |
| 683 | void prefix(Register dst, Register src, Prefix p); |
| 684 | void prefix(Register dst, Address adr, Prefix p); |
| 685 | |
| 686 | void prefix(Address adr); |
| 687 | void prefix(Address adr, Register reg, bool byteinst = false); |
| 688 | void prefix(Address adr, XMMRegister reg); |
| 689 | |
| 690 | int prefix_and_encode(int reg_enc, bool byteinst = false); |
| 691 | int prefix_and_encode(int dst_enc, int src_enc) { |
| 692 | return prefix_and_encode(dst_enc, false, src_enc, false); |
| 693 | } |
| 694 | int prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte); |
| 695 | |
| 696 | // Some prefixq variants always emit exactly one prefix byte, so besides a |
| 697 | // prefix-emitting method we provide a method to get the prefix byte to emit, |
| 698 | // which can then be folded into a byte stream. |
| 699 | int8_t get_prefixq(Address adr); |
| 700 | int8_t get_prefixq(Address adr, Register reg); |
| 701 | |
| 702 | void prefixq(Address adr); |
| 703 | void prefixq(Address adr, Register reg); |
| 704 | void prefixq(Address adr, XMMRegister reg); |
| 705 | |
| 706 | int prefixq_and_encode(int reg_enc); |
| 707 | int prefixq_and_encode(int dst_enc, int src_enc); |
| 708 | |
| 709 | void rex_prefix(Address adr, XMMRegister xreg, |
| 710 | VexSimdPrefix pre, VexOpcode opc, bool rex_w); |
| 711 | int rex_prefix_and_encode(int dst_enc, int src_enc, |
| 712 | VexSimdPrefix pre, VexOpcode opc, bool rex_w); |
| 713 | |
| 714 | void vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc); |
| 715 | |
| 716 | void evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, |
| 717 | int nds_enc, VexSimdPrefix pre, VexOpcode opc); |
| 718 | |
| 719 | void vex_prefix(Address adr, int nds_enc, int xreg_enc, |
| 720 | VexSimdPrefix pre, VexOpcode opc, |
| 721 | InstructionAttr *attributes); |
| 722 | |
| 723 | int vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, |
| 724 | VexSimdPrefix pre, VexOpcode opc, |
| 725 | InstructionAttr *attributes); |
| 726 | |
| 727 | void simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, |
| 728 | VexOpcode opc, InstructionAttr *attributes); |
| 729 | |
| 730 | int simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, |
| 731 | VexOpcode opc, InstructionAttr *attributes); |
| 732 | |
| 733 | // Helper functions for groups of instructions |
| 734 | void emit_arith_b(int op1, int op2, Register dst, int imm8); |
| 735 | |
| 736 | void emit_arith(int op1, int op2, Register dst, int32_t imm32); |
| 737 | // Force generation of a 4 byte immediate value even if it fits into 8bit |
| 738 | void emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32); |
| 739 | void emit_arith(int op1, int op2, Register dst, Register src); |
| 740 | |
| 741 | bool emit_compressed_disp_byte(int &disp); |
| 742 | |
| 743 | void emit_modrm(int mod, int dst_enc, int src_enc); |
| 744 | void emit_modrm_disp8(int mod, int dst_enc, int src_enc, |
| 745 | int disp); |
| 746 | void emit_modrm_sib(int mod, int dst_enc, int src_enc, |
| 747 | Address::ScaleFactor scale, int index_enc, int base_enc); |
| 748 | void emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc, |
| 749 | Address::ScaleFactor scale, int index_enc, int base_enc, |
| 750 | int disp); |
| 751 | |
| 752 | void emit_operand_helper(int reg_enc, |
| 753 | int base_enc, int index_enc, Address::ScaleFactor scale, |
| 754 | int disp, |
| 755 | RelocationHolder const& rspec, |
| 756 | int rip_relative_correction = 0); |
| 757 | |
| 758 | void emit_operand(Register reg, |
| 759 | Register base, Register index, Address::ScaleFactor scale, |
| 760 | int disp, |
| 761 | RelocationHolder const& rspec, |
| 762 | int rip_relative_correction = 0); |
| 763 | |
| 764 | void emit_operand(Register reg, |
| 765 | Register base, XMMRegister index, Address::ScaleFactor scale, |
| 766 | int disp, |
| 767 | RelocationHolder const& rspec); |
| 768 | |
| 769 | void emit_operand(XMMRegister xreg, |
| 770 | Register base, XMMRegister xindex, Address::ScaleFactor scale, |
| 771 | int disp, |
| 772 | RelocationHolder const& rspec); |
| 773 | |
| 774 | void emit_operand(Register reg, Address adr, |
| 775 | int rip_relative_correction = 0); |
| 776 | |
| 777 | void emit_operand(XMMRegister reg, |
| 778 | Register base, Register index, Address::ScaleFactor scale, |
| 779 | int disp, |
| 780 | RelocationHolder const& rspec); |
| 781 | |
| 782 | void emit_operand(XMMRegister reg, Address adr); |
| 783 | |
| 784 | // Immediate-to-memory forms |
| 785 | void emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32); |
| 786 | |
| 787 | protected: |
| 788 | #ifdef ASSERT1 |
| 789 | void check_relocation(RelocationHolder const& rspec, int format); |
| 790 | #endif |
| 791 | |
| 792 | void emit_data(jint data, relocInfo::relocType rtype, int format); |
| 793 | void emit_data(jint data, RelocationHolder const& rspec, int format); |
| 794 | void emit_data64(jlong data, relocInfo::relocType rtype, int format = 0); |
| 795 | void emit_data64(jlong data, RelocationHolder const& rspec, int format = 0); |
| 796 | |
| 797 | bool reachable(AddressLiteral adr) NOT_LP64({ return true;}); |
| 798 | |
| 799 | // These are all easily abused and hence protected |
| 800 | |
| 801 | // 32BIT ONLY SECTION |
| 802 | #ifndef _LP641 |
| 803 | // Make these disappear in 64bit mode since they would never be correct |
| 804 | void cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY |
| 805 | void cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY |
| 806 | |
| 807 | void mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY |
| 808 | void mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY |
| 809 | |
| 810 | void push_literal32(int32_t imm32, RelocationHolder const& rspec); // 32BIT ONLY |
| 811 | #else |
| 812 | // 64BIT ONLY SECTION |
| 813 | void mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec); // 64BIT ONLY |
| 814 | |
| 815 | void cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec); |
| 816 | void cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec); |
| 817 | |
| 818 | void mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec); |
| 819 | void mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec); |
| 820 | #endif // _LP64 |
| 821 | |
| 822 | // These are unique in that we are ensured by the caller that the 32bit |
| 823 | // relative in these instructions will always be able to reach the potentially |
| 824 | // 64bit address described by entry. Since they can take a 64bit address they |
| 825 | // don't have the 32 suffix like the other instructions in this class. |
| 826 | |
| 827 | void call_literal(address entry, RelocationHolder const& rspec); |
| 828 | void jmp_literal(address entry, RelocationHolder const& rspec); |
| 829 | |
| 830 | // Avoid using directly section |
| 831 | // Instructions in this section are actually usable by anyone without danger |
| 832 | // of failure but have performance issues that are addressed my enhanced |
| 833 | // instructions which will do the proper thing base on the particular cpu. |
| 834 | // We protect them because we don't trust you... |
| 835 | |
| 836 | // Don't use next inc() and dec() methods directly. INC & DEC instructions |
| 837 | // could cause a partial flag stall since they don't set CF flag. |
| 838 | // Use MacroAssembler::decrement() & MacroAssembler::increment() methods |
| 839 | // which call inc() & dec() or add() & sub() in accordance with |
| 840 | // the product flag UseIncDec value. |
| 841 | |
| 842 | void decl(Register dst); |
| 843 | void decl(Address dst); |
| 844 | void decq(Address dst); |
| 845 | |
| 846 | void incl(Register dst); |
| 847 | void incl(Address dst); |
| 848 | void incq(Register dst); |
| 849 | void incq(Address dst); |
| 850 | |
| 851 | // New cpus require use of movsd and movss to avoid partial register stall |
| 852 | // when loading from memory. But for old Opteron use movlpd instead of movsd. |
| 853 | // The selection is done in MacroAssembler::movdbl() and movflt(). |
| 854 | |
| 855 | // Move Scalar Single-Precision Floating-Point Values |
| 856 | void movss(XMMRegister dst, Address src); |
| 857 | void movss(XMMRegister dst, XMMRegister src); |
| 858 | void movss(Address dst, XMMRegister src); |
| 859 | |
| 860 | // Move Scalar Double-Precision Floating-Point Values |
| 861 | void movsd(XMMRegister dst, Address src); |
| 862 | void movsd(XMMRegister dst, XMMRegister src); |
| 863 | void movsd(Address dst, XMMRegister src); |
| 864 | void movlpd(XMMRegister dst, Address src); |
| 865 | |
| 866 | // New cpus require use of movaps and movapd to avoid partial register stall |
| 867 | // when moving between registers. |
| 868 | void movaps(XMMRegister dst, XMMRegister src); |
| 869 | void movapd(XMMRegister dst, XMMRegister src); |
| 870 | |
| 871 | // End avoid using directly |
| 872 | |
| 873 | |
| 874 | // Instruction prefixes |
| 875 | void prefix(Prefix p); |
| 876 | |
| 877 | public: |
| 878 | |
| 879 | // Creation |
| 880 | Assembler(CodeBuffer* code) : AbstractAssembler(code) { |
| 881 | init_attributes(); |
| 882 | } |
| 883 | |
| 884 | // Decoding |
| 885 | static address locate_operand(address inst, WhichOperand which); |
| 886 | static address locate_next_instruction(address inst); |
| 887 | |
| 888 | // Utilities |
| 889 | static bool query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len, |
| 890 | int cur_tuple_type, int in_size_in_bits, int cur_encoding); |
| 891 | |
| 892 | // Generic instructions |
| 893 | // Does 32bit or 64bit as needed for the platform. In some sense these |
| 894 | // belong in macro assembler but there is no need for both varieties to exist |
| 895 | |
| 896 | void init_attributes(void); |
| 897 | |
| 898 | void set_attributes(InstructionAttr *attributes) { _attributes = attributes; } |
| 899 | void clear_attributes(void) { _attributes = NULL__null; } |
| 900 | |
| 901 | void set_managed(void) { NOT_LP64(_is_managed = true;) } |
| 902 | void clear_managed(void) { NOT_LP64(_is_managed = false;) } |
| 903 | bool is_managed(void) { |
| 904 | NOT_LP64(return _is_managed;) |
| 905 | LP64_ONLY(return false;)return false; } |
| 906 | |
| 907 | void lea(Register dst, Address src); |
| 908 | |
| 909 | void mov(Register dst, Register src); |
| 910 | |
| 911 | #ifdef _LP641 |
| 912 | // support caching the result of some routines |
| 913 | |
| 914 | // must be called before pusha(), popa(), vzeroupper() - checked with asserts |
| 915 | static void precompute_instructions(); |
| 916 | |
| 917 | void pusha_uncached(); |
| 918 | void popa_uncached(); |
| 919 | #endif |
| 920 | void vzeroupper_uncached(); |
| 921 | void decq(Register dst); |
| 922 | |
| 923 | void pusha(); |
| 924 | void popa(); |
| 925 | |
| 926 | void pushf(); |
| 927 | void popf(); |
| 928 | |
| 929 | void push(int32_t imm32); |
| 930 | |
| 931 | void push(Register src); |
| 932 | |
| 933 | void pop(Register dst); |
| 934 | |
| 935 | // These are dummies to prevent surprise implicit conversions to Register |
| 936 | void push(void* v); |
| 937 | void pop(void* v); |
| 938 | |
| 939 | // These do register sized moves/scans |
| 940 | void rep_mov(); |
| 941 | void rep_stos(); |
| 942 | void rep_stosb(); |
| 943 | void repne_scan(); |
| 944 | #ifdef _LP641 |
| 945 | void repne_scanl(); |
| 946 | #endif |
| 947 | |
| 948 | // Vanilla instructions in lexical order |
| 949 | |
| 950 | void adcl(Address dst, int32_t imm32); |
| 951 | void adcl(Address dst, Register src); |
| 952 | void adcl(Register dst, int32_t imm32); |
| 953 | void adcl(Register dst, Address src); |
| 954 | void adcl(Register dst, Register src); |
| 955 | |
| 956 | void adcq(Register dst, int32_t imm32); |
| 957 | void adcq(Register dst, Address src); |
| 958 | void adcq(Register dst, Register src); |
| 959 | |
| 960 | void addb(Address dst, int imm8); |
| 961 | void addw(Register dst, Register src); |
| 962 | void addw(Address dst, int imm16); |
| 963 | |
| 964 | void addl(Address dst, int32_t imm32); |
| 965 | void addl(Address dst, Register src); |
| 966 | void addl(Register dst, int32_t imm32); |
| 967 | void addl(Register dst, Address src); |
| 968 | void addl(Register dst, Register src); |
| 969 | |
| 970 | void addq(Address dst, int32_t imm32); |
| 971 | void addq(Address dst, Register src); |
| 972 | void addq(Register dst, int32_t imm32); |
| 973 | void addq(Register dst, Address src); |
| 974 | void addq(Register dst, Register src); |
| 975 | |
| 976 | #ifdef _LP641 |
| 977 | //Add Unsigned Integers with Carry Flag |
| 978 | void adcxq(Register dst, Register src); |
| 979 | |
| 980 | //Add Unsigned Integers with Overflow Flag |
| 981 | void adoxq(Register dst, Register src); |
| 982 | #endif |
| 983 | |
| 984 | void addr_nop_4(); |
| 985 | void addr_nop_5(); |
| 986 | void addr_nop_7(); |
| 987 | void addr_nop_8(); |
| 988 | |
| 989 | // Add Scalar Double-Precision Floating-Point Values |
| 990 | void addsd(XMMRegister dst, Address src); |
| 991 | void addsd(XMMRegister dst, XMMRegister src); |
| 992 | |
| 993 | // Add Scalar Single-Precision Floating-Point Values |
| 994 | void addss(XMMRegister dst, Address src); |
| 995 | void addss(XMMRegister dst, XMMRegister src); |
| 996 | |
| 997 | // AES instructions |
| 998 | void aesdec(XMMRegister dst, Address src); |
| 999 | void aesdec(XMMRegister dst, XMMRegister src); |
| 1000 | void aesdeclast(XMMRegister dst, Address src); |
| 1001 | void aesdeclast(XMMRegister dst, XMMRegister src); |
| 1002 | void aesenc(XMMRegister dst, Address src); |
| 1003 | void aesenc(XMMRegister dst, XMMRegister src); |
| 1004 | void aesenclast(XMMRegister dst, Address src); |
| 1005 | void aesenclast(XMMRegister dst, XMMRegister src); |
| 1006 | // Vector AES instructions |
| 1007 | void vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1008 | void vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1009 | void vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1010 | void vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1011 | |
| 1012 | void andw(Register dst, Register src); |
| 1013 | void andb(Address dst, Register src); |
| 1014 | |
| 1015 | void andl(Address dst, int32_t imm32); |
| 1016 | void andl(Register dst, int32_t imm32); |
| 1017 | void andl(Register dst, Address src); |
| 1018 | void andl(Register dst, Register src); |
| 1019 | void andl(Address dst, Register src); |
| 1020 | |
| 1021 | void andq(Address dst, int32_t imm32); |
| 1022 | void andq(Register dst, int32_t imm32); |
| 1023 | void andq(Register dst, Address src); |
| 1024 | void andq(Register dst, Register src); |
| 1025 | void andq(Address dst, Register src); |
| 1026 | |
| 1027 | // BMI instructions |
| 1028 | void andnl(Register dst, Register src1, Register src2); |
| 1029 | void andnl(Register dst, Register src1, Address src2); |
| 1030 | void andnq(Register dst, Register src1, Register src2); |
| 1031 | void andnq(Register dst, Register src1, Address src2); |
| 1032 | |
| 1033 | void blsil(Register dst, Register src); |
| 1034 | void blsil(Register dst, Address src); |
| 1035 | void blsiq(Register dst, Register src); |
| 1036 | void blsiq(Register dst, Address src); |
| 1037 | |
| 1038 | void blsmskl(Register dst, Register src); |
| 1039 | void blsmskl(Register dst, Address src); |
| 1040 | void blsmskq(Register dst, Register src); |
| 1041 | void blsmskq(Register dst, Address src); |
| 1042 | |
| 1043 | void blsrl(Register dst, Register src); |
| 1044 | void blsrl(Register dst, Address src); |
| 1045 | void blsrq(Register dst, Register src); |
| 1046 | void blsrq(Register dst, Address src); |
| 1047 | |
| 1048 | void bsfl(Register dst, Register src); |
| 1049 | void bsrl(Register dst, Register src); |
| 1050 | |
| 1051 | #ifdef _LP641 |
| 1052 | void bsfq(Register dst, Register src); |
| 1053 | void bsrq(Register dst, Register src); |
| 1054 | #endif |
| 1055 | |
| 1056 | void bswapl(Register reg); |
| 1057 | |
| 1058 | void bswapq(Register reg); |
| 1059 | |
| 1060 | void call(Label& L, relocInfo::relocType rtype); |
| 1061 | void call(Register reg); // push pc; pc <- reg |
| 1062 | void call(Address adr); // push pc; pc <- adr |
| 1063 | |
| 1064 | void cdql(); |
| 1065 | |
| 1066 | void cdqq(); |
| 1067 | |
| 1068 | void cld(); |
| 1069 | |
| 1070 | void clflush(Address adr); |
| 1071 | void clflushopt(Address adr); |
| 1072 | void clwb(Address adr); |
| 1073 | |
| 1074 | void cmovl(Condition cc, Register dst, Register src); |
| 1075 | void cmovl(Condition cc, Register dst, Address src); |
| 1076 | |
| 1077 | void cmovq(Condition cc, Register dst, Register src); |
| 1078 | void cmovq(Condition cc, Register dst, Address src); |
| 1079 | |
| 1080 | |
| 1081 | void cmpb(Address dst, int imm8); |
| 1082 | |
| 1083 | void cmpl(Address dst, int32_t imm32); |
| 1084 | |
| 1085 | void cmp(Register dst, int32_t imm32); |
| 1086 | void cmpl(Register dst, int32_t imm32); |
| 1087 | void cmpl(Register dst, Register src); |
| 1088 | void cmpl(Register dst, Address src); |
| 1089 | |
| 1090 | void cmpq(Address dst, int32_t imm32); |
| 1091 | void cmpq(Address dst, Register src); |
| 1092 | |
| 1093 | void cmpq(Register dst, int32_t imm32); |
| 1094 | void cmpq(Register dst, Register src); |
| 1095 | void cmpq(Register dst, Address src); |
| 1096 | |
| 1097 | // these are dummies used to catch attempting to convert NULL to Register |
| 1098 | void cmpl(Register dst, void* junk); // dummy |
| 1099 | void cmpq(Register dst, void* junk); // dummy |
| 1100 | |
| 1101 | void cmpw(Address dst, int imm16); |
| 1102 | |
| 1103 | void cmpxchg8 (Address adr); |
| 1104 | |
| 1105 | void cmpxchgb(Register reg, Address adr); |
| 1106 | void cmpxchgl(Register reg, Address adr); |
| 1107 | |
| 1108 | void cmpxchgq(Register reg, Address adr); |
| 1109 | void cmpxchgw(Register reg, Address adr); |
| 1110 | |
| 1111 | // Ordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS |
| 1112 | void comisd(XMMRegister dst, Address src); |
| 1113 | void comisd(XMMRegister dst, XMMRegister src); |
| 1114 | |
| 1115 | // Ordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS |
| 1116 | void comiss(XMMRegister dst, Address src); |
| 1117 | void comiss(XMMRegister dst, XMMRegister src); |
| 1118 | |
| 1119 | // Identify processor type and features |
| 1120 | void cpuid(); |
| 1121 | |
| 1122 | // CRC32C |
| 1123 | void crc32(Register crc, Register v, int8_t sizeInBytes); |
| 1124 | void crc32(Register crc, Address adr, int8_t sizeInBytes); |
| 1125 | |
| 1126 | // Convert Scalar Double-Precision Floating-Point Value to Scalar Single-Precision Floating-Point Value |
| 1127 | void cvtsd2ss(XMMRegister dst, XMMRegister src); |
| 1128 | void cvtsd2ss(XMMRegister dst, Address src); |
| 1129 | |
| 1130 | // Convert Doubleword Integer to Scalar Double-Precision Floating-Point Value |
| 1131 | void cvtsi2sdl(XMMRegister dst, Register src); |
| 1132 | void cvtsi2sdl(XMMRegister dst, Address src); |
| 1133 | void cvtsi2sdq(XMMRegister dst, Register src); |
| 1134 | void cvtsi2sdq(XMMRegister dst, Address src); |
| 1135 | |
| 1136 | // Convert Doubleword Integer to Scalar Single-Precision Floating-Point Value |
| 1137 | void cvtsi2ssl(XMMRegister dst, Register src); |
| 1138 | void cvtsi2ssl(XMMRegister dst, Address src); |
| 1139 | void cvtsi2ssq(XMMRegister dst, Register src); |
| 1140 | void cvtsi2ssq(XMMRegister dst, Address src); |
| 1141 | |
| 1142 | // Convert Packed Signed Doubleword Integers to Packed Double-Precision Floating-Point Value |
| 1143 | void cvtdq2pd(XMMRegister dst, XMMRegister src); |
| 1144 | void vcvtdq2pd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1145 | |
| 1146 | // Convert Packed Signed Doubleword Integers to Packed Single-Precision Floating-Point Value |
| 1147 | void cvtdq2ps(XMMRegister dst, XMMRegister src); |
| 1148 | void vcvtdq2ps(XMMRegister dst, XMMRegister src, int vector_len); |
| 1149 | |
| 1150 | // Convert Scalar Single-Precision Floating-Point Value to Scalar Double-Precision Floating-Point Value |
| 1151 | void cvtss2sd(XMMRegister dst, XMMRegister src); |
| 1152 | void cvtss2sd(XMMRegister dst, Address src); |
| 1153 | |
| 1154 | // Convert with Truncation Scalar Double-Precision Floating-Point Value to Doubleword Integer |
| 1155 | void cvttsd2sil(Register dst, Address src); |
| 1156 | void cvttsd2sil(Register dst, XMMRegister src); |
| 1157 | void cvttsd2siq(Register dst, Address src); |
| 1158 | void cvttsd2siq(Register dst, XMMRegister src); |
| 1159 | |
| 1160 | // Convert with Truncation Scalar Single-Precision Floating-Point Value to Doubleword Integer |
| 1161 | void cvttss2sil(Register dst, XMMRegister src); |
| 1162 | void cvttss2siq(Register dst, XMMRegister src); |
| 1163 | |
| 1164 | // Convert vector double to int |
| 1165 | void cvttpd2dq(XMMRegister dst, XMMRegister src); |
| 1166 | |
| 1167 | // Convert vector float and double |
| 1168 | void vcvtps2pd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1169 | void vcvtpd2ps(XMMRegister dst, XMMRegister src, int vector_len); |
| 1170 | |
| 1171 | // Convert vector float and int |
| 1172 | void vcvttps2dq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1173 | |
| 1174 | // Convert vector long to vector FP |
| 1175 | void evcvtqq2ps(XMMRegister dst, XMMRegister src, int vector_len); |
| 1176 | void evcvtqq2pd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1177 | |
| 1178 | // Convert vector double to long |
| 1179 | void evcvttpd2qq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1180 | |
| 1181 | // Evex casts with truncation |
| 1182 | void evpmovwb(XMMRegister dst, XMMRegister src, int vector_len); |
| 1183 | void evpmovdw(XMMRegister dst, XMMRegister src, int vector_len); |
| 1184 | void evpmovdb(XMMRegister dst, XMMRegister src, int vector_len); |
| 1185 | void evpmovqd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1186 | void evpmovqb(XMMRegister dst, XMMRegister src, int vector_len); |
| 1187 | void evpmovqw(XMMRegister dst, XMMRegister src, int vector_len); |
| 1188 | |
| 1189 | //Abs of packed Integer values |
| 1190 | void pabsb(XMMRegister dst, XMMRegister src); |
| 1191 | void pabsw(XMMRegister dst, XMMRegister src); |
| 1192 | void pabsd(XMMRegister dst, XMMRegister src); |
| 1193 | void vpabsb(XMMRegister dst, XMMRegister src, int vector_len); |
| 1194 | void vpabsw(XMMRegister dst, XMMRegister src, int vector_len); |
| 1195 | void vpabsd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1196 | void evpabsq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1197 | |
| 1198 | // Divide Scalar Double-Precision Floating-Point Values |
| 1199 | void divsd(XMMRegister dst, Address src); |
| 1200 | void divsd(XMMRegister dst, XMMRegister src); |
| 1201 | |
| 1202 | // Divide Scalar Single-Precision Floating-Point Values |
| 1203 | void divss(XMMRegister dst, Address src); |
| 1204 | void divss(XMMRegister dst, XMMRegister src); |
| 1205 | |
| 1206 | |
| 1207 | #ifndef _LP641 |
| 1208 | private: |
| 1209 | |
| 1210 | void emit_farith(int b1, int b2, int i); |
| 1211 | |
| 1212 | public: |
| 1213 | void emms(); |
| 1214 | |
| 1215 | void fabs(); |
| 1216 | |
| 1217 | void fadd(int i); |
| 1218 | |
| 1219 | void fadd_d(Address src); |
| 1220 | void fadd_s(Address src); |
| 1221 | |
| 1222 | // "Alternate" versions of x87 instructions place result down in FPU |
| 1223 | // stack instead of on TOS |
| 1224 | |
| 1225 | void fadda(int i); // "alternate" fadd |
| 1226 | void faddp(int i = 1); |
| 1227 | |
| 1228 | void fchs(); |
| 1229 | |
| 1230 | void fcom(int i); |
| 1231 | |
| 1232 | void fcomp(int i = 1); |
| 1233 | void fcomp_d(Address src); |
| 1234 | void fcomp_s(Address src); |
| 1235 | |
| 1236 | void fcompp(); |
| 1237 | |
| 1238 | void fcos(); |
| 1239 | |
| 1240 | void fdecstp(); |
| 1241 | |
| 1242 | void fdiv(int i); |
| 1243 | void fdiv_d(Address src); |
| 1244 | void fdivr_s(Address src); |
| 1245 | void fdiva(int i); // "alternate" fdiv |
| 1246 | void fdivp(int i = 1); |
| 1247 | |
| 1248 | void fdivr(int i); |
| 1249 | void fdivr_d(Address src); |
| 1250 | void fdiv_s(Address src); |
| 1251 | |
| 1252 | void fdivra(int i); // "alternate" reversed fdiv |
| 1253 | |
| 1254 | void fdivrp(int i = 1); |
| 1255 | |
| 1256 | void ffree(int i = 0); |
| 1257 | |
| 1258 | void fild_d(Address adr); |
| 1259 | void fild_s(Address adr); |
| 1260 | |
| 1261 | void fincstp(); |
| 1262 | |
| 1263 | void finit(); |
| 1264 | |
| 1265 | void fist_s (Address adr); |
| 1266 | void fistp_d(Address adr); |
| 1267 | void fistp_s(Address adr); |
| 1268 | |
| 1269 | void fld1(); |
| 1270 | |
| 1271 | void fld_d(Address adr); |
| 1272 | void fld_s(Address adr); |
| 1273 | void fld_s(int index); |
| 1274 | |
| 1275 | void fldcw(Address src); |
| 1276 | |
| 1277 | void fldenv(Address src); |
| 1278 | |
| 1279 | void fldlg2(); |
| 1280 | |
| 1281 | void fldln2(); |
| 1282 | |
| 1283 | void fldz(); |
| 1284 | |
| 1285 | void flog(); |
| 1286 | void flog10(); |
| 1287 | |
| 1288 | void fmul(int i); |
| 1289 | |
| 1290 | void fmul_d(Address src); |
| 1291 | void fmul_s(Address src); |
| 1292 | |
| 1293 | void fmula(int i); // "alternate" fmul |
| 1294 | |
| 1295 | void fmulp(int i = 1); |
| 1296 | |
| 1297 | void fnsave(Address dst); |
| 1298 | |
| 1299 | void fnstcw(Address src); |
| 1300 | |
| 1301 | void fnstsw_ax(); |
| 1302 | |
| 1303 | void fprem(); |
| 1304 | void fprem1(); |
| 1305 | |
| 1306 | void frstor(Address src); |
| 1307 | |
| 1308 | void fsin(); |
| 1309 | |
| 1310 | void fsqrt(); |
| 1311 | |
| 1312 | void fst_d(Address adr); |
| 1313 | void fst_s(Address adr); |
| 1314 | |
| 1315 | void fstp_d(Address adr); |
| 1316 | void fstp_d(int index); |
| 1317 | void fstp_s(Address adr); |
| 1318 | |
| 1319 | void fsub(int i); |
| 1320 | void fsub_d(Address src); |
| 1321 | void fsub_s(Address src); |
| 1322 | |
| 1323 | void fsuba(int i); // "alternate" fsub |
| 1324 | |
| 1325 | void fsubp(int i = 1); |
| 1326 | |
| 1327 | void fsubr(int i); |
| 1328 | void fsubr_d(Address src); |
| 1329 | void fsubr_s(Address src); |
| 1330 | |
| 1331 | void fsubra(int i); // "alternate" reversed fsub |
| 1332 | |
| 1333 | void fsubrp(int i = 1); |
| 1334 | |
| 1335 | void ftan(); |
| 1336 | |
| 1337 | void ftst(); |
| 1338 | |
| 1339 | void fucomi(int i = 1); |
| 1340 | void fucomip(int i = 1); |
| 1341 | |
| 1342 | void fwait(); |
| 1343 | |
| 1344 | void fxch(int i = 1); |
| 1345 | |
| 1346 | void fyl2x(); |
| 1347 | void frndint(); |
| 1348 | void f2xm1(); |
| 1349 | void fldl2e(); |
| 1350 | #endif // !_LP64 |
| 1351 | |
| 1352 | // operands that only take the original 32bit registers |
| 1353 | void emit_operand32(Register reg, Address adr); |
| 1354 | |
| 1355 | void fld_x(Address adr); // extended-precision (80-bit) format |
| 1356 | void fstp_x(Address adr); // extended-precision (80-bit) format |
| 1357 | void fxrstor(Address src); |
| 1358 | void xrstor(Address src); |
| 1359 | |
| 1360 | void fxsave(Address dst); |
| 1361 | void xsave(Address dst); |
| 1362 | |
| 1363 | void hlt(); |
| 1364 | |
| 1365 | void idivl(Register src); |
| 1366 | void divl(Register src); // Unsigned division |
| 1367 | |
| 1368 | #ifdef _LP641 |
| 1369 | void idivq(Register src); |
| 1370 | #endif |
| 1371 | |
| 1372 | void imull(Register src); |
| 1373 | void imull(Register dst, Register src); |
| 1374 | void imull(Register dst, Register src, int value); |
| 1375 | void imull(Register dst, Address src, int value); |
| 1376 | void imull(Register dst, Address src); |
| 1377 | |
| 1378 | #ifdef _LP641 |
| 1379 | void imulq(Register dst, Register src); |
| 1380 | void imulq(Register dst, Register src, int value); |
| 1381 | void imulq(Register dst, Address src, int value); |
| 1382 | void imulq(Register dst, Address src); |
| 1383 | void imulq(Register dst); |
| 1384 | #endif |
| 1385 | |
| 1386 | // jcc is the generic conditional branch generator to run- |
| 1387 | // time routines, jcc is used for branches to labels. jcc |
| 1388 | // takes a branch opcode (cc) and a label (L) and generates |
| 1389 | // either a backward branch or a forward branch and links it |
| 1390 | // to the label fixup chain. Usage: |
| 1391 | // |
| 1392 | // Label L; // unbound label |
| 1393 | // jcc(cc, L); // forward branch to unbound label |
| 1394 | // bind(L); // bind label to the current pc |
| 1395 | // jcc(cc, L); // backward branch to bound label |
| 1396 | // bind(L); // illegal: a label may be bound only once |
| 1397 | // |
| 1398 | // Note: The same Label can be used for forward and backward branches |
| 1399 | // but it may be bound only once. |
| 1400 | |
| 1401 | void jcc(Condition cc, Label& L, bool maybe_short = true); |
| 1402 | |
| 1403 | // Conditional jump to a 8-bit offset to L. |
| 1404 | // WARNING: be very careful using this for forward jumps. If the label is |
| 1405 | // not bound within an 8-bit offset of this instruction, a run-time error |
| 1406 | // will occur. |
| 1407 | |
| 1408 | // Use macro to record file and line number. |
| 1409 | #define jccb(cc, L)jccb_0(cc, L, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 1409) jccb_0(cc, L, __FILE__"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp", __LINE__1409) |
| 1410 | |
| 1411 | void jccb_0(Condition cc, Label& L, const char* file, int line); |
| 1412 | |
| 1413 | void jmp(Address entry); // pc <- entry |
| 1414 | |
| 1415 | // Label operations & relative jumps (PPUM Appendix D) |
| 1416 | void jmp(Label& L, bool maybe_short = true); // unconditional jump to L |
| 1417 | |
| 1418 | void jmp(Register entry); // pc <- entry |
| 1419 | |
| 1420 | // Unconditional 8-bit offset jump to L. |
| 1421 | // WARNING: be very careful using this for forward jumps. If the label is |
| 1422 | // not bound within an 8-bit offset of this instruction, a run-time error |
| 1423 | // will occur. |
| 1424 | |
| 1425 | // Use macro to record file and line number. |
| 1426 | #define jmpb(L)jmpb_0(L, "/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp" , 1426) jmpb_0(L, __FILE__"/home/daniel/Projects/java/jdk/src/hotspot/cpu/x86/assembler_x86.hpp", __LINE__1426) |
| 1427 | |
| 1428 | void jmpb_0(Label& L, const char* file, int line); |
| 1429 | |
| 1430 | void ldmxcsr( Address src ); |
| 1431 | |
| 1432 | void leal(Register dst, Address src); |
| 1433 | |
| 1434 | void leaq(Register dst, Address src); |
| 1435 | |
| 1436 | void lfence(); |
| 1437 | |
| 1438 | void lock(); |
| 1439 | void size_prefix(); |
| 1440 | |
| 1441 | void lzcntl(Register dst, Register src); |
| 1442 | |
| 1443 | #ifdef _LP641 |
| 1444 | void lzcntq(Register dst, Register src); |
| 1445 | #endif |
| 1446 | |
| 1447 | enum Membar_mask_bits { |
| 1448 | StoreStore = 1 << 3, |
| 1449 | LoadStore = 1 << 2, |
| 1450 | StoreLoad = 1 << 1, |
| 1451 | LoadLoad = 1 << 0 |
| 1452 | }; |
| 1453 | |
| 1454 | // Serializes memory and blows flags |
| 1455 | void membar(Membar_mask_bits order_constraint); |
| 1456 | |
| 1457 | void mfence(); |
| 1458 | void sfence(); |
| 1459 | |
| 1460 | // Moves |
| 1461 | |
| 1462 | void mov64(Register dst, int64_t imm64); |
| 1463 | void mov64(Register dst, int64_t imm64, relocInfo::relocType rtype, int format); |
| 1464 | |
| 1465 | void movb(Address dst, Register src); |
| 1466 | void movb(Address dst, int imm8); |
| 1467 | void movb(Register dst, Address src); |
| 1468 | |
| 1469 | void movddup(XMMRegister dst, XMMRegister src); |
| 1470 | |
| 1471 | void kandbl(KRegister dst, KRegister src1, KRegister src2); |
| 1472 | void kandwl(KRegister dst, KRegister src1, KRegister src2); |
| 1473 | void kanddl(KRegister dst, KRegister src1, KRegister src2); |
| 1474 | void kandql(KRegister dst, KRegister src1, KRegister src2); |
| 1475 | |
| 1476 | void korbl(KRegister dst, KRegister src1, KRegister src2); |
| 1477 | void korwl(KRegister dst, KRegister src1, KRegister src2); |
| 1478 | void kordl(KRegister dst, KRegister src1, KRegister src2); |
| 1479 | void korql(KRegister dst, KRegister src1, KRegister src2); |
| 1480 | |
| 1481 | void kxorbl(KRegister dst, KRegister src1, KRegister src2); |
| 1482 | void kxorwl(KRegister dst, KRegister src1, KRegister src2); |
| 1483 | void kxordl(KRegister dst, KRegister src1, KRegister src2); |
| 1484 | void kxorql(KRegister dst, KRegister src1, KRegister src2); |
| 1485 | void kmovbl(KRegister dst, Register src); |
| 1486 | void kmovbl(Register dst, KRegister src); |
| 1487 | void kmovbl(KRegister dst, KRegister src); |
| 1488 | void kmovwl(KRegister dst, Register src); |
| 1489 | void kmovwl(KRegister dst, Address src); |
| 1490 | void kmovwl(Register dst, KRegister src); |
| 1491 | void kmovwl(Address dst, KRegister src); |
| 1492 | void kmovwl(KRegister dst, KRegister src); |
| 1493 | void kmovdl(KRegister dst, Register src); |
| 1494 | void kmovdl(Register dst, KRegister src); |
| 1495 | void kmovql(KRegister dst, KRegister src); |
| 1496 | void kmovql(Address dst, KRegister src); |
| 1497 | void kmovql(KRegister dst, Address src); |
| 1498 | void kmovql(KRegister dst, Register src); |
| 1499 | void kmovql(Register dst, KRegister src); |
| 1500 | |
| 1501 | void knotbl(KRegister dst, KRegister src); |
| 1502 | void knotwl(KRegister dst, KRegister src); |
| 1503 | void knotdl(KRegister dst, KRegister src); |
| 1504 | void knotql(KRegister dst, KRegister src); |
| 1505 | |
| 1506 | void kortestbl(KRegister dst, KRegister src); |
| 1507 | void kortestwl(KRegister dst, KRegister src); |
| 1508 | void kortestdl(KRegister dst, KRegister src); |
| 1509 | void kortestql(KRegister dst, KRegister src); |
| 1510 | |
| 1511 | void kxnorbl(KRegister dst, KRegister src1, KRegister src2); |
| 1512 | void kshiftlbl(KRegister dst, KRegister src, int imm8); |
| 1513 | void kshiftrbl(KRegister dst, KRegister src, int imm8); |
| 1514 | void kshiftrwl(KRegister dst, KRegister src, int imm8); |
| 1515 | void kshiftrdl(KRegister dst, KRegister src, int imm8); |
| 1516 | void kshiftrql(KRegister dst, KRegister src, int imm8); |
| 1517 | void ktestq(KRegister src1, KRegister src2); |
| 1518 | void ktestd(KRegister src1, KRegister src2); |
| 1519 | |
| 1520 | void ktestql(KRegister dst, KRegister src); |
| 1521 | void ktestdl(KRegister dst, KRegister src); |
| 1522 | void ktestwl(KRegister dst, KRegister src); |
| 1523 | void ktestbl(KRegister dst, KRegister src); |
| 1524 | |
| 1525 | void movdl(XMMRegister dst, Register src); |
| 1526 | void movdl(Register dst, XMMRegister src); |
| 1527 | void movdl(XMMRegister dst, Address src); |
| 1528 | void movdl(Address dst, XMMRegister src); |
| 1529 | |
| 1530 | // Move Double Quadword |
| 1531 | void movdq(XMMRegister dst, Register src); |
| 1532 | void movdq(Register dst, XMMRegister src); |
| 1533 | |
| 1534 | // Move Aligned Double Quadword |
| 1535 | void movdqa(XMMRegister dst, XMMRegister src); |
| 1536 | void movdqa(XMMRegister dst, Address src); |
| 1537 | |
| 1538 | // Move Unaligned Double Quadword |
| 1539 | void movdqu(Address dst, XMMRegister src); |
| 1540 | void movdqu(XMMRegister dst, Address src); |
| 1541 | void movdqu(XMMRegister dst, XMMRegister src); |
| 1542 | |
| 1543 | // Move Unaligned 256bit Vector |
| 1544 | void vmovdqu(Address dst, XMMRegister src); |
| 1545 | void vmovdqu(XMMRegister dst, Address src); |
| 1546 | void vmovdqu(XMMRegister dst, XMMRegister src); |
| 1547 | |
| 1548 | // Move Unaligned 512bit Vector |
| 1549 | void evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len); |
| 1550 | void evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len); |
| 1551 | void evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len); |
| 1552 | void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 1553 | void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 1554 | void evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len); |
| 1555 | void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 1556 | void evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len); |
| 1557 | void evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 1558 | void evmovdqul(Address dst, XMMRegister src, int vector_len); |
| 1559 | void evmovdqul(XMMRegister dst, Address src, int vector_len); |
| 1560 | void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len); |
| 1561 | void evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 1562 | void evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 1563 | void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 1564 | void evmovdquq(Address dst, XMMRegister src, int vector_len); |
| 1565 | void evmovdquq(XMMRegister dst, Address src, int vector_len); |
| 1566 | void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1567 | void evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 1568 | void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 1569 | void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 1570 | |
| 1571 | // Move lower 64bit to high 64bit in 128bit register |
| 1572 | void movlhps(XMMRegister dst, XMMRegister src); |
| 1573 | |
| 1574 | void movl(Register dst, int32_t imm32); |
| 1575 | void movl(Address dst, int32_t imm32); |
| 1576 | void movl(Register dst, Register src); |
| 1577 | void movl(Register dst, Address src); |
| 1578 | void movl(Address dst, Register src); |
| 1579 | |
| 1580 | // These dummies prevent using movl from converting a zero (like NULL) into Register |
| 1581 | // by giving the compiler two choices it can't resolve |
| 1582 | |
| 1583 | void movl(Address dst, void* junk); |
| 1584 | void movl(Register dst, void* junk); |
| 1585 | |
| 1586 | #ifdef _LP641 |
| 1587 | void movq(Register dst, Register src); |
| 1588 | void movq(Register dst, Address src); |
| 1589 | void movq(Address dst, Register src); |
| 1590 | void movq(Address dst, int32_t imm32); |
| 1591 | void movq(Register dst, int32_t imm32); |
| 1592 | |
| 1593 | // These dummies prevent using movq from converting a zero (like NULL) into Register |
| 1594 | // by giving the compiler two choices it can't resolve |
| 1595 | |
| 1596 | void movq(Address dst, void* dummy); |
| 1597 | void movq(Register dst, void* dummy); |
| 1598 | #endif |
| 1599 | |
| 1600 | // Move Quadword |
| 1601 | void movq(Address dst, XMMRegister src); |
| 1602 | void movq(XMMRegister dst, Address src); |
| 1603 | void movq(XMMRegister dst, XMMRegister src); |
| 1604 | void movq(Register dst, XMMRegister src); |
| 1605 | void movq(XMMRegister dst, Register src); |
| 1606 | |
| 1607 | void movsbl(Register dst, Address src); |
| 1608 | void movsbl(Register dst, Register src); |
| 1609 | |
| 1610 | #ifdef _LP641 |
| 1611 | void movsbq(Register dst, Address src); |
| 1612 | void movsbq(Register dst, Register src); |
| 1613 | |
| 1614 | // Move signed 32bit immediate to 64bit extending sign |
| 1615 | void movslq(Address dst, int32_t imm64); |
| 1616 | void movslq(Register dst, int32_t imm64); |
| 1617 | |
| 1618 | void movslq(Register dst, Address src); |
| 1619 | void movslq(Register dst, Register src); |
| 1620 | void movslq(Register dst, void* src); // Dummy declaration to cause NULL to be ambiguous |
| 1621 | #endif |
| 1622 | |
| 1623 | void movswl(Register dst, Address src); |
| 1624 | void movswl(Register dst, Register src); |
| 1625 | |
| 1626 | #ifdef _LP641 |
| 1627 | void movswq(Register dst, Address src); |
| 1628 | void movswq(Register dst, Register src); |
| 1629 | #endif |
| 1630 | |
| 1631 | void movw(Address dst, int imm16); |
| 1632 | void movw(Register dst, Address src); |
| 1633 | void movw(Address dst, Register src); |
| 1634 | |
| 1635 | void movzbl(Register dst, Address src); |
| 1636 | void movzbl(Register dst, Register src); |
| 1637 | |
| 1638 | #ifdef _LP641 |
| 1639 | void movzbq(Register dst, Address src); |
| 1640 | void movzbq(Register dst, Register src); |
| 1641 | #endif |
| 1642 | |
| 1643 | void movzwl(Register dst, Address src); |
| 1644 | void movzwl(Register dst, Register src); |
| 1645 | |
| 1646 | #ifdef _LP641 |
| 1647 | void movzwq(Register dst, Address src); |
| 1648 | void movzwq(Register dst, Register src); |
| 1649 | #endif |
| 1650 | |
| 1651 | // Unsigned multiply with RAX destination register |
| 1652 | void mull(Address src); |
| 1653 | void mull(Register src); |
| 1654 | |
| 1655 | #ifdef _LP641 |
| 1656 | void mulq(Address src); |
| 1657 | void mulq(Register src); |
| 1658 | void mulxq(Register dst1, Register dst2, Register src); |
| 1659 | #endif |
| 1660 | |
| 1661 | // Multiply Scalar Double-Precision Floating-Point Values |
| 1662 | void mulsd(XMMRegister dst, Address src); |
| 1663 | void mulsd(XMMRegister dst, XMMRegister src); |
| 1664 | |
| 1665 | // Multiply Scalar Single-Precision Floating-Point Values |
| 1666 | void mulss(XMMRegister dst, Address src); |
| 1667 | void mulss(XMMRegister dst, XMMRegister src); |
| 1668 | |
| 1669 | void negl(Register dst); |
| 1670 | void negl(Address dst); |
| 1671 | |
| 1672 | #ifdef _LP641 |
| 1673 | void negq(Register dst); |
| 1674 | void negq(Address dst); |
| 1675 | #endif |
| 1676 | |
| 1677 | void nop(int i = 1); |
| 1678 | |
| 1679 | void notl(Register dst); |
| 1680 | |
| 1681 | #ifdef _LP641 |
| 1682 | void notq(Register dst); |
| 1683 | |
| 1684 | void btsq(Address dst, int imm8); |
| 1685 | void btrq(Address dst, int imm8); |
| 1686 | #endif |
| 1687 | |
| 1688 | void orw(Register dst, Register src); |
| 1689 | |
| 1690 | void orl(Address dst, int32_t imm32); |
| 1691 | void orl(Register dst, int32_t imm32); |
| 1692 | void orl(Register dst, Address src); |
| 1693 | void orl(Register dst, Register src); |
| 1694 | void orl(Address dst, Register src); |
| 1695 | |
| 1696 | void orb(Address dst, int imm8); |
| 1697 | void orb(Address dst, Register src); |
| 1698 | |
| 1699 | void orq(Address dst, int32_t imm32); |
| 1700 | void orq(Address dst, Register src); |
| 1701 | void orq(Register dst, int32_t imm32); |
| 1702 | void orq(Register dst, Address src); |
| 1703 | void orq(Register dst, Register src); |
| 1704 | |
| 1705 | // Pack with signed saturation |
| 1706 | void packsswb(XMMRegister dst, XMMRegister src); |
| 1707 | void vpacksswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1708 | void packssdw(XMMRegister dst, XMMRegister src); |
| 1709 | void vpackssdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1710 | |
| 1711 | // Pack with unsigned saturation |
| 1712 | void packuswb(XMMRegister dst, XMMRegister src); |
| 1713 | void packuswb(XMMRegister dst, Address src); |
| 1714 | void packusdw(XMMRegister dst, XMMRegister src); |
| 1715 | void vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1716 | void vpackusdw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1717 | |
| 1718 | // Permutations |
| 1719 | void vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len); |
| 1720 | void vpermq(XMMRegister dst, XMMRegister src, int imm8); |
| 1721 | void vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1722 | void vpermb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1723 | void vpermb(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 1724 | void vpermw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1725 | void vpermd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 1726 | void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1727 | void vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8); |
| 1728 | void vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8); |
| 1729 | void vpermilps(XMMRegister dst, XMMRegister src, int imm8, int vector_len); |
| 1730 | void vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len); |
| 1731 | void vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len); |
| 1732 | void evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1733 | void evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1734 | void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len); |
| 1735 | |
| 1736 | void pause(); |
| 1737 | |
| 1738 | // Undefined Instruction |
| 1739 | void ud2(); |
| 1740 | |
| 1741 | // SSE4.2 string instructions |
| 1742 | void pcmpestri(XMMRegister xmm1, XMMRegister xmm2, int imm8); |
| 1743 | void pcmpestri(XMMRegister xmm1, Address src, int imm8); |
| 1744 | |
| 1745 | void pcmpeqb(XMMRegister dst, XMMRegister src); |
| 1746 | void vpcmpCCbwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len); |
| 1747 | |
| 1748 | void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1749 | void evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1750 | void evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len); |
| 1751 | void evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len); |
| 1752 | |
| 1753 | void vpcmpgtb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1754 | void evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len); |
| 1755 | void evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len); |
| 1756 | |
| 1757 | void evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len); |
| 1758 | void evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len); |
| 1759 | |
| 1760 | void pcmpeqw(XMMRegister dst, XMMRegister src); |
| 1761 | void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1762 | void evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1763 | void evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len); |
| 1764 | |
| 1765 | void vpcmpgtw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1766 | |
| 1767 | void pcmpeqd(XMMRegister dst, XMMRegister src); |
| 1768 | void vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1769 | void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int vector_len); |
| 1770 | void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len); |
| 1771 | |
| 1772 | void pcmpeqq(XMMRegister dst, XMMRegister src); |
| 1773 | void vpcmpCCq(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, int vector_len); |
| 1774 | void vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1775 | void evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1776 | void evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len); |
| 1777 | |
| 1778 | void pcmpgtq(XMMRegister dst, XMMRegister src); |
| 1779 | void vpcmpgtq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1780 | |
| 1781 | void pmovmskb(Register dst, XMMRegister src); |
| 1782 | void vpmovmskb(Register dst, XMMRegister src, int vec_enc); |
| 1783 | void vmovmskps(Register dst, XMMRegister src, int vec_enc); |
| 1784 | void vmovmskpd(Register dst, XMMRegister src, int vec_enc); |
| 1785 | void vpmaskmovd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 1786 | |
| 1787 | // SSE 4.1 extract |
| 1788 | void pextrd(Register dst, XMMRegister src, int imm8); |
| 1789 | void pextrq(Register dst, XMMRegister src, int imm8); |
| 1790 | void pextrd(Address dst, XMMRegister src, int imm8); |
| 1791 | void pextrq(Address dst, XMMRegister src, int imm8); |
| 1792 | void pextrb(Register dst, XMMRegister src, int imm8); |
| 1793 | void pextrb(Address dst, XMMRegister src, int imm8); |
| 1794 | // SSE 2 extract |
| 1795 | void pextrw(Register dst, XMMRegister src, int imm8); |
| 1796 | void pextrw(Address dst, XMMRegister src, int imm8); |
| 1797 | |
| 1798 | // SSE 4.1 insert |
| 1799 | void pinsrd(XMMRegister dst, Register src, int imm8); |
| 1800 | void pinsrq(XMMRegister dst, Register src, int imm8); |
| 1801 | void pinsrb(XMMRegister dst, Register src, int imm8); |
| 1802 | void pinsrd(XMMRegister dst, Address src, int imm8); |
| 1803 | void pinsrq(XMMRegister dst, Address src, int imm8); |
| 1804 | void pinsrb(XMMRegister dst, Address src, int imm8); |
| 1805 | void insertps(XMMRegister dst, XMMRegister src, int imm8); |
| 1806 | // SSE 2 insert |
| 1807 | void pinsrw(XMMRegister dst, Register src, int imm8); |
| 1808 | void pinsrw(XMMRegister dst, Address src, int imm8); |
| 1809 | |
| 1810 | // AVX insert |
| 1811 | void vpinsrd(XMMRegister dst, XMMRegister nds, Register src, int imm8); |
| 1812 | void vpinsrb(XMMRegister dst, XMMRegister nds, Register src, int imm8); |
| 1813 | void vpinsrq(XMMRegister dst, XMMRegister nds, Register src, int imm8); |
| 1814 | void vpinsrw(XMMRegister dst, XMMRegister nds, Register src, int imm8); |
| 1815 | void vinsertps(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8); |
| 1816 | |
| 1817 | // Zero extend moves |
| 1818 | void pmovzxbw(XMMRegister dst, XMMRegister src); |
| 1819 | void pmovzxbw(XMMRegister dst, Address src); |
| 1820 | void pmovzxbd(XMMRegister dst, XMMRegister src); |
| 1821 | void vpmovzxbw( XMMRegister dst, Address src, int vector_len); |
| 1822 | void pmovzxdq(XMMRegister dst, XMMRegister src); |
| 1823 | void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len); |
| 1824 | void vpmovzxdq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1825 | void vpmovzxbd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1826 | void vpmovzxbq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1827 | void evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len); |
| 1828 | |
| 1829 | // Sign extend moves |
| 1830 | void pmovsxbd(XMMRegister dst, XMMRegister src); |
| 1831 | void pmovsxbq(XMMRegister dst, XMMRegister src); |
| 1832 | void pmovsxbw(XMMRegister dst, XMMRegister src); |
| 1833 | void pmovsxwd(XMMRegister dst, XMMRegister src); |
| 1834 | void vpmovsxbd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1835 | void vpmovsxbq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1836 | void vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len); |
| 1837 | void vpmovsxwd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1838 | void vpmovsxwq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1839 | void vpmovsxdq(XMMRegister dst, XMMRegister src, int vector_len); |
| 1840 | |
| 1841 | void evpmovwb(Address dst, XMMRegister src, int vector_len); |
| 1842 | void evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len); |
| 1843 | |
| 1844 | void vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1845 | |
| 1846 | void evpmovdb(Address dst, XMMRegister src, int vector_len); |
| 1847 | |
| 1848 | // Multiply add |
| 1849 | void pmaddwd(XMMRegister dst, XMMRegister src); |
| 1850 | void vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1851 | void vpmaddubsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 1852 | |
| 1853 | // Multiply add accumulate |
| 1854 | void evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1855 | |
| 1856 | #ifndef _LP641 // no 32bit push/pop on amd64 |
| 1857 | void popl(Address dst); |
| 1858 | #endif |
| 1859 | |
| 1860 | #ifdef _LP641 |
| 1861 | void popq(Address dst); |
| 1862 | void popq(Register dst); |
| 1863 | #endif |
| 1864 | |
| 1865 | void popcntl(Register dst, Address src); |
| 1866 | void popcntl(Register dst, Register src); |
| 1867 | |
| 1868 | void vpopcntd(XMMRegister dst, XMMRegister src, int vector_len); |
| 1869 | |
| 1870 | #ifdef _LP641 |
| 1871 | void popcntq(Register dst, Address src); |
| 1872 | void popcntq(Register dst, Register src); |
| 1873 | #endif |
| 1874 | |
| 1875 | // Prefetches (SSE, SSE2, 3DNOW only) |
| 1876 | |
| 1877 | void prefetchnta(Address src); |
| 1878 | void prefetchr(Address src); |
| 1879 | void prefetcht0(Address src); |
| 1880 | void prefetcht1(Address src); |
| 1881 | void prefetcht2(Address src); |
| 1882 | void prefetchw(Address src); |
| 1883 | |
| 1884 | // Shuffle Bytes |
| 1885 | void pshufb(XMMRegister dst, XMMRegister src); |
| 1886 | void pshufb(XMMRegister dst, Address src); |
| 1887 | void vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1888 | |
| 1889 | // Shuffle Packed Doublewords |
| 1890 | void pshufd(XMMRegister dst, XMMRegister src, int mode); |
| 1891 | void pshufd(XMMRegister dst, Address src, int mode); |
| 1892 | void vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len); |
| 1893 | |
| 1894 | // Shuffle Packed High/Low Words |
| 1895 | void pshufhw(XMMRegister dst, XMMRegister src, int mode); |
| 1896 | void pshuflw(XMMRegister dst, XMMRegister src, int mode); |
| 1897 | void pshuflw(XMMRegister dst, Address src, int mode); |
| 1898 | |
| 1899 | //shuffle floats and doubles |
| 1900 | void pshufps(XMMRegister, XMMRegister, int); |
| 1901 | void pshufpd(XMMRegister, XMMRegister, int); |
| 1902 | void vpshufps(XMMRegister, XMMRegister, XMMRegister, int, int); |
| 1903 | void vpshufpd(XMMRegister, XMMRegister, XMMRegister, int, int); |
| 1904 | |
| 1905 | // Shuffle packed values at 128 bit granularity |
| 1906 | void evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len); |
| 1907 | |
| 1908 | // Shift Right by bytes Logical DoubleQuadword Immediate |
| 1909 | void psrldq(XMMRegister dst, int shift); |
| 1910 | // Shift Left by bytes Logical DoubleQuadword Immediate |
| 1911 | void pslldq(XMMRegister dst, int shift); |
| 1912 | |
| 1913 | // Logical Compare 128bit |
| 1914 | void ptest(XMMRegister dst, XMMRegister src); |
| 1915 | void ptest(XMMRegister dst, Address src); |
| 1916 | // Logical Compare 256bit |
| 1917 | void vptest(XMMRegister dst, XMMRegister src); |
| 1918 | void vptest(XMMRegister dst, Address src); |
| 1919 | |
| 1920 | void evptestmb(KRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 1921 | |
| 1922 | // Vector compare |
| 1923 | void vptest(XMMRegister dst, XMMRegister src, int vector_len); |
| 1924 | |
| 1925 | // Interleave Low Bytes |
| 1926 | void punpcklbw(XMMRegister dst, XMMRegister src); |
| 1927 | void punpcklbw(XMMRegister dst, Address src); |
| 1928 | |
| 1929 | // Interleave Low Doublewords |
| 1930 | void punpckldq(XMMRegister dst, XMMRegister src); |
| 1931 | void punpckldq(XMMRegister dst, Address src); |
| 1932 | |
| 1933 | // Interleave Low Quadwords |
| 1934 | void punpcklqdq(XMMRegister dst, XMMRegister src); |
| 1935 | |
| 1936 | #ifndef _LP641 // no 32bit push/pop on amd64 |
| 1937 | void pushl(Address src); |
| 1938 | #endif |
| 1939 | |
| 1940 | void pushq(Address src); |
| 1941 | |
| 1942 | void rcll(Register dst, int imm8); |
| 1943 | |
| 1944 | void rclq(Register dst, int imm8); |
| 1945 | |
| 1946 | void rcrq(Register dst, int imm8); |
| 1947 | |
| 1948 | void rcpps(XMMRegister dst, XMMRegister src); |
| 1949 | |
| 1950 | void rcpss(XMMRegister dst, XMMRegister src); |
| 1951 | |
| 1952 | void rdtsc(); |
| 1953 | |
| 1954 | void ret(int imm16); |
| 1955 | |
| 1956 | void roll(Register dst); |
| 1957 | |
| 1958 | void roll(Register dst, int imm8); |
| 1959 | |
| 1960 | void rorl(Register dst); |
| 1961 | |
| 1962 | void rorl(Register dst, int imm8); |
| 1963 | |
| 1964 | #ifdef _LP641 |
| 1965 | void rolq(Register dst); |
| 1966 | void rolq(Register dst, int imm8); |
| 1967 | void rorq(Register dst); |
| 1968 | void rorq(Register dst, int imm8); |
| 1969 | void rorxq(Register dst, Register src, int imm8); |
| 1970 | void rorxd(Register dst, Register src, int imm8); |
| 1971 | #endif |
| 1972 | |
| 1973 | void sahf(); |
| 1974 | |
| 1975 | void sall(Register dst, int imm8); |
| 1976 | void sall(Register dst); |
| 1977 | void sall(Address dst, int imm8); |
| 1978 | void sall(Address dst); |
| 1979 | |
| 1980 | void sarl(Address dst, int imm8); |
| 1981 | void sarl(Address dst); |
| 1982 | void sarl(Register dst, int imm8); |
| 1983 | void sarl(Register dst); |
| 1984 | |
| 1985 | #ifdef _LP641 |
| 1986 | void salq(Register dst, int imm8); |
| 1987 | void salq(Register dst); |
| 1988 | void salq(Address dst, int imm8); |
| 1989 | void salq(Address dst); |
| 1990 | |
| 1991 | void sarq(Address dst, int imm8); |
| 1992 | void sarq(Address dst); |
| 1993 | void sarq(Register dst, int imm8); |
| 1994 | void sarq(Register dst); |
| 1995 | #endif |
| 1996 | |
| 1997 | void sbbl(Address dst, int32_t imm32); |
| 1998 | void sbbl(Register dst, int32_t imm32); |
| 1999 | void sbbl(Register dst, Address src); |
| 2000 | void sbbl(Register dst, Register src); |
| 2001 | |
| 2002 | void sbbq(Address dst, int32_t imm32); |
| 2003 | void sbbq(Register dst, int32_t imm32); |
| 2004 | void sbbq(Register dst, Address src); |
| 2005 | void sbbq(Register dst, Register src); |
| 2006 | |
| 2007 | void setb(Condition cc, Register dst); |
| 2008 | |
| 2009 | void sete(Register dst); |
| 2010 | void setl(Register dst); |
| 2011 | void setne(Register dst); |
| 2012 | |
| 2013 | void palignr(XMMRegister dst, XMMRegister src, int imm8); |
| 2014 | void vpalignr(XMMRegister dst, XMMRegister src1, XMMRegister src2, int imm8, int vector_len); |
| 2015 | void evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2016 | |
| 2017 | void pblendw(XMMRegister dst, XMMRegister src, int imm8); |
| 2018 | void vblendps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int imm8, int vector_len); |
| 2019 | |
| 2020 | void sha1rnds4(XMMRegister dst, XMMRegister src, int imm8); |
| 2021 | void sha1nexte(XMMRegister dst, XMMRegister src); |
| 2022 | void sha1msg1(XMMRegister dst, XMMRegister src); |
| 2023 | void sha1msg2(XMMRegister dst, XMMRegister src); |
| 2024 | // xmm0 is implicit additional source to the following instruction. |
| 2025 | void sha256rnds2(XMMRegister dst, XMMRegister src); |
| 2026 | void sha256msg1(XMMRegister dst, XMMRegister src); |
| 2027 | void sha256msg2(XMMRegister dst, XMMRegister src); |
| 2028 | |
| 2029 | void shldl(Register dst, Register src); |
| 2030 | void shldl(Register dst, Register src, int8_t imm8); |
| 2031 | void shrdl(Register dst, Register src); |
| 2032 | void shrdl(Register dst, Register src, int8_t imm8); |
| 2033 | |
| 2034 | void shll(Register dst, int imm8); |
| 2035 | void shll(Register dst); |
| 2036 | |
| 2037 | void shlq(Register dst, int imm8); |
| 2038 | void shlq(Register dst); |
| 2039 | |
| 2040 | void shrl(Register dst, int imm8); |
| 2041 | void shrl(Register dst); |
| 2042 | void shrl(Address dst); |
| 2043 | void shrl(Address dst, int imm8); |
| 2044 | |
| 2045 | void shrq(Register dst, int imm8); |
| 2046 | void shrq(Register dst); |
| 2047 | void shrq(Address dst); |
| 2048 | void shrq(Address dst, int imm8); |
| 2049 | |
| 2050 | void smovl(); // QQQ generic? |
| 2051 | |
| 2052 | // Compute Square Root of Scalar Double-Precision Floating-Point Value |
| 2053 | void sqrtsd(XMMRegister dst, Address src); |
| 2054 | void sqrtsd(XMMRegister dst, XMMRegister src); |
| 2055 | |
| 2056 | void roundsd(XMMRegister dst, Address src, int32_t rmode); |
| 2057 | void roundsd(XMMRegister dst, XMMRegister src, int32_t rmode); |
| 2058 | |
| 2059 | // Compute Square Root of Scalar Single-Precision Floating-Point Value |
| 2060 | void sqrtss(XMMRegister dst, Address src); |
| 2061 | void sqrtss(XMMRegister dst, XMMRegister src); |
| 2062 | |
| 2063 | void std(); |
| 2064 | |
| 2065 | void stmxcsr( Address dst ); |
| 2066 | |
| 2067 | void subl(Address dst, int32_t imm32); |
| 2068 | void subl(Address dst, Register src); |
| 2069 | void subl(Register dst, int32_t imm32); |
| 2070 | void subl(Register dst, Address src); |
| 2071 | void subl(Register dst, Register src); |
| 2072 | |
| 2073 | void subq(Address dst, int32_t imm32); |
| 2074 | void subq(Address dst, Register src); |
| 2075 | void subq(Register dst, int32_t imm32); |
| 2076 | void subq(Register dst, Address src); |
| 2077 | void subq(Register dst, Register src); |
| 2078 | |
| 2079 | // Force generation of a 4 byte immediate value even if it fits into 8bit |
| 2080 | void subl_imm32(Register dst, int32_t imm32); |
| 2081 | void subq_imm32(Register dst, int32_t imm32); |
| 2082 | |
| 2083 | // Subtract Scalar Double-Precision Floating-Point Values |
| 2084 | void subsd(XMMRegister dst, Address src); |
| 2085 | void subsd(XMMRegister dst, XMMRegister src); |
| 2086 | |
| 2087 | // Subtract Scalar Single-Precision Floating-Point Values |
| 2088 | void subss(XMMRegister dst, Address src); |
| 2089 | void subss(XMMRegister dst, XMMRegister src); |
| 2090 | |
| 2091 | void testb(Register dst, int imm8); |
| 2092 | void testb(Address dst, int imm8); |
| 2093 | |
| 2094 | void testl(Register dst, int32_t imm32); |
| 2095 | void testl(Register dst, Register src); |
| 2096 | void testl(Register dst, Address src); |
| 2097 | |
| 2098 | void testq(Address dst, int32_t imm32); |
| 2099 | void testq(Register dst, int32_t imm32); |
| 2100 | void testq(Register dst, Register src); |
| 2101 | void testq(Register dst, Address src); |
| 2102 | |
| 2103 | // BMI - count trailing zeros |
| 2104 | void tzcntl(Register dst, Register src); |
| 2105 | void tzcntq(Register dst, Register src); |
| 2106 | |
| 2107 | // Unordered Compare Scalar Double-Precision Floating-Point Values and set EFLAGS |
| 2108 | void ucomisd(XMMRegister dst, Address src); |
| 2109 | void ucomisd(XMMRegister dst, XMMRegister src); |
| 2110 | |
| 2111 | // Unordered Compare Scalar Single-Precision Floating-Point Values and set EFLAGS |
| 2112 | void ucomiss(XMMRegister dst, Address src); |
| 2113 | void ucomiss(XMMRegister dst, XMMRegister src); |
| 2114 | |
| 2115 | void xabort(int8_t imm8); |
| 2116 | |
| 2117 | void xaddb(Address dst, Register src); |
| 2118 | void xaddw(Address dst, Register src); |
| 2119 | void xaddl(Address dst, Register src); |
| 2120 | void xaddq(Address dst, Register src); |
| 2121 | |
| 2122 | void xbegin(Label& abort, relocInfo::relocType rtype = relocInfo::none); |
| 2123 | |
| 2124 | void xchgb(Register reg, Address adr); |
| 2125 | void xchgw(Register reg, Address adr); |
| 2126 | void xchgl(Register reg, Address adr); |
| 2127 | void xchgl(Register dst, Register src); |
| 2128 | |
| 2129 | void xchgq(Register reg, Address adr); |
| 2130 | void xchgq(Register dst, Register src); |
| 2131 | |
| 2132 | void xend(); |
| 2133 | |
| 2134 | // Get Value of Extended Control Register |
| 2135 | void xgetbv(); |
| 2136 | |
| 2137 | void xorl(Register dst, int32_t imm32); |
| 2138 | void xorl(Address dst, int32_t imm32); |
| 2139 | void xorl(Register dst, Address src); |
| 2140 | void xorl(Register dst, Register src); |
| 2141 | void xorl(Address dst, Register src); |
| 2142 | |
| 2143 | void xorb(Address dst, Register src); |
| 2144 | void xorb(Register dst, Address src); |
| 2145 | void xorw(Register dst, Register src); |
| 2146 | |
| 2147 | void xorq(Register dst, Address src); |
| 2148 | void xorq(Address dst, int32_t imm32); |
| 2149 | void xorq(Register dst, Register src); |
| 2150 | void xorq(Register dst, int32_t imm32); |
| 2151 | void xorq(Address dst, Register src); |
| 2152 | |
| 2153 | void set_byte_if_not_zero(Register dst); // sets reg to 1 if not zero, otherwise 0 |
| 2154 | |
| 2155 | // AVX 3-operands scalar instructions (encoded with VEX prefix) |
| 2156 | |
| 2157 | void vaddsd(XMMRegister dst, XMMRegister nds, Address src); |
| 2158 | void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2159 | void vaddss(XMMRegister dst, XMMRegister nds, Address src); |
| 2160 | void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2161 | void vdivsd(XMMRegister dst, XMMRegister nds, Address src); |
| 2162 | void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2163 | void vdivss(XMMRegister dst, XMMRegister nds, Address src); |
| 2164 | void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2165 | void vfmadd231sd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2166 | void vfmadd231ss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2167 | void vmulsd(XMMRegister dst, XMMRegister nds, Address src); |
| 2168 | void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2169 | void vmulss(XMMRegister dst, XMMRegister nds, Address src); |
| 2170 | void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2171 | void vsubsd(XMMRegister dst, XMMRegister nds, Address src); |
| 2172 | void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2173 | void vsubss(XMMRegister dst, XMMRegister nds, Address src); |
| 2174 | void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2175 | |
| 2176 | void vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2177 | void vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2178 | void vminss(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2179 | void vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src); |
| 2180 | |
| 2181 | void shlxl(Register dst, Register src1, Register src2); |
| 2182 | void shlxq(Register dst, Register src1, Register src2); |
| 2183 | void shrxl(Register dst, Register src1, Register src2); |
| 2184 | void shrxq(Register dst, Register src1, Register src2); |
| 2185 | |
| 2186 | void bzhiq(Register dst, Register src1, Register src2); |
| 2187 | void pdep(Register dst, Register src1, Register src2); |
| 2188 | void pext(Register dst, Register src1, Register src2); |
| 2189 | |
| 2190 | |
| 2191 | //====================VECTOR ARITHMETIC===================================== |
| 2192 | // Add Packed Floating-Point Values |
| 2193 | void addpd(XMMRegister dst, XMMRegister src); |
| 2194 | void addpd(XMMRegister dst, Address src); |
| 2195 | void addps(XMMRegister dst, XMMRegister src); |
| 2196 | void vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2197 | void vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2198 | void vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2199 | void vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2200 | |
| 2201 | // Subtract Packed Floating-Point Values |
| 2202 | void subpd(XMMRegister dst, XMMRegister src); |
| 2203 | void subps(XMMRegister dst, XMMRegister src); |
| 2204 | void vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2205 | void vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2206 | void vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2207 | void vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2208 | |
| 2209 | // Multiply Packed Floating-Point Values |
| 2210 | void mulpd(XMMRegister dst, XMMRegister src); |
| 2211 | void mulpd(XMMRegister dst, Address src); |
| 2212 | void mulps(XMMRegister dst, XMMRegister src); |
| 2213 | void vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2214 | void vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2215 | void vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2216 | void vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2217 | |
| 2218 | void vfmadd231pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2219 | void vfmadd231ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2220 | void vfmadd231pd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2221 | void vfmadd231ps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2222 | |
| 2223 | // Divide Packed Floating-Point Values |
| 2224 | void divpd(XMMRegister dst, XMMRegister src); |
| 2225 | void divps(XMMRegister dst, XMMRegister src); |
| 2226 | void vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2227 | void vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2228 | void vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2229 | void vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2230 | |
| 2231 | // Sqrt Packed Floating-Point Values |
| 2232 | void vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len); |
| 2233 | void vsqrtpd(XMMRegister dst, Address src, int vector_len); |
| 2234 | void vsqrtps(XMMRegister dst, XMMRegister src, int vector_len); |
| 2235 | void vsqrtps(XMMRegister dst, Address src, int vector_len); |
| 2236 | |
| 2237 | // Round Packed Double precision value. |
| 2238 | void vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len); |
| 2239 | void vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len); |
| 2240 | void vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len); |
| 2241 | void vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len); |
| 2242 | |
| 2243 | // Bitwise Logical AND of Packed Floating-Point Values |
| 2244 | void andpd(XMMRegister dst, XMMRegister src); |
| 2245 | void andps(XMMRegister dst, XMMRegister src); |
| 2246 | void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2247 | void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2248 | void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2249 | void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2250 | |
| 2251 | void unpckhpd(XMMRegister dst, XMMRegister src); |
| 2252 | void unpcklpd(XMMRegister dst, XMMRegister src); |
| 2253 | |
| 2254 | // Bitwise Logical XOR of Packed Floating-Point Values |
| 2255 | void xorpd(XMMRegister dst, XMMRegister src); |
| 2256 | void xorps(XMMRegister dst, XMMRegister src); |
| 2257 | void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2258 | void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2259 | void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2260 | void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2261 | |
| 2262 | // Add horizontal packed integers |
| 2263 | void vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2264 | void vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2265 | void phaddw(XMMRegister dst, XMMRegister src); |
| 2266 | void phaddd(XMMRegister dst, XMMRegister src); |
| 2267 | |
| 2268 | // Add packed integers |
| 2269 | void paddb(XMMRegister dst, XMMRegister src); |
| 2270 | void paddw(XMMRegister dst, XMMRegister src); |
| 2271 | void paddd(XMMRegister dst, XMMRegister src); |
| 2272 | void paddd(XMMRegister dst, Address src); |
| 2273 | void paddq(XMMRegister dst, XMMRegister src); |
| 2274 | void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2275 | void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2276 | void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2277 | void vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2278 | void vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2279 | void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2280 | void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2281 | void vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2282 | |
| 2283 | // Leaf level assembler routines for masked operations. |
| 2284 | void evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2285 | void evpaddb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2286 | void evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2287 | void evpaddw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2288 | void evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2289 | void evpaddd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2290 | void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2291 | void evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2292 | void evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2293 | void evaddps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2294 | void evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2295 | void evaddpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2296 | void evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2297 | void evpsubb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2298 | void evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2299 | void evpsubw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2300 | void evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2301 | void evpsubd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2302 | void evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2303 | void evpsubq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2304 | void evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2305 | void evsubps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2306 | void evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2307 | void evsubpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2308 | void evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2309 | void evpmullw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2310 | void evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2311 | void evpmulld(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2312 | void evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2313 | void evpmullq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2314 | void evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2315 | void evmulps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2316 | void evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2317 | void evmulpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2318 | void evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2319 | void evdivps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2320 | void evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2321 | void evdivpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2322 | void evpabsb(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 2323 | void evpabsb(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 2324 | void evpabsw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 2325 | void evpabsw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 2326 | void evpabsd(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 2327 | void evpabsd(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 2328 | void evpabsq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len); |
| 2329 | void evpabsq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len); |
| 2330 | void evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2331 | void evpfma213ps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2332 | void evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2333 | void evpfma213pd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2334 | void evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2335 | void evpermb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2336 | void evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2337 | void evpermw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2338 | void evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2339 | void evpermd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2340 | void evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2341 | void evpermq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2342 | void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2343 | void evpslld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2344 | void evpsllq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2345 | void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2346 | void evpsrld(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2347 | void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2348 | void evpsraw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2349 | void evpsrad(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2350 | void evpsraq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2351 | void evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2352 | void evsqrtps(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2353 | void evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2354 | void evsqrtpd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2355 | |
| 2356 | void evpsllw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2357 | void evpslld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2358 | void evpsllq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2359 | void evpsrlw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2360 | void evpsrld(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2361 | void evpsrlq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2362 | void evpsraw(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2363 | void evpsrad(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2364 | void evpsraq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2365 | |
| 2366 | void evpsllvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2367 | void evpsllvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2368 | void evpsllvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2369 | void evpsrlvw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2370 | void evpsrlvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2371 | void evpsrlvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2372 | void evpsravw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2373 | void evpsravd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2374 | void evpsravq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2375 | void evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2376 | void evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2377 | void evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2378 | void evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2379 | void evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2380 | void evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2381 | void evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2382 | void evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2383 | void evpmaxsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2384 | void evpmaxsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2385 | void evpmaxsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2386 | void evpmaxsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2387 | void evpminsb(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2388 | void evpminsw(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2389 | void evpminsd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2390 | void evpminsq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2391 | void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2392 | void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2393 | void evporq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2394 | void evporq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2395 | void evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2396 | void evpandd(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2397 | void evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2398 | void evpandq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2399 | void evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2400 | void evpxord(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2401 | void evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2402 | void evpxorq(XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len); |
| 2403 | |
| 2404 | void evprold(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2405 | void evprolq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2406 | void evprolvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2407 | void evprolvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2408 | void evprord(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2409 | void evprorq(XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vector_len); |
| 2410 | void evprorvd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2411 | void evprorvq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2412 | |
| 2413 | // Sub packed integers |
| 2414 | void psubb(XMMRegister dst, XMMRegister src); |
| 2415 | void psubw(XMMRegister dst, XMMRegister src); |
| 2416 | void psubd(XMMRegister dst, XMMRegister src); |
| 2417 | void psubq(XMMRegister dst, XMMRegister src); |
| 2418 | void vpsubusb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2419 | void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2420 | void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2421 | void vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2422 | void vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2423 | void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2424 | void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2425 | void vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2426 | void vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2427 | |
| 2428 | // Multiply packed integers (only shorts and ints) |
| 2429 | void pmullw(XMMRegister dst, XMMRegister src); |
| 2430 | void pmulld(XMMRegister dst, XMMRegister src); |
| 2431 | void pmuludq(XMMRegister dst, XMMRegister src); |
| 2432 | void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2433 | void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2434 | void vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2435 | void vpmuludq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2436 | void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2437 | void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2438 | void vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2439 | void vpmulhuw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2440 | |
| 2441 | // Minimum of packed integers |
| 2442 | void pminsb(XMMRegister dst, XMMRegister src); |
| 2443 | void vpminsb(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2444 | void pminsw(XMMRegister dst, XMMRegister src); |
| 2445 | void vpminsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2446 | void pminsd(XMMRegister dst, XMMRegister src); |
| 2447 | void vpminsd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2448 | void vpminsq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2449 | void minps(XMMRegister dst, XMMRegister src); |
| 2450 | void vminps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2451 | void minpd(XMMRegister dst, XMMRegister src); |
| 2452 | void vminpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2453 | |
| 2454 | // Maximum of packed integers |
| 2455 | void pmaxsb(XMMRegister dst, XMMRegister src); |
| 2456 | void vpmaxsb(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2457 | void pmaxsw(XMMRegister dst, XMMRegister src); |
| 2458 | void vpmaxsw(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2459 | void pmaxsd(XMMRegister dst, XMMRegister src); |
| 2460 | void vpmaxsd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2461 | void vpmaxsq(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2462 | void maxps(XMMRegister dst, XMMRegister src); |
| 2463 | void vmaxps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2464 | void maxpd(XMMRegister dst, XMMRegister src); |
| 2465 | void vmaxpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2466 | |
| 2467 | // Shift left packed integers |
| 2468 | void psllw(XMMRegister dst, int shift); |
| 2469 | void pslld(XMMRegister dst, int shift); |
| 2470 | void psllq(XMMRegister dst, int shift); |
| 2471 | void psllw(XMMRegister dst, XMMRegister shift); |
| 2472 | void pslld(XMMRegister dst, XMMRegister shift); |
| 2473 | void psllq(XMMRegister dst, XMMRegister shift); |
| 2474 | void vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2475 | void vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2476 | void vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2477 | void vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2478 | void vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2479 | void vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2480 | void vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2481 | |
| 2482 | // Logical shift right packed integers |
| 2483 | void psrlw(XMMRegister dst, int shift); |
| 2484 | void psrld(XMMRegister dst, int shift); |
| 2485 | void psrlq(XMMRegister dst, int shift); |
| 2486 | void psrlw(XMMRegister dst, XMMRegister shift); |
| 2487 | void psrld(XMMRegister dst, XMMRegister shift); |
| 2488 | void psrlq(XMMRegister dst, XMMRegister shift); |
| 2489 | void vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2490 | void vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2491 | void vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2492 | void vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2493 | void vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2494 | void vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2495 | void vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2496 | void evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2497 | void evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2498 | |
| 2499 | // Arithmetic shift right packed integers (only shorts and ints, no instructions for longs) |
| 2500 | void psraw(XMMRegister dst, int shift); |
| 2501 | void psrad(XMMRegister dst, int shift); |
| 2502 | void psraw(XMMRegister dst, XMMRegister shift); |
| 2503 | void psrad(XMMRegister dst, XMMRegister shift); |
| 2504 | void vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2505 | void vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2506 | void vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2507 | void vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2508 | void evpsravw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2509 | void evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2510 | void evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2511 | |
| 2512 | // Variable shift left packed integers |
| 2513 | void vpsllvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2514 | void vpsllvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2515 | |
| 2516 | // Variable shift right packed integers |
| 2517 | void vpsrlvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2518 | void vpsrlvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2519 | |
| 2520 | // Variable shift right arithmetic packed integers |
| 2521 | void vpsravd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2522 | void evpsravq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2523 | |
| 2524 | void vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2525 | void vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2526 | |
| 2527 | // And packed integers |
| 2528 | void pand(XMMRegister dst, XMMRegister src); |
| 2529 | void vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2530 | void vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2531 | void vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2532 | |
| 2533 | // Andn packed integers |
| 2534 | void pandn(XMMRegister dst, XMMRegister src); |
| 2535 | void vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2536 | |
| 2537 | // Or packed integers |
| 2538 | void por(XMMRegister dst, XMMRegister src); |
| 2539 | void vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2540 | void vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2541 | void vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2542 | |
| 2543 | // Xor packed integers |
| 2544 | void pxor(XMMRegister dst, XMMRegister src); |
| 2545 | void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2546 | void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2547 | void vpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2548 | void evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2549 | void evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len); |
| 2550 | |
| 2551 | // Ternary logic instruction. |
| 2552 | void vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len); |
| 2553 | void vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len); |
| 2554 | void vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len); |
| 2555 | |
| 2556 | // Vector Rotate Left/Right instruction. |
| 2557 | void evprolvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2558 | void evprolvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2559 | void evprorvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2560 | void evprorvq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len); |
| 2561 | void evprold(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2562 | void evprolq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2563 | void evprord(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2564 | void evprorq(XMMRegister dst, XMMRegister src, int shift, int vector_len); |
| 2565 | |
| 2566 | // vinserti forms |
| 2567 | void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2568 | void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8); |
| 2569 | void vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2570 | void vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8); |
| 2571 | void vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2572 | |
| 2573 | // vinsertf forms |
| 2574 | void vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2575 | void vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8); |
| 2576 | void vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2577 | void vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8); |
| 2578 | void vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8); |
| 2579 | void vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8); |
| 2580 | |
| 2581 | // vextracti forms |
| 2582 | void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2583 | void vextracti128(Address dst, XMMRegister src, uint8_t imm8); |
| 2584 | void vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2585 | void vextracti32x4(Address dst, XMMRegister src, uint8_t imm8); |
| 2586 | void vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2587 | void vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2588 | void vextracti64x4(Address dst, XMMRegister src, uint8_t imm8); |
| 2589 | |
| 2590 | // vextractf forms |
| 2591 | void vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2592 | void vextractf128(Address dst, XMMRegister src, uint8_t imm8); |
| 2593 | void vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2594 | void vextractf32x4(Address dst, XMMRegister src, uint8_t imm8); |
| 2595 | void vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2596 | void vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8); |
| 2597 | void vextractf64x4(Address dst, XMMRegister src, uint8_t imm8); |
| 2598 | |
| 2599 | // xmm/mem sourced byte/word/dword/qword replicate |
| 2600 | void vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len); |
| 2601 | void vpbroadcastb(XMMRegister dst, Address src, int vector_len); |
| 2602 | void vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len); |
| 2603 | void vpbroadcastw(XMMRegister dst, Address src, int vector_len); |
| 2604 | void vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len); |
| 2605 | void vpbroadcastd(XMMRegister dst, Address src, int vector_len); |
| 2606 | void vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len); |
| 2607 | void vpbroadcastq(XMMRegister dst, Address src, int vector_len); |
| 2608 | |
| 2609 | void evbroadcasti32x4(XMMRegister dst, Address src, int vector_len); |
| 2610 | void evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len); |
| 2611 | void evbroadcasti64x2(XMMRegister dst, Address src, int vector_len); |
| 2612 | |
| 2613 | // scalar single/double/128bit precision replicate |
| 2614 | void vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len); |
| 2615 | void vbroadcastss(XMMRegister dst, Address src, int vector_len); |
| 2616 | void vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len); |
| 2617 | void vbroadcastsd(XMMRegister dst, Address src, int vector_len); |
| 2618 | void vbroadcastf128(XMMRegister dst, Address src, int vector_len); |
| 2619 | |
| 2620 | // gpr sourced byte/word/dword/qword replicate |
| 2621 | void evpbroadcastb(XMMRegister dst, Register src, int vector_len); |
| 2622 | void evpbroadcastw(XMMRegister dst, Register src, int vector_len); |
| 2623 | void evpbroadcastd(XMMRegister dst, Register src, int vector_len); |
| 2624 | void evpbroadcastq(XMMRegister dst, Register src, int vector_len); |
| 2625 | |
| 2626 | // Gather AVX2 and AVX3 |
| 2627 | void vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int vector_len); |
| 2628 | void vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int vector_len); |
| 2629 | void vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int vector_len); |
| 2630 | void vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int vector_len); |
| 2631 | void evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len); |
| 2632 | void evpgatherdq(XMMRegister dst, KRegister mask, Address src, int vector_len); |
| 2633 | void evgatherdpd(XMMRegister dst, KRegister mask, Address src, int vector_len); |
| 2634 | void evgatherdps(XMMRegister dst, KRegister mask, Address src, int vector_len); |
| 2635 | |
| 2636 | //Scatter AVX3 only |
| 2637 | void evpscatterdd(Address dst, KRegister mask, XMMRegister src, int vector_len); |
| 2638 | void evpscatterdq(Address dst, KRegister mask, XMMRegister src, int vector_len); |
| 2639 | void evscatterdps(Address dst, KRegister mask, XMMRegister src, int vector_len); |
| 2640 | void evscatterdpd(Address dst, KRegister mask, XMMRegister src, int vector_len); |
| 2641 | |
| 2642 | // Carry-Less Multiplication Quadword |
| 2643 | void pclmulqdq(XMMRegister dst, XMMRegister src, int mask); |
| 2644 | void vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask); |
| 2645 | void evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len); |
| 2646 | // AVX instruction which is used to clear upper 128 bits of YMM registers and |
| 2647 | // to avoid transaction penalty between AVX and SSE states. There is no |
| 2648 | // penalty if legacy SSE instructions are encoded using VEX prefix because |
| 2649 | // they always clear upper 128 bits. It should be used before calling |
| 2650 | // runtime code and native libraries. |
| 2651 | void vzeroupper(); |
| 2652 | |
| 2653 | // Vector double compares |
| 2654 | void vcmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len); |
| 2655 | void evcmppd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, |
| 2656 | ComparisonPredicateFP comparison, int vector_len); |
| 2657 | |
| 2658 | // Vector float compares |
| 2659 | void vcmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int comparison, int vector_len); |
| 2660 | void evcmpps(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, |
| 2661 | ComparisonPredicateFP comparison, int vector_len); |
| 2662 | |
| 2663 | // Vector integer compares |
| 2664 | void vpcmpgtd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); |
| 2665 | void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, |
| 2666 | int comparison, bool is_signed, int vector_len); |
| 2667 | void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, Address src, |
| 2668 | int comparison, bool is_signed, int vector_len); |
| 2669 | |
| 2670 | // Vector long compares |
| 2671 | void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, |
| 2672 | int comparison, bool is_signed, int vector_len); |
| 2673 | void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, Address src, |
| 2674 | int comparison, bool is_signed, int vector_len); |
| 2675 | |
| 2676 | // Vector byte compares |
| 2677 | void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, |
| 2678 | int comparison, bool is_signed, int vector_len); |
| 2679 | void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, |
| 2680 | int comparison, bool is_signed, int vector_len); |
| 2681 | |
| 2682 | // Vector short compares |
| 2683 | void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, |
| 2684 | int comparison, bool is_signed, int vector_len); |
| 2685 | void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src, |
| 2686 | int comparison, bool is_signed, int vector_len); |
| 2687 | |
| 2688 | void evpmovb2m(KRegister dst, XMMRegister src, int vector_len); |
| 2689 | void evpmovw2m(KRegister dst, XMMRegister src, int vector_len); |
| 2690 | void evpmovd2m(KRegister dst, XMMRegister src, int vector_len); |
| 2691 | void evpmovq2m(KRegister dst, XMMRegister src, int vector_len); |
| 2692 | void evpmovm2b(XMMRegister dst, KRegister src, int vector_len); |
| 2693 | void evpmovm2w(XMMRegister dst, KRegister src, int vector_len); |
| 2694 | void evpmovm2d(XMMRegister dst, KRegister src, int vector_len); |
| 2695 | void evpmovm2q(XMMRegister dst, KRegister src, int vector_len); |
| 2696 | |
| 2697 | // Vector blends |
| 2698 | void blendvps(XMMRegister dst, XMMRegister src); |
| 2699 | void blendvpd(XMMRegister dst, XMMRegister src); |
| 2700 | void pblendvb(XMMRegister dst, XMMRegister src); |
| 2701 | void blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2702 | void vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len); |
| 2703 | void vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len); |
| 2704 | void vpblendvb(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len); |
| 2705 | void vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len); |
| 2706 | void evblendmpd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2707 | void evblendmps(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2708 | void evpblendmb(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2709 | void evpblendmw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2710 | void evpblendmd(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2711 | void evpblendmq(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len); |
| 2712 | protected: |
| 2713 | // Next instructions require address alignment 16 bytes SSE mode. |
| 2714 | // They should be called only from corresponding MacroAssembler instructions. |
| 2715 | void andpd(XMMRegister dst, Address src); |
| 2716 | void andps(XMMRegister dst, Address src); |
| 2717 | void xorpd(XMMRegister dst, Address src); |
| 2718 | void xorps(XMMRegister dst, Address src); |
| 2719 | |
| 2720 | }; |
| 2721 | |
| 2722 | // The Intel x86/Amd64 Assembler attributes: All fields enclosed here are to guide encoding level decisions. |
| 2723 | // Specific set functions are for specialized use, else defaults or whatever was supplied to object construction |
| 2724 | // are applied. |
| 2725 | class InstructionAttr { |
| 2726 | public: |
| 2727 | InstructionAttr( |
| 2728 | int vector_len, // The length of vector to be applied in encoding - for both AVX and EVEX |
| 2729 | bool rex_vex_w, // Width of data: if 32-bits or less, false, else if 64-bit or specially defined, true |
| 2730 | bool legacy_mode, // Details if either this instruction is conditionally encoded to AVX or earlier if true else possibly EVEX |
| 2731 | bool no_reg_mask, // when true, k0 is used when EVEX encoding is chosen, else embedded_opmask_register_specifier is used |
| 2732 | bool uses_vl) // This instruction may have legacy constraints based on vector length for EVEX |
| 2733 | : |
| 2734 | _rex_vex_w(rex_vex_w), |
| 2735 | _legacy_mode(legacy_mode || UseAVX < 3), |
| 2736 | _no_reg_mask(no_reg_mask), |
| 2737 | _uses_vl(uses_vl), |
| 2738 | _rex_vex_w_reverted(false), |
| 2739 | _is_evex_instruction(false), |
| 2740 | _is_clear_context(true), |
| 2741 | _is_extended_context(false), |
| 2742 | _avx_vector_len(vector_len), |
| 2743 | _tuple_type(Assembler::EVEX_ETUP), |
| 2744 | _input_size_in_bits(Assembler::EVEX_NObit), |
| 2745 | _evex_encoding(0), |
| 2746 | _embedded_opmask_register_specifier(0), // hard code k0 |
| 2747 | _current_assembler(NULL__null) { } |
| 2748 | |
| 2749 | ~InstructionAttr() { |
| 2750 | if (_current_assembler != NULL__null) { |
| 2751 | _current_assembler->clear_attributes(); |
| 2752 | } |
| 2753 | _current_assembler = NULL__null; |
| 2754 | } |
| 2755 | |
| 2756 | private: |
| 2757 | bool _rex_vex_w; |
| 2758 | bool _legacy_mode; |
| 2759 | bool _no_reg_mask; |
| 2760 | bool _uses_vl; |
| 2761 | bool _rex_vex_w_reverted; |
| 2762 | bool _is_evex_instruction; |
| 2763 | bool _is_clear_context; |
| 2764 | bool _is_extended_context; |
| 2765 | int _avx_vector_len; |
| 2766 | int _tuple_type; |
| 2767 | int _input_size_in_bits; |
| 2768 | int _evex_encoding; |
| 2769 | int _embedded_opmask_register_specifier; |
| 2770 | |
| 2771 | Assembler *_current_assembler; |
| 2772 | |
| 2773 | public: |
| 2774 | // query functions for field accessors |
| 2775 | bool is_rex_vex_w(void) const { return _rex_vex_w; } |
| 2776 | bool is_legacy_mode(void) const { return _legacy_mode; } |
| 2777 | bool is_no_reg_mask(void) const { return _no_reg_mask; } |
| 2778 | bool uses_vl(void) const { return _uses_vl; } |
| 2779 | bool is_rex_vex_w_reverted(void) { return _rex_vex_w_reverted; } |
| 2780 | bool is_evex_instruction(void) const { return _is_evex_instruction; } |
| 2781 | bool is_clear_context(void) const { return _is_clear_context; } |
| 2782 | bool is_extended_context(void) const { return _is_extended_context; } |
| 2783 | int get_vector_len(void) const { return _avx_vector_len; } |
| 2784 | int get_tuple_type(void) const { return _tuple_type; } |
| 2785 | int get_input_size(void) const { return _input_size_in_bits; } |
| 2786 | int get_evex_encoding(void) const { return _evex_encoding; } |
| 2787 | int get_embedded_opmask_register_specifier(void) const { return _embedded_opmask_register_specifier; } |
| 2788 | |
| 2789 | // Set the vector len manually |
| 2790 | void set_vector_len(int vector_len) { _avx_vector_len = vector_len; } |
| 2791 | |
| 2792 | // Set revert rex_vex_w for avx encoding |
| 2793 | void set_rex_vex_w_reverted(void) { _rex_vex_w_reverted = true; } |
| 2794 | |
| 2795 | // Set rex_vex_w based on state |
| 2796 | void set_rex_vex_w(bool state) { _rex_vex_w = state; } |
| 2797 | |
| 2798 | // Set the instruction to be encoded in AVX mode |
| 2799 | void set_is_legacy_mode(void) { _legacy_mode = true; } |
| 2800 | |
| 2801 | // Set the current instuction to be encoded as an EVEX instuction |
| 2802 | void set_is_evex_instruction(void) { _is_evex_instruction = true; } |
| 2803 | |
| 2804 | // Internal encoding data used in compressed immediate offset programming |
| 2805 | void set_evex_encoding(int value) { _evex_encoding = value; } |
| 2806 | |
| 2807 | // When the Evex.Z field is set (true), it is used to clear all non directed XMM/YMM/ZMM components. |
| 2808 | // This method unsets it so that merge semantics are used instead. |
| 2809 | void reset_is_clear_context(void) { _is_clear_context = false; } |
| 2810 | |
| 2811 | // Map back to current asembler so that we can manage object level assocation |
| 2812 | void set_current_assembler(Assembler *current_assembler) { _current_assembler = current_assembler; } |
| 2813 | |
| 2814 | // Address modifiers used for compressed displacement calculation |
| 2815 | void set_address_attributes(int tuple_type, int input_size_in_bits); |
| 2816 | |
| 2817 | // Set embedded opmask register specifier. |
| 2818 | void set_embedded_opmask_register_specifier(KRegister mask) { |
| 2819 | _embedded_opmask_register_specifier = (*mask).encoding() & 0x7; |
| 2820 | } |
| 2821 | |
| 2822 | }; |
| 2823 | |
| 2824 | #endif // CPU_X86_ASSEMBLER_X86_HPP |