| File: | jdk/src/hotspot/share/opto/vector.cpp |
| Warning: | line 202, column 7 Value stored to 'jvms' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* |
| 2 | * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #include "precompiled.hpp" |
| 26 | #include "ci/ciSymbols.hpp" |
| 27 | #include "gc/shared/barrierSet.hpp" |
| 28 | #include "opto/castnode.hpp" |
| 29 | #include "opto/graphKit.hpp" |
| 30 | #include "opto/phaseX.hpp" |
| 31 | #include "opto/rootnode.hpp" |
| 32 | #include "opto/vector.hpp" |
| 33 | #include "utilities/macros.hpp" |
| 34 | |
| 35 | static bool is_vector_mask(ciKlass* klass) { |
| 36 | return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); |
| 37 | } |
| 38 | |
| 39 | static bool is_vector_shuffle(ciKlass* klass) { |
| 40 | return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); |
| 41 | } |
| 42 | |
| 43 | |
| 44 | void PhaseVector::optimize_vector_boxes() { |
| 45 | Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]); |
| 46 | |
| 47 | // Signal GraphKit it's post-parse phase. |
| 48 | assert(C->inlining_incrementally() == false, "sanity")do { if (!(C->inlining_incrementally() == false)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 48, "assert(" "C->inlining_incrementally() == false" ") failed" , "sanity"); ::breakpoint(); } } while (0); |
| 49 | C->set_inlining_incrementally(true); |
| 50 | |
| 51 | C->for_igvn()->clear(); |
| 52 | C->initial_gvn()->replace_with(&_igvn); |
| 53 | |
| 54 | expand_vunbox_nodes(); |
| 55 | scalarize_vbox_nodes(); |
| 56 | |
| 57 | C->inline_vector_reboxing_calls(); |
| 58 | |
| 59 | expand_vbox_nodes(); |
| 60 | eliminate_vbox_alloc_nodes(); |
| 61 | |
| 62 | C->set_inlining_incrementally(false); |
| 63 | |
| 64 | do_cleanup(); |
| 65 | } |
| 66 | |
| 67 | void PhaseVector::do_cleanup() { |
| 68 | if (C->failing()) return; |
| 69 | { |
| 70 | Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]); |
| 71 | ResourceMark rm; |
| 72 | PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn()); |
| 73 | if (C->failing()) return; |
| 74 | } |
| 75 | { |
| 76 | Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]); |
| 77 | _igvn = PhaseIterGVN(C->initial_gvn()); |
| 78 | _igvn.optimize(); |
| 79 | if (C->failing()) return; |
| 80 | } |
| 81 | C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3); |
| 82 | } |
| 83 | |
| 84 | void PhaseVector::scalarize_vbox_nodes() { |
| 85 | if (C->failing()) return; |
| 86 | |
| 87 | if (!EnableVectorReboxing) { |
| 88 | return; // don't scalarize vector boxes |
| 89 | } |
| 90 | |
| 91 | int macro_idx = C->macro_count() - 1; |
| 92 | while (macro_idx >= 0) { |
| 93 | Node * n = C->macro_node(macro_idx); |
| 94 | assert(n->is_macro(), "only macro nodes expected here")do { if (!(n->is_macro())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 94, "assert(" "n->is_macro()" ") failed", "only macro nodes expected here" ); ::breakpoint(); } } while (0); |
| 95 | if (n->Opcode() == Op_VectorBox) { |
| 96 | VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); |
| 97 | scalarize_vbox_node(vbox); |
| 98 | if (C->failing()) return; |
| 99 | C->print_method(PHASE_SCALARIZE_VBOX, vbox, 3); |
| 100 | } |
| 101 | if (C->failing()) return; |
| 102 | macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | void PhaseVector::expand_vbox_nodes() { |
| 107 | if (C->failing()) return; |
| 108 | |
| 109 | int macro_idx = C->macro_count() - 1; |
| 110 | while (macro_idx >= 0) { |
| 111 | Node * n = C->macro_node(macro_idx); |
| 112 | assert(n->is_macro(), "only macro nodes expected here")do { if (!(n->is_macro())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 112, "assert(" "n->is_macro()" ") failed", "only macro nodes expected here" ); ::breakpoint(); } } while (0); |
| 113 | if (n->Opcode() == Op_VectorBox) { |
| 114 | VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); |
| 115 | expand_vbox_node(vbox); |
| 116 | if (C->failing()) return; |
| 117 | } |
| 118 | if (C->failing()) return; |
| 119 | macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | void PhaseVector::expand_vunbox_nodes() { |
| 124 | if (C->failing()) return; |
| 125 | |
| 126 | int macro_idx = C->macro_count() - 1; |
| 127 | while (macro_idx >= 0) { |
| 128 | Node * n = C->macro_node(macro_idx); |
| 129 | assert(n->is_macro(), "only macro nodes expected here")do { if (!(n->is_macro())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 129, "assert(" "n->is_macro()" ") failed", "only macro nodes expected here" ); ::breakpoint(); } } while (0); |
| 130 | if (n->Opcode() == Op_VectorUnbox) { |
| 131 | VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n); |
| 132 | expand_vunbox_node(vec_unbox); |
| 133 | if (C->failing()) return; |
| 134 | C->print_method(PHASE_EXPAND_VUNBOX, vec_unbox, 3); |
| 135 | } |
| 136 | if (C->failing()) return; |
| 137 | macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | void PhaseVector::eliminate_vbox_alloc_nodes() { |
| 142 | if (C->failing()) return; |
| 143 | |
| 144 | int macro_idx = C->macro_count() - 1; |
| 145 | while (macro_idx >= 0) { |
| 146 | Node * n = C->macro_node(macro_idx); |
| 147 | assert(n->is_macro(), "only macro nodes expected here")do { if (!(n->is_macro())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 147, "assert(" "n->is_macro()" ") failed", "only macro nodes expected here" ); ::breakpoint(); } } while (0); |
| 148 | if (n->Opcode() == Op_VectorBoxAllocate) { |
| 149 | VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(n); |
| 150 | eliminate_vbox_alloc_node(vbox_alloc); |
| 151 | if (C->failing()) return; |
| 152 | C->print_method(PHASE_ELIMINATE_VBOX_ALLOC, vbox_alloc, 3); |
| 153 | } |
| 154 | if (C->failing()) return; |
| 155 | macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1); |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | static JVMState* clone_jvms(Compile* C, SafePointNode* sfpt) { |
| 160 | JVMState* new_jvms = sfpt->jvms()->clone_shallow(C); |
| 161 | uint size = sfpt->req(); |
| 162 | SafePointNode* map = new SafePointNode(size, new_jvms); |
| 163 | for (uint i = 0; i < size; i++) { |
| 164 | map->init_req(i, sfpt->in(i)); |
| 165 | } |
| 166 | new_jvms->set_map(map); |
| 167 | return new_jvms; |
| 168 | } |
| 169 | |
| 170 | void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) { |
| 171 | Node* vec_value = vec_box->in(VectorBoxNode::Value); |
| 172 | PhaseGVN& gvn = *C->initial_gvn(); |
| 173 | |
| 174 | // Process merged VBAs |
| 175 | |
| 176 | if (EnableVectorAggressiveReboxing) { |
| 177 | Unique_Node_List calls(C->comp_arena()); |
| 178 | for (DUIterator_Fast imax, i = vec_box->fast_outs(imax); i < imax; i++) { |
| 179 | Node* use = vec_box->fast_out(i); |
| 180 | if (use->is_CallJava()) { |
| 181 | CallJavaNode* call = use->as_CallJava(); |
| 182 | if (call->has_non_debug_use(vec_box) && vec_box->in(VectorBoxNode::Box)->is_Phi()) { |
| 183 | calls.push(call); |
| 184 | } |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | while (calls.size() > 0) { |
| 189 | CallJavaNode* call = calls.pop()->as_CallJava(); |
| 190 | // Attach new VBA to the call and use it instead of Phi (VBA ... VBA). |
| 191 | |
| 192 | JVMState* jvms = clone_jvms(C, call); |
| 193 | GraphKit kit(jvms); |
| 194 | PhaseGVN& gvn = kit.gvn(); |
| 195 | |
| 196 | // Adjust JVMS from post-call to pre-call state: put args on stack |
| 197 | uint nargs = call->method()->arg_size(); |
| 198 | kit.ensure_stack(kit.sp() + nargs); |
| 199 | for (uint i = TypeFunc::Parms; i < call->tf()->domain()->cnt(); i++) { |
| 200 | kit.push(call->in(i)); |
| 201 | } |
| 202 | jvms = kit.sync_jvms(); |
Value stored to 'jvms' is never read | |
| 203 | |
| 204 | Node* new_vbox = NULL__null; |
| 205 | { |
| 206 | Node* vect = vec_box->in(VectorBoxNode::Value); |
| 207 | const TypeInstPtr* vbox_type = vec_box->box_type(); |
| 208 | const TypeVect* vt = vec_box->vec_type(); |
| 209 | BasicType elem_bt = vt->element_basic_type(); |
| 210 | int num_elem = vt->length(); |
| 211 | |
| 212 | new_vbox = kit.box_vector(vect, vbox_type, elem_bt, num_elem, /*deoptimize=*/true); |
| 213 | |
| 214 | kit.replace_in_map(vec_box, new_vbox); |
| 215 | } |
| 216 | |
| 217 | kit.dec_sp(nargs); |
| 218 | jvms = kit.sync_jvms(); |
| 219 | |
| 220 | call->set_req(TypeFunc::Control , kit.control()); |
| 221 | call->set_req(TypeFunc::I_O , kit.i_o()); |
| 222 | call->set_req(TypeFunc::Memory , kit.reset_memory()); |
| 223 | call->set_req(TypeFunc::FramePtr, kit.frameptr()); |
| 224 | call->replace_edge(vec_box, new_vbox); |
| 225 | |
| 226 | C->record_for_igvn(call); |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | // Process debug uses at safepoints |
| 231 | Unique_Node_List safepoints(C->comp_arena()); |
| 232 | |
| 233 | Unique_Node_List worklist(C->comp_arena()); |
| 234 | worklist.push(vec_box); |
| 235 | while (worklist.size() > 0) { |
| 236 | Node* n = worklist.pop(); |
| 237 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
| 238 | Node* use = n->fast_out(i); |
| 239 | if (use->is_SafePoint()) { |
| 240 | SafePointNode* sfpt = use->as_SafePoint(); |
| 241 | if (!sfpt->is_Call() || !sfpt->as_Call()->has_non_debug_use(n)) { |
| 242 | safepoints.push(sfpt); |
| 243 | } |
| 244 | } else if (use->is_ConstraintCast()) { |
| 245 | worklist.push(use); // reversed version of Node::uncast() |
| 246 | } |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass(); |
| 251 | int n_fields = iklass->nof_nonstatic_fields(); |
| 252 | assert(n_fields == 1, "sanity")do { if (!(n_fields == 1)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 252, "assert(" "n_fields == 1" ") failed", "sanity"); ::breakpoint (); } } while (0); |
| 253 | |
| 254 | // If a mask is feeding into safepoint[s], then its value should be |
| 255 | // packed into a boolean/byte vector first, this will simplify the |
| 256 | // re-materialization logic for both predicated and non-predicated |
| 257 | // targets. |
| 258 | bool is_mask = is_vector_mask(iklass); |
| 259 | if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) { |
| 260 | const TypeVect* vt = vec_value->bottom_type()->is_vect(); |
| 261 | BasicType bt = vt->element_basic_type(); |
| 262 | vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length())); |
| 263 | } |
| 264 | |
| 265 | while (safepoints.size() > 0) { |
| 266 | SafePointNode* sfpt = safepoints.pop()->as_SafePoint(); |
| 267 | |
| 268 | uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); |
| 269 | Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(), |
| 270 | #ifdef ASSERT1 |
| 271 | vec_box, |
| 272 | #endif // ASSERT |
| 273 | first_ind, n_fields); |
| 274 | sobj->init_req(0, C->root()); |
| 275 | sfpt->add_req(vec_value); |
| 276 | |
| 277 | sobj = gvn.transform(sobj); |
| 278 | |
| 279 | JVMState *jvms = sfpt->jvms(); |
| 280 | |
| 281 | jvms->set_endoff(sfpt->req()); |
| 282 | // Now make a pass over the debug information replacing any references |
| 283 | // to the allocated object with vector value. |
| 284 | for (uint i = jvms->debug_start(); i < jvms->debug_end(); i++) { |
| 285 | Node* debug = sfpt->in(i); |
| 286 | if (debug != NULL__null && debug->uncast(/*keep_deps*/false) == vec_box) { |
| 287 | sfpt->set_req(i, sobj); |
| 288 | } |
| 289 | } |
| 290 | C->record_for_igvn(sfpt); |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) { |
| 295 | if (vec_box->outcnt() > 0) { |
| 296 | Node* vbox = vec_box->in(VectorBoxNode::Box); |
| 297 | Node* vect = vec_box->in(VectorBoxNode::Value); |
| 298 | Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type()); |
| 299 | C->gvn_replace_by(vec_box, result); |
| 300 | C->print_method(PHASE_EXPAND_VBOX, vec_box, 3); |
| 301 | } |
| 302 | C->remove_macro_node(vec_box); |
| 303 | } |
| 304 | |
| 305 | Node* PhaseVector::expand_vbox_node_helper(Node* vbox, |
| 306 | Node* vect, |
| 307 | const TypeInstPtr* box_type, |
| 308 | const TypeVect* vect_type) { |
| 309 | if (vbox->is_Phi() && vect->is_Phi()) { |
| 310 | assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "")do { if (!(vbox->as_Phi()->region() == vect->as_Phi( )->region())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 310, "assert(" "vbox->as_Phi()->region() == vect->as_Phi()->region()" ") failed", ""); ::breakpoint(); } } while (0); |
| 311 | Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type); |
| 312 | for (uint i = 1; i < vbox->req(); i++) { |
| 313 | Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type); |
| 314 | new_phi->set_req(i, new_box); |
| 315 | } |
| 316 | new_phi = C->initial_gvn()->transform(new_phi); |
| 317 | return new_phi; |
| 318 | } else if (vbox->is_Phi() && (vect->is_Vector() || vect->is_LoadVector())) { |
| 319 | // Handle the case when the allocation input to VectorBoxNode is a phi |
| 320 | // but the vector input is not, which can definitely be the case if the |
| 321 | // vector input has been value-numbered. It seems to be safe to do by |
| 322 | // construction because VectorBoxNode and VectorBoxAllocate come in a |
| 323 | // specific order as a result of expanding an intrinsic call. After that, if |
| 324 | // any of the inputs to VectorBoxNode are value-numbered they can only |
| 325 | // move up and are guaranteed to dominate. |
| 326 | Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type); |
| 327 | for (uint i = 1; i < vbox->req(); i++) { |
| 328 | Node* new_box = expand_vbox_node_helper(vbox->in(i), vect, box_type, vect_type); |
| 329 | new_phi->set_req(i, new_box); |
| 330 | } |
| 331 | new_phi = C->initial_gvn()->transform(new_phi); |
| 332 | return new_phi; |
| 333 | } else if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) { |
| 334 | VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0)); |
| 335 | return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type); |
| 336 | } else { |
| 337 | assert(!vbox->is_Phi(), "")do { if (!(!vbox->is_Phi())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 337, "assert(" "!vbox->is_Phi()" ") failed", ""); ::breakpoint (); } } while (0); |
| 338 | // TODO: assert that expanded vbox is initialized with the same value (vect). |
| 339 | return vbox; // already expanded |
| 340 | } |
| 341 | } |
| 342 | |
| 343 | Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc, |
| 344 | Node* value, |
| 345 | const TypeInstPtr* box_type, |
| 346 | const TypeVect* vect_type) { |
| 347 | JVMState* jvms = clone_jvms(C, vbox_alloc); |
| 348 | GraphKit kit(jvms); |
| 349 | PhaseGVN& gvn = kit.gvn(); |
| 350 | |
| 351 | ciInstanceKlass* box_klass = box_type->klass()->as_instance_klass(); |
| 352 | BasicType bt = vect_type->element_basic_type(); |
| 353 | int num_elem = vect_type->length(); |
| 354 | |
| 355 | bool is_mask = is_vector_mask(box_klass); |
| 356 | // If boxed mask value is present in a predicate register, it must be |
| 357 | // spilled to a vector though a VectorStoreMaskOperation before actual StoreVector |
| 358 | // operation to vector payload field. |
| 359 | if (is_mask && (value->bottom_type()->isa_vectmask() || bt != T_BOOLEAN)) { |
| 360 | value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem)); |
| 361 | // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array. |
| 362 | bt = T_BOOLEAN; |
| 363 | assert(value->bottom_type()->is_vect()->element_basic_type() == bt,do { if (!(value->bottom_type()->is_vect()->element_basic_type () == bt)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 364, "assert(" "value->bottom_type()->is_vect()->element_basic_type() == bt" ") failed", "must be consistent with mask representation"); :: breakpoint(); } } while (0) |
| 364 | "must be consistent with mask representation")do { if (!(value->bottom_type()->is_vect()->element_basic_type () == bt)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 364, "assert(" "value->bottom_type()->is_vect()->element_basic_type() == bt" ") failed", "must be consistent with mask representation"); :: breakpoint(); } } while (0); |
| 365 | } |
| 366 | |
| 367 | // Generate array allocation for the field which holds the values. |
| 368 | const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt)); |
| 369 | Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1); |
| 370 | |
| 371 | // Store the vector value into the array. |
| 372 | // (The store should be captured by InitializeNode and turned into initialized store later.) |
| 373 | Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt); |
| 374 | const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr(); |
| 375 | Node* arr_mem = kit.memory(arr_adr); |
| 376 | Node* vstore = gvn.transform(StoreVectorNode::make(0, |
| 377 | kit.control(), |
| 378 | arr_mem, |
| 379 | arr_adr, |
| 380 | arr_adr_type, |
| 381 | value, |
| 382 | num_elem)); |
| 383 | kit.set_memory(vstore, arr_adr_type); |
| 384 | |
| 385 | C->set_max_vector_size(MAX2(C->max_vector_size(), vect_type->length_in_bytes())); |
| 386 | |
| 387 | // Generate the allocate for the Vector object. |
| 388 | const TypeKlassPtr* klass_type = box_type->as_klass_type(); |
| 389 | Node* klass_node = kit.makecon(klass_type); |
| 390 | Node* vec_obj = kit.new_instance(klass_node); |
| 391 | |
| 392 | // Store the allocated array into object. |
| 393 | ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), |
| 394 | ciSymbols::object_signature(), |
| 395 | false); |
| 396 | assert(field != NULL, "")do { if (!(field != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 396, "assert(" "field != __null" ") failed", ""); ::breakpoint (); } } while (0); |
| 397 | Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes()); |
| 398 | const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr(); |
| 399 | |
| 400 | // The store should be captured by InitializeNode and turned into initialized store later. |
| 401 | Node* field_store = gvn.transform(kit.access_store_at(vec_obj, |
| 402 | vec_field, |
| 403 | vec_adr_type, |
| 404 | arr, |
| 405 | TypeOopPtr::make_from_klass(field->type()->as_klass()), |
| 406 | T_OBJECT, |
| 407 | IN_HEAP)); |
| 408 | kit.set_memory(field_store, vec_adr_type); |
| 409 | |
| 410 | kit.replace_call(vbox_alloc, vec_obj, true); |
| 411 | C->remove_macro_node(vbox_alloc); |
| 412 | |
| 413 | return vec_obj; |
| 414 | } |
| 415 | |
| 416 | void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) { |
| 417 | if (vec_unbox->outcnt() > 0) { |
| 418 | GraphKit kit; |
| 419 | PhaseGVN& gvn = kit.gvn(); |
| 420 | |
| 421 | Node* obj = vec_unbox->obj(); |
| 422 | const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr(); |
| 423 | ciInstanceKlass* from_kls = tinst->klass()->as_instance_klass(); |
| 424 | const TypeVect* vt = vec_unbox->bottom_type()->is_vect(); |
| 425 | BasicType bt = vt->element_basic_type(); |
| 426 | BasicType masktype = bt; |
| 427 | |
| 428 | if (is_vector_mask(from_kls)) { |
| 429 | bt = T_BOOLEAN; |
| 430 | } else if (is_vector_shuffle(from_kls)) { |
| 431 | bt = T_BYTE; |
| 432 | } |
| 433 | |
| 434 | ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(), |
| 435 | ciSymbols::object_signature(), |
| 436 | false); |
| 437 | assert(field != NULL, "")do { if (!(field != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 437, "assert(" "field != __null" ") failed", ""); ::breakpoint (); } } while (0); |
| 438 | int offset = field->offset_in_bytes(); |
| 439 | Node* vec_adr = kit.basic_plus_adr(obj, offset); |
| 440 | |
| 441 | Node* mem = vec_unbox->mem(); |
| 442 | Node* ctrl = vec_unbox->in(0); |
| 443 | Node* vec_field_ld; |
| 444 | { |
| 445 | DecoratorSet decorators = MO_UNORDERED | IN_HEAP; |
| 446 | C2AccessValuePtr addr(vec_adr, vec_adr->bottom_type()->is_ptr()); |
| 447 | MergeMemNode* local_mem = MergeMemNode::make(mem); |
| 448 | gvn.record_for_igvn(local_mem); |
| 449 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 450 | C2OptAccess access(gvn, ctrl, local_mem, decorators, T_OBJECT, obj, addr); |
| 451 | const Type* type = TypeOopPtr::make_from_klass(field->type()->as_klass()); |
| 452 | vec_field_ld = bs->load_at(access, type); |
| 453 | } |
| 454 | |
| 455 | // For proper aliasing, attach concrete payload type. |
| 456 | ciKlass* payload_klass = ciTypeArrayKlass::make(bt); |
| 457 | const Type* payload_type = TypeAryPtr::make_from_klass(payload_klass)->cast_to_ptr_type(TypePtr::NotNull); |
| 458 | vec_field_ld = gvn.transform(new CastPPNode(vec_field_ld, payload_type)); |
| 459 | |
| 460 | Node* adr = kit.array_element_address(vec_field_ld, gvn.intcon(0), bt); |
| 461 | const TypePtr* adr_type = adr->bottom_type()->is_ptr(); |
| 462 | int num_elem = vt->length(); |
| 463 | Node* vec_val_load = LoadVectorNode::make(0, |
| 464 | ctrl, |
| 465 | mem, |
| 466 | adr, |
| 467 | adr_type, |
| 468 | num_elem, |
| 469 | bt); |
| 470 | vec_val_load = gvn.transform(vec_val_load); |
| 471 | |
| 472 | C->set_max_vector_size(MAX2(C->max_vector_size(), vt->length_in_bytes())); |
| 473 | |
| 474 | if (is_vector_mask(from_kls)) { |
| 475 | vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem))); |
| 476 | } else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) { |
| 477 | assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency")do { if (!(vec_unbox->bottom_type()->is_vect()->element_basic_type () == masktype)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/vector.cpp" , 477, "assert(" "vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype" ") failed", "expect shuffle type consistency"); ::breakpoint (); } } while (0); |
| 478 | vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem))); |
| 479 | } |
| 480 | |
| 481 | gvn.hash_delete(vec_unbox); |
| 482 | vec_unbox->disconnect_inputs(C); |
| 483 | C->gvn_replace_by(vec_unbox, vec_val_load); |
| 484 | } |
| 485 | C->remove_macro_node(vec_unbox); |
| 486 | } |
| 487 | |
| 488 | void PhaseVector::eliminate_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc) { |
| 489 | JVMState* jvms = clone_jvms(C, vbox_alloc); |
| 490 | GraphKit kit(jvms); |
| 491 | // Remove VBA, but leave a safepoint behind. |
| 492 | // Otherwise, it may end up with a loop without any safepoint polls. |
| 493 | kit.replace_call(vbox_alloc, kit.map(), true); |
| 494 | C->remove_macro_node(vbox_alloc); |
| 495 | } |