| File: | jdk/src/hotspot/share/opto/macro.cpp |
| Warning: | line 1279, column 9 Value stored to 'toobig_false' during its initialization is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* |
| 2 | * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #include "precompiled.hpp" |
| 26 | #include "compiler/compileLog.hpp" |
| 27 | #include "gc/shared/collectedHeap.inline.hpp" |
| 28 | #include "gc/shared/tlab_globals.hpp" |
| 29 | #include "libadt/vectset.hpp" |
| 30 | #include "memory/universe.hpp" |
| 31 | #include "opto/addnode.hpp" |
| 32 | #include "opto/arraycopynode.hpp" |
| 33 | #include "opto/callnode.hpp" |
| 34 | #include "opto/castnode.hpp" |
| 35 | #include "opto/cfgnode.hpp" |
| 36 | #include "opto/compile.hpp" |
| 37 | #include "opto/convertnode.hpp" |
| 38 | #include "opto/graphKit.hpp" |
| 39 | #include "opto/intrinsicnode.hpp" |
| 40 | #include "opto/locknode.hpp" |
| 41 | #include "opto/loopnode.hpp" |
| 42 | #include "opto/macro.hpp" |
| 43 | #include "opto/memnode.hpp" |
| 44 | #include "opto/narrowptrnode.hpp" |
| 45 | #include "opto/node.hpp" |
| 46 | #include "opto/opaquenode.hpp" |
| 47 | #include "opto/phaseX.hpp" |
| 48 | #include "opto/rootnode.hpp" |
| 49 | #include "opto/runtime.hpp" |
| 50 | #include "opto/subnode.hpp" |
| 51 | #include "opto/subtypenode.hpp" |
| 52 | #include "opto/type.hpp" |
| 53 | #include "prims/jvmtiExport.hpp" |
| 54 | #include "runtime/sharedRuntime.hpp" |
| 55 | #include "utilities/macros.hpp" |
| 56 | #include "utilities/powerOfTwo.hpp" |
| 57 | #if INCLUDE_G1GC1 |
| 58 | #include "gc/g1/g1ThreadLocalData.hpp" |
| 59 | #endif // INCLUDE_G1GC |
| 60 | #if INCLUDE_SHENANDOAHGC1 |
| 61 | #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" |
| 62 | #endif |
| 63 | |
| 64 | |
| 65 | // |
| 66 | // Replace any references to "oldref" in inputs to "use" with "newref". |
| 67 | // Returns the number of replacements made. |
| 68 | // |
| 69 | int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) { |
| 70 | int nreplacements = 0; |
| 71 | uint req = use->req(); |
| 72 | for (uint j = 0; j < use->len(); j++) { |
| 73 | Node *uin = use->in(j); |
| 74 | if (uin == oldref) { |
| 75 | if (j < req) |
| 76 | use->set_req(j, newref); |
| 77 | else |
| 78 | use->set_prec(j, newref); |
| 79 | nreplacements++; |
| 80 | } else if (j >= req && uin == NULL__null) { |
| 81 | break; |
| 82 | } |
| 83 | } |
| 84 | return nreplacements; |
| 85 | } |
| 86 | |
| 87 | void PhaseMacroExpand::migrate_outs(Node *old, Node *target) { |
| 88 | assert(old != NULL, "sanity")do { if (!(old != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 88, "assert(" "old != __null" ") failed", "sanity"); ::breakpoint (); } } while (0); |
| 89 | for (DUIterator_Fast imax, i = old->fast_outs(imax); i < imax; i++) { |
| 90 | Node* use = old->fast_out(i); |
| 91 | _igvn.rehash_node_delayed(use); |
| 92 | imax -= replace_input(use, old, target); |
| 93 | // back up iterator |
| 94 | --i; |
| 95 | } |
| 96 | assert(old->outcnt() == 0, "all uses must be deleted")do { if (!(old->outcnt() == 0)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 96, "assert(" "old->outcnt() == 0" ") failed", "all uses must be deleted" ); ::breakpoint(); } } while (0); |
| 97 | } |
| 98 | |
| 99 | Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) { |
| 100 | Node* cmp; |
| 101 | if (mask != 0) { |
| 102 | Node* and_node = transform_later(new AndXNodeAndLNode(word, MakeConXlongcon(mask))); |
| 103 | cmp = transform_later(new CmpXNodeCmpLNode(and_node, MakeConXlongcon(bits))); |
| 104 | } else { |
| 105 | cmp = word; |
| 106 | } |
| 107 | Node* bol = transform_later(new BoolNode(cmp, BoolTest::ne)); |
| 108 | IfNode* iff = new IfNode( ctrl, bol, PROB_MIN(1e-6f), COUNT_UNKNOWN(-1.0f) ); |
| 109 | transform_later(iff); |
| 110 | |
| 111 | // Fast path taken. |
| 112 | Node *fast_taken = transform_later(new IfFalseNode(iff)); |
| 113 | |
| 114 | // Fast path not-taken, i.e. slow path |
| 115 | Node *slow_taken = transform_later(new IfTrueNode(iff)); |
| 116 | |
| 117 | if (return_fast_path) { |
| 118 | region->init_req(edge, slow_taken); // Capture slow-control |
| 119 | return fast_taken; |
| 120 | } else { |
| 121 | region->init_req(edge, fast_taken); // Capture fast-control |
| 122 | return slow_taken; |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | //--------------------copy_predefined_input_for_runtime_call-------------------- |
| 127 | void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) { |
| 128 | // Set fixed predefined input arguments |
| 129 | call->init_req( TypeFunc::Control, ctrl ); |
| 130 | call->init_req( TypeFunc::I_O , oldcall->in( TypeFunc::I_O) ); |
| 131 | call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ????? |
| 132 | call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) ); |
| 133 | call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) ); |
| 134 | } |
| 135 | |
| 136 | //------------------------------make_slow_call--------------------------------- |
| 137 | CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, |
| 138 | address slow_call, const char* leaf_name, Node* slow_path, |
| 139 | Node* parm0, Node* parm1, Node* parm2) { |
| 140 | |
| 141 | // Slow-path call |
| 142 | CallNode *call = leaf_name |
| 143 | ? (CallNode*)new CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM ) |
| 144 | : (CallNode*)new CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), TypeRawPtr::BOTTOM ); |
| 145 | |
| 146 | // Slow path call has no side-effects, uses few values |
| 147 | copy_predefined_input_for_runtime_call(slow_path, oldcall, call ); |
| 148 | if (parm0 != NULL__null) call->init_req(TypeFunc::Parms+0, parm0); |
| 149 | if (parm1 != NULL__null) call->init_req(TypeFunc::Parms+1, parm1); |
| 150 | if (parm2 != NULL__null) call->init_req(TypeFunc::Parms+2, parm2); |
| 151 | call->copy_call_debug_info(&_igvn, oldcall); |
| 152 | call->set_cnt(PROB_UNLIKELY_MAG(4)(1e-4f)); // Same effect as RC_UNCOMMON. |
| 153 | _igvn.replace_node(oldcall, call); |
| 154 | transform_later(call); |
| 155 | |
| 156 | return call; |
| 157 | } |
| 158 | |
| 159 | void PhaseMacroExpand::eliminate_gc_barrier(Node* p2x) { |
| 160 | BarrierSetC2 *bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 161 | bs->eliminate_gc_barrier(this, p2x); |
| 162 | } |
| 163 | |
| 164 | // Search for a memory operation for the specified memory slice. |
| 165 | static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) { |
| 166 | Node *orig_mem = mem; |
| 167 | Node *alloc_mem = alloc->in(TypeFunc::Memory); |
| 168 | const TypeOopPtr *tinst = phase->C->get_adr_type(alias_idx)->isa_oopptr(); |
| 169 | while (true) { |
| 170 | if (mem == alloc_mem || mem == start_mem ) { |
| 171 | return mem; // hit one of our sentinels |
| 172 | } else if (mem->is_MergeMem()) { |
| 173 | mem = mem->as_MergeMem()->memory_at(alias_idx); |
| 174 | } else if (mem->is_Proj() && mem->as_Proj()->_con == TypeFunc::Memory) { |
| 175 | Node *in = mem->in(0); |
| 176 | // we can safely skip over safepoints, calls, locks and membars because we |
| 177 | // already know that the object is safe to eliminate. |
| 178 | if (in->is_Initialize() && in->as_Initialize()->allocation() == alloc) { |
| 179 | return in; |
| 180 | } else if (in->is_Call()) { |
| 181 | CallNode *call = in->as_Call(); |
| 182 | if (call->may_modify(tinst, phase)) { |
| 183 | assert(call->is_ArrayCopy(), "ArrayCopy is the only call node that doesn't make allocation escape")do { if (!(call->is_ArrayCopy())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 183, "assert(" "call->is_ArrayCopy()" ") failed", "ArrayCopy is the only call node that doesn't make allocation escape" ); ::breakpoint(); } } while (0); |
| 184 | if (call->as_ArrayCopy()->modifies(offset, offset, phase, false)) { |
| 185 | return in; |
| 186 | } |
| 187 | } |
| 188 | mem = in->in(TypeFunc::Memory); |
| 189 | } else if (in->is_MemBar()) { |
| 190 | ArrayCopyNode* ac = NULL__null; |
| 191 | if (ArrayCopyNode::may_modify(tinst, in->as_MemBar(), phase, ac)) { |
| 192 | if (ac != NULL__null) { |
| 193 | assert(ac->is_clonebasic(), "Only basic clone is a non escaping clone")do { if (!(ac->is_clonebasic())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 193, "assert(" "ac->is_clonebasic()" ") failed", "Only basic clone is a non escaping clone" ); ::breakpoint(); } } while (0); |
| 194 | return ac; |
| 195 | } |
| 196 | } |
| 197 | mem = in->in(TypeFunc::Memory); |
| 198 | } else { |
| 199 | #ifdef ASSERT1 |
| 200 | in->dump(); |
| 201 | mem->dump(); |
| 202 | assert(false, "unexpected projection")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 202, "assert(" "false" ") failed", "unexpected projection") ; ::breakpoint(); } } while (0); |
| 203 | #endif |
| 204 | } |
| 205 | } else if (mem->is_Store()) { |
| 206 | const TypePtr* atype = mem->as_Store()->adr_type(); |
| 207 | int adr_idx = phase->C->get_alias_index(atype); |
| 208 | if (adr_idx == alias_idx) { |
| 209 | assert(atype->isa_oopptr(), "address type must be oopptr")do { if (!(atype->isa_oopptr())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 209, "assert(" "atype->isa_oopptr()" ") failed", "address type must be oopptr" ); ::breakpoint(); } } while (0); |
| 210 | int adr_offset = atype->offset(); |
| 211 | uint adr_iid = atype->is_oopptr()->instance_id(); |
| 212 | // Array elements references have the same alias_idx |
| 213 | // but different offset and different instance_id. |
| 214 | if (adr_offset == offset && adr_iid == alloc->_idx) { |
| 215 | return mem; |
| 216 | } |
| 217 | } else { |
| 218 | assert(adr_idx == Compile::AliasIdxRaw, "address must match or be raw")do { if (!(adr_idx == Compile::AliasIdxRaw)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 218, "assert(" "adr_idx == Compile::AliasIdxRaw" ") failed" , "address must match or be raw"); ::breakpoint(); } } while ( 0); |
| 219 | } |
| 220 | mem = mem->in(MemNode::Memory); |
| 221 | } else if (mem->is_ClearArray()) { |
| 222 | if (!ClearArrayNode::step_through(&mem, alloc->_idx, phase)) { |
| 223 | // Can not bypass initialization of the instance |
| 224 | // we are looking. |
| 225 | debug_only(intptr_t offset;)intptr_t offset; |
| 226 | assert(alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset), "sanity")do { if (!(alloc == AllocateNode::Ideal_allocation(mem->in (3), phase, offset))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 226, "assert(" "alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset)" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 227 | InitializeNode* init = alloc->as_Allocate()->initialization(); |
| 228 | // We are looking for stored value, return Initialize node |
| 229 | // or memory edge from Allocate node. |
| 230 | if (init != NULL__null) { |
| 231 | return init; |
| 232 | } else { |
| 233 | return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers). |
| 234 | } |
| 235 | } |
| 236 | // Otherwise skip it (the call updated 'mem' value). |
| 237 | } else if (mem->Opcode() == Op_SCMemProj) { |
| 238 | mem = mem->in(0); |
| 239 | Node* adr = NULL__null; |
| 240 | if (mem->is_LoadStore()) { |
| 241 | adr = mem->in(MemNode::Address); |
| 242 | } else { |
| 243 | assert(mem->Opcode() == Op_EncodeISOArray ||do { if (!(mem->Opcode() == Op_EncodeISOArray || mem->Opcode () == Op_StrCompressedCopy)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 244, "assert(" "mem->Opcode() == Op_EncodeISOArray || mem->Opcode() == Op_StrCompressedCopy" ") failed", "sanity"); ::breakpoint(); } } while (0) |
| 244 | mem->Opcode() == Op_StrCompressedCopy, "sanity")do { if (!(mem->Opcode() == Op_EncodeISOArray || mem->Opcode () == Op_StrCompressedCopy)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 244, "assert(" "mem->Opcode() == Op_EncodeISOArray || mem->Opcode() == Op_StrCompressedCopy" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 245 | adr = mem->in(3); // Destination array |
| 246 | } |
| 247 | const TypePtr* atype = adr->bottom_type()->is_ptr(); |
| 248 | int adr_idx = phase->C->get_alias_index(atype); |
| 249 | if (adr_idx == alias_idx) { |
| 250 | DEBUG_ONLY(mem->dump();)mem->dump(); |
| 251 | assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 251, "assert(" "false" ") failed", "Object is not scalar replaceable if a LoadStore node accesses its field" ); ::breakpoint(); } } while (0); |
| 252 | return NULL__null; |
| 253 | } |
| 254 | mem = mem->in(MemNode::Memory); |
| 255 | } else if (mem->Opcode() == Op_StrInflatedCopy) { |
| 256 | Node* adr = mem->in(3); // Destination array |
| 257 | const TypePtr* atype = adr->bottom_type()->is_ptr(); |
| 258 | int adr_idx = phase->C->get_alias_index(atype); |
| 259 | if (adr_idx == alias_idx) { |
| 260 | DEBUG_ONLY(mem->dump();)mem->dump(); |
| 261 | assert(false, "Object is not scalar replaceable if a StrInflatedCopy node accesses its field")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 261, "assert(" "false" ") failed", "Object is not scalar replaceable if a StrInflatedCopy node accesses its field" ); ::breakpoint(); } } while (0); |
| 262 | return NULL__null; |
| 263 | } |
| 264 | mem = mem->in(MemNode::Memory); |
| 265 | } else { |
| 266 | return mem; |
| 267 | } |
| 268 | assert(mem != orig_mem, "dead memory loop")do { if (!(mem != orig_mem)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 268, "assert(" "mem != orig_mem" ") failed", "dead memory loop" ); ::breakpoint(); } } while (0); |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | // Generate loads from source of the arraycopy for fields of |
| 273 | // destination needed at a deoptimization point |
| 274 | Node* PhaseMacroExpand::make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, Node* ctl, Node* mem, BasicType ft, const Type *ftype, AllocateNode *alloc) { |
| 275 | BasicType bt = ft; |
| 276 | const Type *type = ftype; |
| 277 | if (ft == T_NARROWOOP) { |
| 278 | bt = T_OBJECT; |
| 279 | type = ftype->make_oopptr(); |
| 280 | } |
| 281 | Node* res = NULL__null; |
| 282 | if (ac->is_clonebasic()) { |
| 283 | assert(ac->in(ArrayCopyNode::Src) != ac->in(ArrayCopyNode::Dest), "clone source equals destination")do { if (!(ac->in(ArrayCopyNode::Src) != ac->in(ArrayCopyNode ::Dest))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 283, "assert(" "ac->in(ArrayCopyNode::Src) != ac->in(ArrayCopyNode::Dest)" ") failed", "clone source equals destination"); ::breakpoint (); } } while (0); |
| 284 | Node* base = ac->in(ArrayCopyNode::Src); |
| 285 | Node* adr = _igvn.transform(new AddPNode(base, base, MakeConXlongcon(offset))); |
| 286 | const TypePtr* adr_type = _igvn.type(base)->is_ptr()->add_offset(offset); |
| 287 | MergeMemNode* mergemen = _igvn.transform(MergeMemNode::make(mem))->as_MergeMem(); |
| 288 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 289 | res = ArrayCopyNode::load(bs, &_igvn, ctl, mergemen, adr, adr_type, type, bt); |
| 290 | } else { |
| 291 | if (ac->modifies(offset, offset, &_igvn, true)) { |
| 292 | assert(ac->in(ArrayCopyNode::Dest) == alloc->result_cast(), "arraycopy destination should be allocation's result")do { if (!(ac->in(ArrayCopyNode::Dest) == alloc->result_cast ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 292, "assert(" "ac->in(ArrayCopyNode::Dest) == alloc->result_cast()" ") failed", "arraycopy destination should be allocation's result" ); ::breakpoint(); } } while (0); |
| 293 | uint shift = exact_log2(type2aelembytes(bt)); |
| 294 | Node* src_pos = ac->in(ArrayCopyNode::SrcPos); |
| 295 | Node* dest_pos = ac->in(ArrayCopyNode::DestPos); |
| 296 | const TypeInt* src_pos_t = _igvn.type(src_pos)->is_int(); |
| 297 | const TypeInt* dest_pos_t = _igvn.type(dest_pos)->is_int(); |
| 298 | |
| 299 | Node* adr = NULL__null; |
| 300 | const TypePtr* adr_type = NULL__null; |
| 301 | if (src_pos_t->is_con() && dest_pos_t->is_con()) { |
| 302 | intptr_t off = ((src_pos_t->get_con() - dest_pos_t->get_con()) << shift) + offset; |
| 303 | Node* base = ac->in(ArrayCopyNode::Src); |
| 304 | adr = _igvn.transform(new AddPNode(base, base, MakeConXlongcon(off))); |
| 305 | adr_type = _igvn.type(base)->is_ptr()->add_offset(off); |
| 306 | if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) { |
| 307 | // Don't emit a new load from src if src == dst but try to get the value from memory instead |
| 308 | return value_from_mem(ac->in(TypeFunc::Memory), ctl, ft, ftype, adr_type->isa_oopptr(), alloc); |
| 309 | } |
| 310 | } else { |
| 311 | Node* diff = _igvn.transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos))); |
| 312 | #ifdef _LP641 |
| 313 | diff = _igvn.transform(new ConvI2LNode(diff)); |
| 314 | #endif |
| 315 | diff = _igvn.transform(new LShiftXNodeLShiftLNode(diff, intcon(shift))); |
| 316 | |
| 317 | Node* off = _igvn.transform(new AddXNodeAddLNode(MakeConXlongcon(offset), diff)); |
| 318 | Node* base = ac->in(ArrayCopyNode::Src); |
| 319 | adr = _igvn.transform(new AddPNode(base, base, off)); |
| 320 | adr_type = _igvn.type(base)->is_ptr()->add_offset(Type::OffsetBot); |
| 321 | if (ac->in(ArrayCopyNode::Src) == ac->in(ArrayCopyNode::Dest)) { |
| 322 | // Non constant offset in the array: we can't statically |
| 323 | // determine the value |
| 324 | return NULL__null; |
| 325 | } |
| 326 | } |
| 327 | MergeMemNode* mergemen = _igvn.transform(MergeMemNode::make(mem))->as_MergeMem(); |
| 328 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 329 | res = ArrayCopyNode::load(bs, &_igvn, ctl, mergemen, adr, adr_type, type, bt); |
| 330 | } |
| 331 | } |
| 332 | if (res != NULL__null) { |
| 333 | if (ftype->isa_narrowoop()) { |
| 334 | // PhaseMacroExpand::scalar_replacement adds DecodeN nodes |
| 335 | res = _igvn.transform(new EncodePNode(res, ftype)); |
| 336 | } |
| 337 | return res; |
| 338 | } |
| 339 | return NULL__null; |
| 340 | } |
| 341 | |
| 342 | // |
| 343 | // Given a Memory Phi, compute a value Phi containing the values from stores |
| 344 | // on the input paths. |
| 345 | // Note: this function is recursive, its depth is limited by the "level" argument |
| 346 | // Returns the computed Phi, or NULL if it cannot compute it. |
| 347 | Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level) { |
| 348 | assert(mem->is_Phi(), "sanity")do { if (!(mem->is_Phi())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 348, "assert(" "mem->is_Phi()" ") failed", "sanity"); :: breakpoint(); } } while (0); |
| 349 | int alias_idx = C->get_alias_index(adr_t); |
| 350 | int offset = adr_t->offset(); |
| 351 | int instance_id = adr_t->instance_id(); |
| 352 | |
| 353 | // Check if an appropriate value phi already exists. |
| 354 | Node* region = mem->in(0); |
| 355 | for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) { |
| 356 | Node* phi = region->fast_out(k); |
| 357 | if (phi->is_Phi() && phi != mem && |
| 358 | phi->as_Phi()->is_same_inst_field(phi_type, (int)mem->_idx, instance_id, alias_idx, offset)) { |
| 359 | return phi; |
| 360 | } |
| 361 | } |
| 362 | // Check if an appropriate new value phi already exists. |
| 363 | Node* new_phi = value_phis->find(mem->_idx); |
| 364 | if (new_phi != NULL__null) |
| 365 | return new_phi; |
| 366 | |
| 367 | if (level <= 0) { |
| 368 | return NULL__null; // Give up: phi tree too deep |
| 369 | } |
| 370 | Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory); |
| 371 | Node *alloc_mem = alloc->in(TypeFunc::Memory); |
| 372 | |
| 373 | uint length = mem->req(); |
| 374 | GrowableArray <Node *> values(length, length, NULL__null); |
| 375 | |
| 376 | // create a new Phi for the value |
| 377 | PhiNode *phi = new PhiNode(mem->in(0), phi_type, NULL__null, mem->_idx, instance_id, alias_idx, offset); |
| 378 | transform_later(phi); |
| 379 | value_phis->push(phi, mem->_idx); |
| 380 | |
| 381 | for (uint j = 1; j < length; j++) { |
| 382 | Node *in = mem->in(j); |
| 383 | if (in == NULL__null || in->is_top()) { |
| 384 | values.at_put(j, in); |
| 385 | } else { |
| 386 | Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn); |
| 387 | if (val == start_mem || val == alloc_mem) { |
| 388 | // hit a sentinel, return appropriate 0 value |
| 389 | values.at_put(j, _igvn.zerocon(ft)); |
| 390 | continue; |
| 391 | } |
| 392 | if (val->is_Initialize()) { |
| 393 | val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); |
| 394 | } |
| 395 | if (val == NULL__null) { |
| 396 | return NULL__null; // can't find a value on this path |
| 397 | } |
| 398 | if (val == mem) { |
| 399 | values.at_put(j, mem); |
| 400 | } else if (val->is_Store()) { |
| 401 | Node* n = val->in(MemNode::ValueIn); |
| 402 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 403 | n = bs->step_over_gc_barrier(n); |
| 404 | if (is_subword_type(ft)) { |
| 405 | n = Compile::narrow_value(ft, n, phi_type, &_igvn, true); |
| 406 | } |
| 407 | values.at_put(j, n); |
| 408 | } else if(val->is_Proj() && val->in(0) == alloc) { |
| 409 | values.at_put(j, _igvn.zerocon(ft)); |
| 410 | } else if (val->is_Phi()) { |
| 411 | val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1); |
| 412 | if (val == NULL__null) { |
| 413 | return NULL__null; |
| 414 | } |
| 415 | values.at_put(j, val); |
| 416 | } else if (val->Opcode() == Op_SCMemProj) { |
| 417 | assert(val->in(0)->is_LoadStore() ||do { if (!(val->in(0)->is_LoadStore() || val->in(0)-> Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 419, "assert(" "val->in(0)->is_LoadStore() || val->in(0)->Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy" ") failed", "sanity"); ::breakpoint(); } } while (0) |
| 418 | val->in(0)->Opcode() == Op_EncodeISOArray ||do { if (!(val->in(0)->is_LoadStore() || val->in(0)-> Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 419, "assert(" "val->in(0)->is_LoadStore() || val->in(0)->Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy" ") failed", "sanity"); ::breakpoint(); } } while (0) |
| 419 | val->in(0)->Opcode() == Op_StrCompressedCopy, "sanity")do { if (!(val->in(0)->is_LoadStore() || val->in(0)-> Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 419, "assert(" "val->in(0)->is_LoadStore() || val->in(0)->Opcode() == Op_EncodeISOArray || val->in(0)->Opcode() == Op_StrCompressedCopy" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 420 | assert(false, "Object is not scalar replaceable if a LoadStore node accesses its field")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 420, "assert(" "false" ") failed", "Object is not scalar replaceable if a LoadStore node accesses its field" ); ::breakpoint(); } } while (0); |
| 421 | return NULL__null; |
| 422 | } else if (val->is_ArrayCopy()) { |
| 423 | Node* res = make_arraycopy_load(val->as_ArrayCopy(), offset, val->in(0), val->in(TypeFunc::Memory), ft, phi_type, alloc); |
| 424 | if (res == NULL__null) { |
| 425 | return NULL__null; |
| 426 | } |
| 427 | values.at_put(j, res); |
| 428 | } else { |
| 429 | DEBUG_ONLY( val->dump(); )val->dump(); |
| 430 | assert(false, "unknown node on this path")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 430, "assert(" "false" ") failed", "unknown node on this path" ); ::breakpoint(); } } while (0); |
| 431 | return NULL__null; // unknown node on this path |
| 432 | } |
| 433 | } |
| 434 | } |
| 435 | // Set Phi's inputs |
| 436 | for (uint j = 1; j < length; j++) { |
| 437 | if (values.at(j) == mem) { |
| 438 | phi->init_req(j, phi); |
| 439 | } else { |
| 440 | phi->init_req(j, values.at(j)); |
| 441 | } |
| 442 | } |
| 443 | return phi; |
| 444 | } |
| 445 | |
| 446 | // Search the last value stored into the object's field. |
| 447 | Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc) { |
| 448 | assert(adr_t->is_known_instance_field(), "instance required")do { if (!(adr_t->is_known_instance_field())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 448, "assert(" "adr_t->is_known_instance_field()" ") failed" , "instance required"); ::breakpoint(); } } while (0); |
| 449 | int instance_id = adr_t->instance_id(); |
| 450 | assert((uint)instance_id == alloc->_idx, "wrong allocation")do { if (!((uint)instance_id == alloc->_idx)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 450, "assert(" "(uint)instance_id == alloc->_idx" ") failed" , "wrong allocation"); ::breakpoint(); } } while (0); |
| 451 | |
| 452 | int alias_idx = C->get_alias_index(adr_t); |
| 453 | int offset = adr_t->offset(); |
| 454 | Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory); |
| 455 | Node *alloc_ctrl = alloc->in(TypeFunc::Control); |
| 456 | Node *alloc_mem = alloc->in(TypeFunc::Memory); |
| 457 | VectorSet visited; |
| 458 | |
| 459 | bool done = sfpt_mem == alloc_mem; |
| 460 | Node *mem = sfpt_mem; |
| 461 | while (!done) { |
| 462 | if (visited.test_set(mem->_idx)) { |
| 463 | return NULL__null; // found a loop, give up |
| 464 | } |
| 465 | mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn); |
| 466 | if (mem == start_mem || mem == alloc_mem) { |
| 467 | done = true; // hit a sentinel, return appropriate 0 value |
| 468 | } else if (mem->is_Initialize()) { |
| 469 | mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); |
| 470 | if (mem == NULL__null) { |
| 471 | done = true; // Something go wrong. |
| 472 | } else if (mem->is_Store()) { |
| 473 | const TypePtr* atype = mem->as_Store()->adr_type(); |
| 474 | assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice")do { if (!(C->get_alias_index(atype) == Compile::AliasIdxRaw )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 474, "assert(" "C->get_alias_index(atype) == Compile::AliasIdxRaw" ") failed", "store is correct memory slice"); ::breakpoint() ; } } while (0); |
| 475 | done = true; |
| 476 | } |
| 477 | } else if (mem->is_Store()) { |
| 478 | const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr(); |
| 479 | assert(atype != NULL, "address type must be oopptr")do { if (!(atype != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 479, "assert(" "atype != __null" ") failed", "address type must be oopptr" ); ::breakpoint(); } } while (0); |
| 480 | assert(C->get_alias_index(atype) == alias_idx &&do { if (!(C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset () == offset && atype->instance_id() == instance_id )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 482, "assert(" "C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset() == offset && atype->instance_id() == instance_id" ") failed", "store is correct memory slice"); ::breakpoint() ; } } while (0) |
| 481 | atype->is_known_instance_field() && atype->offset() == offset &&do { if (!(C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset () == offset && atype->instance_id() == instance_id )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 482, "assert(" "C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset() == offset && atype->instance_id() == instance_id" ") failed", "store is correct memory slice"); ::breakpoint() ; } } while (0) |
| 482 | atype->instance_id() == instance_id, "store is correct memory slice")do { if (!(C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset () == offset && atype->instance_id() == instance_id )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 482, "assert(" "C->get_alias_index(atype) == alias_idx && atype->is_known_instance_field() && atype->offset() == offset && atype->instance_id() == instance_id" ") failed", "store is correct memory slice"); ::breakpoint() ; } } while (0); |
| 483 | done = true; |
| 484 | } else if (mem->is_Phi()) { |
| 485 | // try to find a phi's unique input |
| 486 | Node *unique_input = NULL__null; |
| 487 | Node *top = C->top(); |
| 488 | for (uint i = 1; i < mem->req(); i++) { |
| 489 | Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn); |
| 490 | if (n == NULL__null || n == top || n == mem) { |
| 491 | continue; |
| 492 | } else if (unique_input == NULL__null) { |
| 493 | unique_input = n; |
| 494 | } else if (unique_input != n) { |
| 495 | unique_input = top; |
| 496 | break; |
| 497 | } |
| 498 | } |
| 499 | if (unique_input != NULL__null && unique_input != top) { |
| 500 | mem = unique_input; |
| 501 | } else { |
| 502 | done = true; |
| 503 | } |
| 504 | } else if (mem->is_ArrayCopy()) { |
| 505 | done = true; |
| 506 | } else { |
| 507 | DEBUG_ONLY( mem->dump(); )mem->dump(); |
| 508 | assert(false, "unexpected node")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 508, "assert(" "false" ") failed", "unexpected node"); ::breakpoint (); } } while (0); |
| 509 | } |
| 510 | } |
| 511 | if (mem != NULL__null) { |
| 512 | if (mem == start_mem || mem == alloc_mem) { |
| 513 | // hit a sentinel, return appropriate 0 value |
| 514 | return _igvn.zerocon(ft); |
| 515 | } else if (mem->is_Store()) { |
| 516 | Node* n = mem->in(MemNode::ValueIn); |
| 517 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 518 | n = bs->step_over_gc_barrier(n); |
| 519 | return n; |
| 520 | } else if (mem->is_Phi()) { |
| 521 | // attempt to produce a Phi reflecting the values on the input paths of the Phi |
| 522 | Node_Stack value_phis(8); |
| 523 | Node* phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit); |
| 524 | if (phi != NULL__null) { |
| 525 | return phi; |
| 526 | } else { |
| 527 | // Kill all new Phis |
| 528 | while(value_phis.is_nonempty()) { |
| 529 | Node* n = value_phis.node(); |
| 530 | _igvn.replace_node(n, C->top()); |
| 531 | value_phis.pop(); |
| 532 | } |
| 533 | } |
| 534 | } else if (mem->is_ArrayCopy()) { |
| 535 | Node* ctl = mem->in(0); |
| 536 | Node* m = mem->in(TypeFunc::Memory); |
| 537 | if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) { |
| 538 | // pin the loads in the uncommon trap path |
| 539 | ctl = sfpt_ctl; |
| 540 | m = sfpt_mem; |
| 541 | } |
| 542 | return make_arraycopy_load(mem->as_ArrayCopy(), offset, ctl, m, ft, ftype, alloc); |
| 543 | } |
| 544 | } |
| 545 | // Something go wrong. |
| 546 | return NULL__null; |
| 547 | } |
| 548 | |
| 549 | // Check the possibility of scalar replacement. |
| 550 | bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) { |
| 551 | // Scan the uses of the allocation to check for anything that would |
| 552 | // prevent us from eliminating it. |
| 553 | NOT_PRODUCT( const char* fail_eliminate = NULL; )const char* fail_eliminate = __null; |
| 554 | DEBUG_ONLY( Node* disq_node = NULL; )Node* disq_node = __null; |
| 555 | bool can_eliminate = true; |
| 556 | |
| 557 | Node* res = alloc->result_cast(); |
| 558 | const TypeOopPtr* res_type = NULL__null; |
| 559 | if (res == NULL__null) { |
| 560 | // All users were eliminated. |
| 561 | } else if (!res->is_CheckCastPP()) { |
| 562 | NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)fail_eliminate = "Allocation does not have unique CheckCastPP" ; |
| 563 | can_eliminate = false; |
| 564 | } else { |
| 565 | res_type = _igvn.type(res)->isa_oopptr(); |
| 566 | if (res_type == NULL__null) { |
| 567 | NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";)fail_eliminate = "Neither instance or array allocation"; |
| 568 | can_eliminate = false; |
| 569 | } else if (res_type->isa_aryptr()) { |
| 570 | int length = alloc->in(AllocateNode::ALength)->find_int_con(-1); |
| 571 | if (length < 0) { |
| 572 | NOT_PRODUCT(fail_eliminate = "Array's size is not constant";)fail_eliminate = "Array's size is not constant"; |
| 573 | can_eliminate = false; |
| 574 | } |
| 575 | } |
| 576 | } |
| 577 | |
| 578 | if (can_eliminate && res != NULL__null) { |
| 579 | for (DUIterator_Fast jmax, j = res->fast_outs(jmax); |
| 580 | j < jmax && can_eliminate; j++) { |
| 581 | Node* use = res->fast_out(j); |
| 582 | |
| 583 | if (use->is_AddP()) { |
| 584 | const TypePtr* addp_type = _igvn.type(use)->is_ptr(); |
| 585 | int offset = addp_type->offset(); |
| 586 | |
| 587 | if (offset == Type::OffsetTop || offset == Type::OffsetBot) { |
| 588 | NOT_PRODUCT(fail_eliminate = "Undefined field referrence";)fail_eliminate = "Undefined field referrence"; |
| 589 | can_eliminate = false; |
| 590 | break; |
| 591 | } |
| 592 | for (DUIterator_Fast kmax, k = use->fast_outs(kmax); |
| 593 | k < kmax && can_eliminate; k++) { |
| 594 | Node* n = use->fast_out(k); |
| 595 | if (!n->is_Store() && n->Opcode() != Op_CastP2X |
| 596 | SHENANDOAHGC_ONLY(&& (!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(n)))&& (!UseShenandoahGC || !ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call (n)) ) { |
| 597 | DEBUG_ONLY(disq_node = n;)disq_node = n; |
| 598 | if (n->is_Load() || n->is_LoadStore()) { |
| 599 | NOT_PRODUCT(fail_eliminate = "Field load";)fail_eliminate = "Field load"; |
| 600 | } else { |
| 601 | NOT_PRODUCT(fail_eliminate = "Not store field referrence";)fail_eliminate = "Not store field referrence"; |
| 602 | } |
| 603 | can_eliminate = false; |
| 604 | } |
| 605 | } |
| 606 | } else if (use->is_ArrayCopy() && |
| 607 | (use->as_ArrayCopy()->is_clonebasic() || |
| 608 | use->as_ArrayCopy()->is_arraycopy_validated() || |
| 609 | use->as_ArrayCopy()->is_copyof_validated() || |
| 610 | use->as_ArrayCopy()->is_copyofrange_validated()) && |
| 611 | use->in(ArrayCopyNode::Dest) == res) { |
| 612 | // ok to eliminate |
| 613 | } else if (use->is_SafePoint()) { |
| 614 | SafePointNode* sfpt = use->as_SafePoint(); |
| 615 | if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) { |
| 616 | // Object is passed as argument. |
| 617 | DEBUG_ONLY(disq_node = use;)disq_node = use; |
| 618 | NOT_PRODUCT(fail_eliminate = "Object is passed as argument";)fail_eliminate = "Object is passed as argument"; |
| 619 | can_eliminate = false; |
| 620 | } |
| 621 | Node* sfptMem = sfpt->memory(); |
| 622 | if (sfptMem == NULL__null || sfptMem->is_top()) { |
| 623 | DEBUG_ONLY(disq_node = use;)disq_node = use; |
| 624 | NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";)fail_eliminate = "NULL or TOP memory"; |
| 625 | can_eliminate = false; |
| 626 | } else { |
| 627 | safepoints.append_if_missing(sfpt); |
| 628 | } |
| 629 | } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark |
| 630 | if (use->is_Phi()) { |
| 631 | if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) { |
| 632 | NOT_PRODUCT(fail_eliminate = "Object is return value";)fail_eliminate = "Object is return value"; |
| 633 | } else { |
| 634 | NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";)fail_eliminate = "Object is referenced by Phi"; |
| 635 | } |
| 636 | DEBUG_ONLY(disq_node = use;)disq_node = use; |
| 637 | } else { |
| 638 | if (use->Opcode() == Op_Return) { |
| 639 | NOT_PRODUCT(fail_eliminate = "Object is return value";)fail_eliminate = "Object is return value"; |
| 640 | }else { |
| 641 | NOT_PRODUCT(fail_eliminate = "Object is referenced by node";)fail_eliminate = "Object is referenced by node"; |
| 642 | } |
| 643 | DEBUG_ONLY(disq_node = use;)disq_node = use; |
| 644 | } |
| 645 | can_eliminate = false; |
| 646 | } |
| 647 | } |
| 648 | } |
| 649 | |
| 650 | #ifndef PRODUCT |
| 651 | if (PrintEliminateAllocations) { |
| 652 | if (can_eliminate) { |
| 653 | tty->print("Scalar "); |
| 654 | if (res == NULL__null) |
| 655 | alloc->dump(); |
| 656 | else |
| 657 | res->dump(); |
| 658 | } else if (alloc->_is_scalar_replaceable) { |
| 659 | tty->print("NotScalar (%s)", fail_eliminate); |
| 660 | if (res == NULL__null) |
| 661 | alloc->dump(); |
| 662 | else |
| 663 | res->dump(); |
| 664 | #ifdef ASSERT1 |
| 665 | if (disq_node != NULL__null) { |
| 666 | tty->print(" >>>> "); |
| 667 | disq_node->dump(); |
| 668 | } |
| 669 | #endif /*ASSERT*/ |
| 670 | } |
| 671 | } |
| 672 | #endif |
| 673 | return can_eliminate; |
| 674 | } |
| 675 | |
| 676 | // Do scalar replacement. |
| 677 | bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) { |
| 678 | GrowableArray <SafePointNode *> safepoints_done; |
| 679 | |
| 680 | ciKlass* klass = NULL__null; |
| 681 | ciInstanceKlass* iklass = NULL__null; |
| 682 | int nfields = 0; |
| 683 | int array_base = 0; |
| 684 | int element_size = 0; |
| 685 | BasicType basic_elem_type = T_ILLEGAL; |
| 686 | ciType* elem_type = NULL__null; |
| 687 | |
| 688 | Node* res = alloc->result_cast(); |
| 689 | assert(res == NULL || res->is_CheckCastPP(), "unexpected AllocateNode result")do { if (!(res == __null || res->is_CheckCastPP())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 689, "assert(" "res == __null || res->is_CheckCastPP()" ") failed" , "unexpected AllocateNode result"); ::breakpoint(); } } while (0); |
| 690 | const TypeOopPtr* res_type = NULL__null; |
| 691 | if (res != NULL__null) { // Could be NULL when there are no users |
| 692 | res_type = _igvn.type(res)->isa_oopptr(); |
| 693 | } |
| 694 | |
| 695 | if (res != NULL__null) { |
| 696 | klass = res_type->klass(); |
| 697 | if (res_type->isa_instptr()) { |
| 698 | // find the fields of the class which will be needed for safepoint debug information |
| 699 | assert(klass->is_instance_klass(), "must be an instance klass.")do { if (!(klass->is_instance_klass())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 699, "assert(" "klass->is_instance_klass()" ") failed", "must be an instance klass." ); ::breakpoint(); } } while (0); |
| 700 | iklass = klass->as_instance_klass(); |
| 701 | nfields = iklass->nof_nonstatic_fields(); |
| 702 | } else { |
| 703 | // find the array's elements which will be needed for safepoint debug information |
| 704 | nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1); |
| 705 | assert(klass->is_array_klass() && nfields >= 0, "must be an array klass.")do { if (!(klass->is_array_klass() && nfields >= 0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 705, "assert(" "klass->is_array_klass() && nfields >= 0" ") failed", "must be an array klass."); ::breakpoint(); } } while (0); |
| 706 | elem_type = klass->as_array_klass()->element_type(); |
| 707 | basic_elem_type = elem_type->basic_type(); |
| 708 | array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type); |
| 709 | element_size = type2aelembytes(basic_elem_type); |
| 710 | } |
| 711 | } |
| 712 | // |
| 713 | // Process the safepoint uses |
| 714 | // |
| 715 | while (safepoints.length() > 0) { |
| 716 | SafePointNode* sfpt = safepoints.pop(); |
| 717 | Node* mem = sfpt->memory(); |
| 718 | Node* ctl = sfpt->control(); |
| 719 | assert(sfpt->jvms() != NULL, "missed JVMS")do { if (!(sfpt->jvms() != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 719, "assert(" "sfpt->jvms() != __null" ") failed", "missed JVMS" ); ::breakpoint(); } } while (0); |
| 720 | // Fields of scalar objs are referenced only at the end |
| 721 | // of regular debuginfo at the last (youngest) JVMS. |
| 722 | // Record relative start index. |
| 723 | uint first_ind = (sfpt->req() - sfpt->jvms()->scloff()); |
| 724 | SafePointScalarObjectNode* sobj = new SafePointScalarObjectNode(res_type, |
| 725 | #ifdef ASSERT1 |
| 726 | alloc, |
| 727 | #endif |
| 728 | first_ind, nfields); |
| 729 | sobj->init_req(0, C->root()); |
| 730 | transform_later(sobj); |
| 731 | |
| 732 | // Scan object's fields adding an input to the safepoint for each field. |
| 733 | for (int j = 0; j < nfields; j++) { |
| 734 | intptr_t offset; |
| 735 | ciField* field = NULL__null; |
| 736 | if (iklass != NULL__null) { |
| 737 | field = iklass->nonstatic_field_at(j); |
| 738 | offset = field->offset(); |
| 739 | elem_type = field->type(); |
| 740 | basic_elem_type = field->layout_type(); |
| 741 | } else { |
| 742 | offset = array_base + j * (intptr_t)element_size; |
| 743 | } |
| 744 | |
| 745 | const Type *field_type; |
| 746 | // The next code is taken from Parse::do_get_xxx(). |
| 747 | if (is_reference_type(basic_elem_type)) { |
| 748 | if (!elem_type->is_loaded()) { |
| 749 | field_type = TypeInstPtr::BOTTOM; |
| 750 | } else if (field != NULL__null && field->is_static_constant()) { |
| 751 | // This can happen if the constant oop is non-perm. |
| 752 | ciObject* con = field->constant_value().as_object(); |
| 753 | // Do not "join" in the previous type; it doesn't add value, |
| 754 | // and may yield a vacuous result if the field is of interface type. |
| 755 | field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr(); |
| 756 | assert(field_type != NULL, "field singleton type must be consistent")do { if (!(field_type != __null)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 756, "assert(" "field_type != __null" ") failed", "field singleton type must be consistent" ); ::breakpoint(); } } while (0); |
| 757 | } else { |
| 758 | field_type = TypeOopPtr::make_from_klass(elem_type->as_klass()); |
| 759 | } |
| 760 | if (UseCompressedOops) { |
| 761 | field_type = field_type->make_narrowoop(); |
| 762 | basic_elem_type = T_NARROWOOP; |
| 763 | } |
| 764 | } else { |
| 765 | field_type = Type::get_const_basic_type(basic_elem_type); |
| 766 | } |
| 767 | |
| 768 | const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr(); |
| 769 | |
| 770 | Node *field_val = value_from_mem(mem, ctl, basic_elem_type, field_type, field_addr_type, alloc); |
| 771 | if (field_val == NULL__null) { |
| 772 | // We weren't able to find a value for this field, |
| 773 | // give up on eliminating this allocation. |
| 774 | |
| 775 | // Remove any extra entries we added to the safepoint. |
| 776 | uint last = sfpt->req() - 1; |
| 777 | for (int k = 0; k < j; k++) { |
| 778 | sfpt->del_req(last--); |
| 779 | } |
| 780 | _igvn._worklist.push(sfpt); |
| 781 | // rollback processed safepoints |
| 782 | while (safepoints_done.length() > 0) { |
| 783 | SafePointNode* sfpt_done = safepoints_done.pop(); |
| 784 | // remove any extra entries we added to the safepoint |
| 785 | last = sfpt_done->req() - 1; |
| 786 | for (int k = 0; k < nfields; k++) { |
| 787 | sfpt_done->del_req(last--); |
| 788 | } |
| 789 | JVMState *jvms = sfpt_done->jvms(); |
| 790 | jvms->set_endoff(sfpt_done->req()); |
| 791 | // Now make a pass over the debug information replacing any references |
| 792 | // to SafePointScalarObjectNode with the allocated object. |
| 793 | int start = jvms->debug_start(); |
| 794 | int end = jvms->debug_end(); |
| 795 | for (int i = start; i < end; i++) { |
| 796 | if (sfpt_done->in(i)->is_SafePointScalarObject()) { |
| 797 | SafePointScalarObjectNode* scobj = sfpt_done->in(i)->as_SafePointScalarObject(); |
| 798 | if (scobj->first_index(jvms) == sfpt_done->req() && |
| 799 | scobj->n_fields() == (uint)nfields) { |
| 800 | assert(scobj->alloc() == alloc, "sanity")do { if (!(scobj->alloc() == alloc)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 800, "assert(" "scobj->alloc() == alloc" ") failed", "sanity" ); ::breakpoint(); } } while (0); |
| 801 | sfpt_done->set_req(i, res); |
| 802 | } |
| 803 | } |
| 804 | } |
| 805 | _igvn._worklist.push(sfpt_done); |
| 806 | } |
| 807 | #ifndef PRODUCT |
| 808 | if (PrintEliminateAllocations) { |
| 809 | if (field != NULL__null) { |
| 810 | tty->print("=== At SafePoint node %d can't find value of Field: ", |
| 811 | sfpt->_idx); |
| 812 | field->print(); |
| 813 | int field_idx = C->get_alias_index(field_addr_type); |
| 814 | tty->print(" (alias_idx=%d)", field_idx); |
| 815 | } else { // Array's element |
| 816 | tty->print("=== At SafePoint node %d can't find value of array element [%d]", |
| 817 | sfpt->_idx, j); |
| 818 | } |
| 819 | tty->print(", which prevents elimination of: "); |
| 820 | if (res == NULL__null) |
| 821 | alloc->dump(); |
| 822 | else |
| 823 | res->dump(); |
| 824 | } |
| 825 | #endif |
| 826 | return false; |
| 827 | } |
| 828 | if (UseCompressedOops && field_type->isa_narrowoop()) { |
| 829 | // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation |
| 830 | // to be able scalar replace the allocation. |
| 831 | if (field_val->is_EncodeP()) { |
| 832 | field_val = field_val->in(1); |
| 833 | } else { |
| 834 | field_val = transform_later(new DecodeNNode(field_val, field_val->get_ptr_type())); |
| 835 | } |
| 836 | } |
| 837 | sfpt->add_req(field_val); |
| 838 | } |
| 839 | JVMState *jvms = sfpt->jvms(); |
| 840 | jvms->set_endoff(sfpt->req()); |
| 841 | // Now make a pass over the debug information replacing any references |
| 842 | // to the allocated object with "sobj" |
| 843 | int start = jvms->debug_start(); |
| 844 | int end = jvms->debug_end(); |
| 845 | sfpt->replace_edges_in_range(res, sobj, start, end, &_igvn); |
| 846 | _igvn._worklist.push(sfpt); |
| 847 | safepoints_done.append_if_missing(sfpt); // keep it for rollback |
| 848 | } |
| 849 | return true; |
| 850 | } |
| 851 | |
| 852 | static void disconnect_projections(MultiNode* n, PhaseIterGVN& igvn) { |
| 853 | Node* ctl_proj = n->proj_out_or_null(TypeFunc::Control); |
| 854 | Node* mem_proj = n->proj_out_or_null(TypeFunc::Memory); |
| 855 | if (ctl_proj != NULL__null) { |
| 856 | igvn.replace_node(ctl_proj, n->in(0)); |
| 857 | } |
| 858 | if (mem_proj != NULL__null) { |
| 859 | igvn.replace_node(mem_proj, n->in(TypeFunc::Memory)); |
| 860 | } |
| 861 | } |
| 862 | |
| 863 | // Process users of eliminated allocation. |
| 864 | void PhaseMacroExpand::process_users_of_allocation(CallNode *alloc) { |
| 865 | Node* res = alloc->result_cast(); |
| 866 | if (res != NULL__null) { |
| 867 | for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) { |
| 868 | Node *use = res->last_out(j); |
| 869 | uint oc1 = res->outcnt(); |
| 870 | |
| 871 | if (use->is_AddP()) { |
| 872 | for (DUIterator_Last kmin, k = use->last_outs(kmin); k >= kmin; ) { |
| 873 | Node *n = use->last_out(k); |
| 874 | uint oc2 = use->outcnt(); |
| 875 | if (n->is_Store()) { |
| 876 | #ifdef ASSERT1 |
| 877 | // Verify that there is no dependent MemBarVolatile nodes, |
| 878 | // they should be removed during IGVN, see MemBarNode::Ideal(). |
| 879 | for (DUIterator_Fast pmax, p = n->fast_outs(pmax); |
| 880 | p < pmax; p++) { |
| 881 | Node* mb = n->fast_out(p); |
| 882 | assert(mb->is_Initialize() || !mb->is_MemBar() ||do { if (!(mb->is_Initialize() || !mb->is_MemBar() || mb ->req() <= MemBarNode::Precedent || mb->in(MemBarNode ::Precedent) != n)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 885, "assert(" "mb->is_Initialize() || !mb->is_MemBar() || mb->req() <= MemBarNode::Precedent || mb->in(MemBarNode::Precedent) != n" ") failed", "MemBarVolatile should be eliminated for non-escaping object" ); ::breakpoint(); } } while (0) |
| 883 | mb->req() <= MemBarNode::Precedent ||do { if (!(mb->is_Initialize() || !mb->is_MemBar() || mb ->req() <= MemBarNode::Precedent || mb->in(MemBarNode ::Precedent) != n)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 885, "assert(" "mb->is_Initialize() || !mb->is_MemBar() || mb->req() <= MemBarNode::Precedent || mb->in(MemBarNode::Precedent) != n" ") failed", "MemBarVolatile should be eliminated for non-escaping object" ); ::breakpoint(); } } while (0) |
| 884 | mb->in(MemBarNode::Precedent) != n,do { if (!(mb->is_Initialize() || !mb->is_MemBar() || mb ->req() <= MemBarNode::Precedent || mb->in(MemBarNode ::Precedent) != n)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 885, "assert(" "mb->is_Initialize() || !mb->is_MemBar() || mb->req() <= MemBarNode::Precedent || mb->in(MemBarNode::Precedent) != n" ") failed", "MemBarVolatile should be eliminated for non-escaping object" ); ::breakpoint(); } } while (0) |
| 885 | "MemBarVolatile should be eliminated for non-escaping object")do { if (!(mb->is_Initialize() || !mb->is_MemBar() || mb ->req() <= MemBarNode::Precedent || mb->in(MemBarNode ::Precedent) != n)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 885, "assert(" "mb->is_Initialize() || !mb->is_MemBar() || mb->req() <= MemBarNode::Precedent || mb->in(MemBarNode::Precedent) != n" ") failed", "MemBarVolatile should be eliminated for non-escaping object" ); ::breakpoint(); } } while (0); |
| 886 | } |
| 887 | #endif |
| 888 | _igvn.replace_node(n, n->in(MemNode::Memory)); |
| 889 | } else { |
| 890 | eliminate_gc_barrier(n); |
| 891 | } |
| 892 | k -= (oc2 - use->outcnt()); |
| 893 | } |
| 894 | _igvn.remove_dead_node(use); |
| 895 | } else if (use->is_ArrayCopy()) { |
| 896 | // Disconnect ArrayCopy node |
| 897 | ArrayCopyNode* ac = use->as_ArrayCopy(); |
| 898 | if (ac->is_clonebasic()) { |
| 899 | Node* membar_after = ac->proj_out(TypeFunc::Control)->unique_ctrl_out(); |
| 900 | disconnect_projections(ac, _igvn); |
| 901 | assert(alloc->in(TypeFunc::Memory)->is_Proj() && alloc->in(TypeFunc::Memory)->in(0)->Opcode() == Op_MemBarCPUOrder, "mem barrier expected before allocation")do { if (!(alloc->in(TypeFunc::Memory)->is_Proj() && alloc->in(TypeFunc::Memory)->in(0)->Opcode() == Op_MemBarCPUOrder )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 901, "assert(" "alloc->in(TypeFunc::Memory)->is_Proj() && alloc->in(TypeFunc::Memory)->in(0)->Opcode() == Op_MemBarCPUOrder" ") failed", "mem barrier expected before allocation"); ::breakpoint (); } } while (0); |
| 902 | Node* membar_before = alloc->in(TypeFunc::Memory)->in(0); |
| 903 | disconnect_projections(membar_before->as_MemBar(), _igvn); |
| 904 | if (membar_after->is_MemBar()) { |
| 905 | disconnect_projections(membar_after->as_MemBar(), _igvn); |
| 906 | } |
| 907 | } else { |
| 908 | assert(ac->is_arraycopy_validated() ||do { if (!(ac->is_arraycopy_validated() || ac->is_copyof_validated () || ac->is_copyofrange_validated())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 910, "assert(" "ac->is_arraycopy_validated() || ac->is_copyof_validated() || ac->is_copyofrange_validated()" ") failed", "unsupported"); ::breakpoint(); } } while (0) |
| 909 | ac->is_copyof_validated() ||do { if (!(ac->is_arraycopy_validated() || ac->is_copyof_validated () || ac->is_copyofrange_validated())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 910, "assert(" "ac->is_arraycopy_validated() || ac->is_copyof_validated() || ac->is_copyofrange_validated()" ") failed", "unsupported"); ::breakpoint(); } } while (0) |
| 910 | ac->is_copyofrange_validated(), "unsupported")do { if (!(ac->is_arraycopy_validated() || ac->is_copyof_validated () || ac->is_copyofrange_validated())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 910, "assert(" "ac->is_arraycopy_validated() || ac->is_copyof_validated() || ac->is_copyofrange_validated()" ") failed", "unsupported"); ::breakpoint(); } } while (0); |
| 911 | CallProjections callprojs; |
| 912 | ac->extract_projections(&callprojs, true); |
| 913 | |
| 914 | _igvn.replace_node(callprojs.fallthrough_ioproj, ac->in(TypeFunc::I_O)); |
| 915 | _igvn.replace_node(callprojs.fallthrough_memproj, ac->in(TypeFunc::Memory)); |
| 916 | _igvn.replace_node(callprojs.fallthrough_catchproj, ac->in(TypeFunc::Control)); |
| 917 | |
| 918 | // Set control to top. IGVN will remove the remaining projections |
| 919 | ac->set_req(0, top()); |
| 920 | ac->replace_edge(res, top(), &_igvn); |
| 921 | |
| 922 | // Disconnect src right away: it can help find new |
| 923 | // opportunities for allocation elimination |
| 924 | Node* src = ac->in(ArrayCopyNode::Src); |
| 925 | ac->replace_edge(src, top(), &_igvn); |
| 926 | // src can be top at this point if src and dest of the |
| 927 | // arraycopy were the same |
| 928 | if (src->outcnt() == 0 && !src->is_top()) { |
| 929 | _igvn.remove_dead_node(src); |
| 930 | } |
| 931 | } |
| 932 | _igvn._worklist.push(ac); |
| 933 | } else { |
| 934 | eliminate_gc_barrier(use); |
| 935 | } |
| 936 | j -= (oc1 - res->outcnt()); |
| 937 | } |
| 938 | assert(res->outcnt() == 0, "all uses of allocated objects must be deleted")do { if (!(res->outcnt() == 0)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 938, "assert(" "res->outcnt() == 0" ") failed", "all uses of allocated objects must be deleted" ); ::breakpoint(); } } while (0); |
| 939 | _igvn.remove_dead_node(res); |
| 940 | } |
| 941 | |
| 942 | // |
| 943 | // Process other users of allocation's projections |
| 944 | // |
| 945 | if (_callprojs.resproj != NULL__null && _callprojs.resproj->outcnt() != 0) { |
| 946 | // First disconnect stores captured by Initialize node. |
| 947 | // If Initialize node is eliminated first in the following code, |
| 948 | // it will kill such stores and DUIterator_Last will assert. |
| 949 | for (DUIterator_Fast jmax, j = _callprojs.resproj->fast_outs(jmax); j < jmax; j++) { |
| 950 | Node* use = _callprojs.resproj->fast_out(j); |
| 951 | if (use->is_AddP()) { |
| 952 | // raw memory addresses used only by the initialization |
| 953 | _igvn.replace_node(use, C->top()); |
| 954 | --j; --jmax; |
| 955 | } |
| 956 | } |
| 957 | for (DUIterator_Last jmin, j = _callprojs.resproj->last_outs(jmin); j >= jmin; ) { |
| 958 | Node* use = _callprojs.resproj->last_out(j); |
| 959 | uint oc1 = _callprojs.resproj->outcnt(); |
| 960 | if (use->is_Initialize()) { |
| 961 | // Eliminate Initialize node. |
| 962 | InitializeNode *init = use->as_Initialize(); |
| 963 | assert(init->outcnt() <= 2, "only a control and memory projection expected")do { if (!(init->outcnt() <= 2)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 963, "assert(" "init->outcnt() <= 2" ") failed", "only a control and memory projection expected" ); ::breakpoint(); } } while (0); |
| 964 | Node *ctrl_proj = init->proj_out_or_null(TypeFunc::Control); |
| 965 | if (ctrl_proj != NULL__null) { |
| 966 | _igvn.replace_node(ctrl_proj, init->in(TypeFunc::Control)); |
| 967 | #ifdef ASSERT1 |
| 968 | // If the InitializeNode has no memory out, it will die, and tmp will become NULL |
| 969 | Node* tmp = init->in(TypeFunc::Control); |
| 970 | assert(tmp == NULL || tmp == _callprojs.fallthrough_catchproj, "allocation control projection")do { if (!(tmp == __null || tmp == _callprojs.fallthrough_catchproj )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 970, "assert(" "tmp == __null || tmp == _callprojs.fallthrough_catchproj" ") failed", "allocation control projection"); ::breakpoint() ; } } while (0); |
| 971 | #endif |
| 972 | } |
| 973 | Node *mem_proj = init->proj_out_or_null(TypeFunc::Memory); |
| 974 | if (mem_proj != NULL__null) { |
| 975 | Node *mem = init->in(TypeFunc::Memory); |
| 976 | #ifdef ASSERT1 |
| 977 | if (mem->is_MergeMem()) { |
| 978 | assert(mem->in(TypeFunc::Memory) == _callprojs.fallthrough_memproj, "allocation memory projection")do { if (!(mem->in(TypeFunc::Memory) == _callprojs.fallthrough_memproj )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 978, "assert(" "mem->in(TypeFunc::Memory) == _callprojs.fallthrough_memproj" ") failed", "allocation memory projection"); ::breakpoint(); } } while (0); |
| 979 | } else { |
| 980 | assert(mem == _callprojs.fallthrough_memproj, "allocation memory projection")do { if (!(mem == _callprojs.fallthrough_memproj)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 980, "assert(" "mem == _callprojs.fallthrough_memproj" ") failed" , "allocation memory projection"); ::breakpoint(); } } while ( 0); |
| 981 | } |
| 982 | #endif |
| 983 | _igvn.replace_node(mem_proj, mem); |
| 984 | } |
| 985 | } else { |
| 986 | assert(false, "only Initialize or AddP expected")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 986, "assert(" "false" ") failed", "only Initialize or AddP expected" ); ::breakpoint(); } } while (0); |
| 987 | } |
| 988 | j -= (oc1 - _callprojs.resproj->outcnt()); |
| 989 | } |
| 990 | } |
| 991 | if (_callprojs.fallthrough_catchproj != NULL__null) { |
| 992 | _igvn.replace_node(_callprojs.fallthrough_catchproj, alloc->in(TypeFunc::Control)); |
| 993 | } |
| 994 | if (_callprojs.fallthrough_memproj != NULL__null) { |
| 995 | _igvn.replace_node(_callprojs.fallthrough_memproj, alloc->in(TypeFunc::Memory)); |
| 996 | } |
| 997 | if (_callprojs.catchall_memproj != NULL__null) { |
| 998 | _igvn.replace_node(_callprojs.catchall_memproj, C->top()); |
| 999 | } |
| 1000 | if (_callprojs.fallthrough_ioproj != NULL__null) { |
| 1001 | _igvn.replace_node(_callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O)); |
| 1002 | } |
| 1003 | if (_callprojs.catchall_ioproj != NULL__null) { |
| 1004 | _igvn.replace_node(_callprojs.catchall_ioproj, C->top()); |
| 1005 | } |
| 1006 | if (_callprojs.catchall_catchproj != NULL__null) { |
| 1007 | _igvn.replace_node(_callprojs.catchall_catchproj, C->top()); |
| 1008 | } |
| 1009 | } |
| 1010 | |
| 1011 | bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) { |
| 1012 | // If reallocation fails during deoptimization we'll pop all |
| 1013 | // interpreter frames for this compiled frame and that won't play |
| 1014 | // nice with JVMTI popframe. |
| 1015 | // We avoid this issue by eager reallocation when the popframe request |
| 1016 | // is received. |
| 1017 | if (!EliminateAllocations || !alloc->_is_non_escaping) { |
| 1018 | return false; |
| 1019 | } |
| 1020 | Node* klass = alloc->in(AllocateNode::KlassNode); |
| 1021 | const TypeKlassPtr* tklass = _igvn.type(klass)->is_klassptr(); |
| 1022 | Node* res = alloc->result_cast(); |
| 1023 | // Eliminate boxing allocations which are not used |
| 1024 | // regardless scalar replacable status. |
| 1025 | bool boxing_alloc = C->eliminate_boxing() && |
| 1026 | tklass->klass()->is_instance_klass() && |
| 1027 | tklass->klass()->as_instance_klass()->is_box_klass(); |
| 1028 | if (!alloc->_is_scalar_replaceable && (!boxing_alloc || (res != NULL__null))) { |
| 1029 | return false; |
| 1030 | } |
| 1031 | |
| 1032 | alloc->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 1033 | |
| 1034 | GrowableArray <SafePointNode *> safepoints; |
| 1035 | if (!can_eliminate_allocation(alloc, safepoints)) { |
| 1036 | return false; |
| 1037 | } |
| 1038 | |
| 1039 | if (!alloc->_is_scalar_replaceable) { |
| 1040 | assert(res == NULL, "sanity")do { if (!(res == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1040, "assert(" "res == __null" ") failed", "sanity"); ::breakpoint (); } } while (0); |
| 1041 | // We can only eliminate allocation if all debug info references |
| 1042 | // are already replaced with SafePointScalarObject because |
| 1043 | // we can't search for a fields value without instance_id. |
| 1044 | if (safepoints.length() > 0) { |
| 1045 | return false; |
| 1046 | } |
| 1047 | } |
| 1048 | |
| 1049 | if (!scalar_replacement(alloc, safepoints)) { |
| 1050 | return false; |
| 1051 | } |
| 1052 | |
| 1053 | CompileLog* log = C->log(); |
| 1054 | if (log != NULL__null) { |
| 1055 | log->head("eliminate_allocation type='%d'", |
| 1056 | log->identify(tklass->klass())); |
| 1057 | JVMState* p = alloc->jvms(); |
| 1058 | while (p != NULL__null) { |
| 1059 | log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); |
| 1060 | p = p->caller(); |
| 1061 | } |
| 1062 | log->tail("eliminate_allocation"); |
| 1063 | } |
| 1064 | |
| 1065 | process_users_of_allocation(alloc); |
| 1066 | |
| 1067 | #ifndef PRODUCT |
| 1068 | if (PrintEliminateAllocations) { |
| 1069 | if (alloc->is_AllocateArray()) |
| 1070 | tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx); |
| 1071 | else |
| 1072 | tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx); |
| 1073 | } |
| 1074 | #endif |
| 1075 | |
| 1076 | return true; |
| 1077 | } |
| 1078 | |
| 1079 | bool PhaseMacroExpand::eliminate_boxing_node(CallStaticJavaNode *boxing) { |
| 1080 | // EA should remove all uses of non-escaping boxing node. |
| 1081 | if (!C->eliminate_boxing() || boxing->proj_out_or_null(TypeFunc::Parms) != NULL__null) { |
| 1082 | return false; |
| 1083 | } |
| 1084 | |
| 1085 | assert(boxing->result_cast() == NULL, "unexpected boxing node result")do { if (!(boxing->result_cast() == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1085, "assert(" "boxing->result_cast() == __null" ") failed" , "unexpected boxing node result"); ::breakpoint(); } } while (0); |
| 1086 | |
| 1087 | boxing->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 1088 | |
| 1089 | const TypeTuple* r = boxing->tf()->range(); |
| 1090 | assert(r->cnt() > TypeFunc::Parms, "sanity")do { if (!(r->cnt() > TypeFunc::Parms)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1090, "assert(" "r->cnt() > TypeFunc::Parms" ") failed" , "sanity"); ::breakpoint(); } } while (0); |
| 1091 | const TypeInstPtr* t = r->field_at(TypeFunc::Parms)->isa_instptr(); |
| 1092 | assert(t != NULL, "sanity")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1092, "assert(" "t != __null" ") failed", "sanity"); ::breakpoint (); } } while (0); |
| 1093 | |
| 1094 | CompileLog* log = C->log(); |
| 1095 | if (log != NULL__null) { |
| 1096 | log->head("eliminate_boxing type='%d'", |
| 1097 | log->identify(t->klass())); |
| 1098 | JVMState* p = boxing->jvms(); |
| 1099 | while (p != NULL__null) { |
| 1100 | log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); |
| 1101 | p = p->caller(); |
| 1102 | } |
| 1103 | log->tail("eliminate_boxing"); |
| 1104 | } |
| 1105 | |
| 1106 | process_users_of_allocation(boxing); |
| 1107 | |
| 1108 | #ifndef PRODUCT |
| 1109 | if (PrintEliminateAllocations) { |
| 1110 | tty->print("++++ Eliminated: %d ", boxing->_idx); |
| 1111 | boxing->method()->print_short_name(tty); |
| 1112 | tty->cr(); |
| 1113 | } |
| 1114 | #endif |
| 1115 | |
| 1116 | return true; |
| 1117 | } |
| 1118 | |
| 1119 | //---------------------------set_eden_pointers------------------------- |
| 1120 | void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) { |
| 1121 | if (UseTLAB) { // Private allocation: load from TLS |
| 1122 | Node* thread = transform_later(new ThreadLocalNode()); |
| 1123 | int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset()); |
| 1124 | int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset()); |
| 1125 | eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset); |
| 1126 | eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset); |
| 1127 | } else { // Shared allocation: load from globals |
| 1128 | CollectedHeap* ch = Universe::heap(); |
| 1129 | address top_adr = (address)ch->top_addr(); |
| 1130 | address end_adr = (address)ch->end_addr(); |
| 1131 | eden_top_adr = makecon(TypeRawPtr::make(top_adr)); |
| 1132 | eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr); |
| 1133 | } |
| 1134 | } |
| 1135 | |
| 1136 | |
| 1137 | Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) { |
| 1138 | Node* adr = basic_plus_adr(base, offset); |
| 1139 | const TypePtr* adr_type = adr->bottom_type()->is_ptr(); |
| 1140 | Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt, MemNode::unordered); |
| 1141 | transform_later(value); |
| 1142 | return value; |
| 1143 | } |
| 1144 | |
| 1145 | |
| 1146 | Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) { |
| 1147 | Node* adr = basic_plus_adr(base, offset); |
| 1148 | mem = StoreNode::make(_igvn, ctl, mem, adr, NULL__null, value, bt, MemNode::unordered); |
| 1149 | transform_later(mem); |
| 1150 | return mem; |
| 1151 | } |
| 1152 | |
| 1153 | //============================================================================= |
| 1154 | // |
| 1155 | // A L L O C A T I O N |
| 1156 | // |
| 1157 | // Allocation attempts to be fast in the case of frequent small objects. |
| 1158 | // It breaks down like this: |
| 1159 | // |
| 1160 | // 1) Size in doublewords is computed. This is a constant for objects and |
| 1161 | // variable for most arrays. Doubleword units are used to avoid size |
| 1162 | // overflow of huge doubleword arrays. We need doublewords in the end for |
| 1163 | // rounding. |
| 1164 | // |
| 1165 | // 2) Size is checked for being 'too large'. Too-large allocations will go |
| 1166 | // the slow path into the VM. The slow path can throw any required |
| 1167 | // exceptions, and does all the special checks for very large arrays. The |
| 1168 | // size test can constant-fold away for objects. For objects with |
| 1169 | // finalizers it constant-folds the otherway: you always go slow with |
| 1170 | // finalizers. |
| 1171 | // |
| 1172 | // 3) If NOT using TLABs, this is the contended loop-back point. |
| 1173 | // Load-Locked the heap top. If using TLABs normal-load the heap top. |
| 1174 | // |
| 1175 | // 4) Check that heap top + size*8 < max. If we fail go the slow ` route. |
| 1176 | // NOTE: "top+size*8" cannot wrap the 4Gig line! Here's why: for largish |
| 1177 | // "size*8" we always enter the VM, where "largish" is a constant picked small |
| 1178 | // enough that there's always space between the eden max and 4Gig (old space is |
| 1179 | // there so it's quite large) and large enough that the cost of entering the VM |
| 1180 | // is dwarfed by the cost to initialize the space. |
| 1181 | // |
| 1182 | // 5) If NOT using TLABs, Store-Conditional the adjusted heap top back |
| 1183 | // down. If contended, repeat at step 3. If using TLABs normal-store |
| 1184 | // adjusted heap top back down; there is no contention. |
| 1185 | // |
| 1186 | // 6) If !ZeroTLAB then Bulk-clear the object/array. Fill in klass & mark |
| 1187 | // fields. |
| 1188 | // |
| 1189 | // 7) Merge with the slow-path; cast the raw memory pointer to the correct |
| 1190 | // oop flavor. |
| 1191 | // |
| 1192 | //============================================================================= |
| 1193 | // FastAllocateSizeLimit value is in DOUBLEWORDS. |
| 1194 | // Allocations bigger than this always go the slow route. |
| 1195 | // This value must be small enough that allocation attempts that need to |
| 1196 | // trigger exceptions go the slow route. Also, it must be small enough so |
| 1197 | // that heap_top + size_in_bytes does not wrap around the 4Gig limit. |
| 1198 | //=============================================================================j// |
| 1199 | // %%% Here is an old comment from parseHelper.cpp; is it outdated? |
| 1200 | // The allocator will coalesce int->oop copies away. See comment in |
| 1201 | // coalesce.cpp about how this works. It depends critically on the exact |
| 1202 | // code shape produced here, so if you are changing this code shape |
| 1203 | // make sure the GC info for the heap-top is correct in and around the |
| 1204 | // slow-path call. |
| 1205 | // |
| 1206 | |
| 1207 | void PhaseMacroExpand::expand_allocate_common( |
| 1208 | AllocateNode* alloc, // allocation node to be expanded |
| 1209 | Node* length, // array length for an array allocation |
| 1210 | const TypeFunc* slow_call_type, // Type of slow call |
| 1211 | address slow_call_address // Address of slow call |
| 1212 | ) |
| 1213 | { |
| 1214 | Node* ctrl = alloc->in(TypeFunc::Control); |
| 1215 | Node* mem = alloc->in(TypeFunc::Memory); |
| 1216 | Node* i_o = alloc->in(TypeFunc::I_O); |
| 1217 | Node* size_in_bytes = alloc->in(AllocateNode::AllocSize); |
| 1218 | Node* klass_node = alloc->in(AllocateNode::KlassNode); |
| 1219 | Node* initial_slow_test = alloc->in(AllocateNode::InitialTest); |
| 1220 | assert(ctrl != NULL, "must have control")do { if (!(ctrl != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1220, "assert(" "ctrl != __null" ") failed", "must have control" ); ::breakpoint(); } } while (0); |
| 1221 | |
| 1222 | // We need a Region and corresponding Phi's to merge the slow-path and fast-path results. |
| 1223 | // they will not be used if "always_slow" is set |
| 1224 | enum { slow_result_path = 1, fast_result_path = 2 }; |
| 1225 | Node *result_region = NULL__null; |
| 1226 | Node *result_phi_rawmem = NULL__null; |
| 1227 | Node *result_phi_rawoop = NULL__null; |
| 1228 | Node *result_phi_i_o = NULL__null; |
| 1229 | |
| 1230 | // The initial slow comparison is a size check, the comparison |
| 1231 | // we want to do is a BoolTest::gt |
| 1232 | bool expand_fast_path = true; |
| 1233 | int tv = _igvn.find_int_con(initial_slow_test, -1); |
| 1234 | if (tv >= 0) { |
| 1235 | // InitialTest has constant result |
| 1236 | // 0 - can fit in TLAB |
| 1237 | // 1 - always too big or negative |
| 1238 | assert(tv <= 1, "0 or 1 if a constant")do { if (!(tv <= 1)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1238, "assert(" "tv <= 1" ") failed", "0 or 1 if a constant" ); ::breakpoint(); } } while (0); |
| 1239 | expand_fast_path = (tv == 0); |
| 1240 | initial_slow_test = NULL__null; |
| 1241 | } else { |
| 1242 | initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn); |
| 1243 | } |
| 1244 | |
| 1245 | if (!UseTLAB && !Universe::heap()->supports_inline_contig_alloc()) { |
| 1246 | // Force slow-path allocation |
| 1247 | expand_fast_path = false; |
| 1248 | initial_slow_test = NULL__null; |
| 1249 | } |
| 1250 | |
| 1251 | bool allocation_has_use = (alloc->result_cast() != NULL__null); |
| 1252 | if (!allocation_has_use) { |
| 1253 | InitializeNode* init = alloc->initialization(); |
| 1254 | if (init != NULL__null) { |
| 1255 | init->remove(&_igvn); |
| 1256 | } |
| 1257 | if (expand_fast_path && (initial_slow_test == NULL__null)) { |
| 1258 | // Remove allocation node and return. |
| 1259 | // Size is a non-negative constant -> no initial check needed -> directly to fast path. |
| 1260 | // Also, no usages -> empty fast path -> no fall out to slow path -> nothing left. |
| 1261 | #ifndef PRODUCT |
| 1262 | if (PrintEliminateAllocations) { |
| 1263 | tty->print("NotUsed "); |
| 1264 | Node* res = alloc->proj_out_or_null(TypeFunc::Parms); |
| 1265 | if (res != NULL__null) { |
| 1266 | res->dump(); |
| 1267 | } else { |
| 1268 | alloc->dump(); |
| 1269 | } |
| 1270 | } |
| 1271 | #endif |
| 1272 | yank_alloc_node(alloc); |
| 1273 | return; |
| 1274 | } |
| 1275 | } |
| 1276 | |
| 1277 | enum { too_big_or_final_path = 1, need_gc_path = 2 }; |
| 1278 | Node *slow_region = NULL__null; |
| 1279 | Node *toobig_false = ctrl; |
Value stored to 'toobig_false' during its initialization is never read | |
| 1280 | |
| 1281 | // generate the initial test if necessary |
| 1282 | if (initial_slow_test != NULL__null ) { |
| 1283 | assert (expand_fast_path, "Only need test if there is a fast path")do { if (!(expand_fast_path)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1283, "assert(" "expand_fast_path" ") failed", "Only need test if there is a fast path" ); ::breakpoint(); } } while (0); |
| 1284 | slow_region = new RegionNode(3); |
| 1285 | |
| 1286 | // Now make the initial failure test. Usually a too-big test but |
| 1287 | // might be a TRUE for finalizers or a fancy class check for |
| 1288 | // newInstance0. |
| 1289 | IfNode *toobig_iff = new IfNode(ctrl, initial_slow_test, PROB_MIN(1e-6f), COUNT_UNKNOWN(-1.0f)); |
| 1290 | transform_later(toobig_iff); |
| 1291 | // Plug the failing-too-big test into the slow-path region |
| 1292 | Node *toobig_true = new IfTrueNode( toobig_iff ); |
| 1293 | transform_later(toobig_true); |
| 1294 | slow_region ->init_req( too_big_or_final_path, toobig_true ); |
| 1295 | toobig_false = new IfFalseNode( toobig_iff ); |
| 1296 | transform_later(toobig_false); |
| 1297 | } else { |
| 1298 | // No initial test, just fall into next case |
| 1299 | assert(allocation_has_use || !expand_fast_path, "Should already have been handled")do { if (!(allocation_has_use || !expand_fast_path)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1299, "assert(" "allocation_has_use || !expand_fast_path" ") failed" , "Should already have been handled"); ::breakpoint(); } } while (0); |
| 1300 | toobig_false = ctrl; |
| 1301 | debug_only(slow_region = NodeSentinel)slow_region = (Node*)-1; |
| 1302 | } |
| 1303 | |
| 1304 | // If we are here there are several possibilities |
| 1305 | // - expand_fast_path is false - then only a slow path is expanded. That's it. |
| 1306 | // no_initial_check means a constant allocation. |
| 1307 | // - If check always evaluates to false -> expand_fast_path is false (see above) |
| 1308 | // - If check always evaluates to true -> directly into fast path (but may bailout to slowpath) |
| 1309 | // if !allocation_has_use the fast path is empty |
| 1310 | // if !allocation_has_use && no_initial_check |
| 1311 | // - Then there are no fastpath that can fall out to slowpath -> no allocation code at all. |
| 1312 | // removed by yank_alloc_node above. |
| 1313 | |
| 1314 | Node *slow_mem = mem; // save the current memory state for slow path |
| 1315 | // generate the fast allocation code unless we know that the initial test will always go slow |
| 1316 | if (expand_fast_path) { |
| 1317 | // Fast path modifies only raw memory. |
| 1318 | if (mem->is_MergeMem()) { |
| 1319 | mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); |
| 1320 | } |
| 1321 | |
| 1322 | // allocate the Region and Phi nodes for the result |
| 1323 | result_region = new RegionNode(3); |
| 1324 | result_phi_rawmem = new PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM); |
| 1325 | result_phi_i_o = new PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch |
| 1326 | |
| 1327 | // Grab regular I/O before optional prefetch may change it. |
| 1328 | // Slow-path does no I/O so just set it to the original I/O. |
| 1329 | result_phi_i_o->init_req(slow_result_path, i_o); |
| 1330 | |
| 1331 | // Name successful fast-path variables |
| 1332 | Node* fast_oop_ctrl; |
| 1333 | Node* fast_oop_rawmem; |
| 1334 | if (allocation_has_use) { |
| 1335 | Node* needgc_ctrl = NULL__null; |
| 1336 | result_phi_rawoop = new PhiNode(result_region, TypeRawPtr::BOTTOM); |
| 1337 | |
| 1338 | intx prefetch_lines = length != NULL__null ? AllocatePrefetchLines : AllocateInstancePrefetchLines; |
| 1339 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 1340 | Node* fast_oop = bs->obj_allocate(this, mem, toobig_false, size_in_bytes, i_o, needgc_ctrl, |
| 1341 | fast_oop_ctrl, fast_oop_rawmem, |
| 1342 | prefetch_lines); |
| 1343 | |
| 1344 | if (initial_slow_test != NULL__null) { |
| 1345 | // This completes all paths into the slow merge point |
| 1346 | slow_region->init_req(need_gc_path, needgc_ctrl); |
| 1347 | transform_later(slow_region); |
| 1348 | } else { |
| 1349 | // No initial slow path needed! |
| 1350 | // Just fall from the need-GC path straight into the VM call. |
| 1351 | slow_region = needgc_ctrl; |
| 1352 | } |
| 1353 | |
| 1354 | InitializeNode* init = alloc->initialization(); |
| 1355 | fast_oop_rawmem = initialize_object(alloc, |
| 1356 | fast_oop_ctrl, fast_oop_rawmem, fast_oop, |
| 1357 | klass_node, length, size_in_bytes); |
| 1358 | expand_initialize_membar(alloc, init, fast_oop_ctrl, fast_oop_rawmem); |
| 1359 | expand_dtrace_alloc_probe(alloc, fast_oop, fast_oop_ctrl, fast_oop_rawmem); |
| 1360 | |
| 1361 | result_phi_rawoop->init_req(fast_result_path, fast_oop); |
| 1362 | } else { |
| 1363 | assert (initial_slow_test != NULL, "sanity")do { if (!(initial_slow_test != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1363, "assert(" "initial_slow_test != __null" ") failed", "sanity" ); ::breakpoint(); } } while (0); |
| 1364 | fast_oop_ctrl = toobig_false; |
| 1365 | fast_oop_rawmem = mem; |
| 1366 | transform_later(slow_region); |
| 1367 | } |
| 1368 | |
| 1369 | // Plug in the successful fast-path into the result merge point |
| 1370 | result_region ->init_req(fast_result_path, fast_oop_ctrl); |
| 1371 | result_phi_i_o ->init_req(fast_result_path, i_o); |
| 1372 | result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem); |
| 1373 | } else { |
| 1374 | slow_region = ctrl; |
| 1375 | result_phi_i_o = i_o; // Rename it to use in the following code. |
| 1376 | } |
| 1377 | |
| 1378 | // Generate slow-path call |
| 1379 | CallNode *call = new CallStaticJavaNode(slow_call_type, slow_call_address, |
| 1380 | OptoRuntime::stub_name(slow_call_address), |
| 1381 | TypePtr::BOTTOM); |
| 1382 | call->init_req(TypeFunc::Control, slow_region); |
| 1383 | call->init_req(TypeFunc::I_O, top()); // does no i/o |
| 1384 | call->init_req(TypeFunc::Memory, slow_mem); // may gc ptrs |
| 1385 | call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr)); |
| 1386 | call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr)); |
| 1387 | |
| 1388 | call->init_req(TypeFunc::Parms+0, klass_node); |
| 1389 | if (length != NULL__null) { |
| 1390 | call->init_req(TypeFunc::Parms+1, length); |
| 1391 | } |
| 1392 | |
| 1393 | // Copy debug information and adjust JVMState information, then replace |
| 1394 | // allocate node with the call |
| 1395 | call->copy_call_debug_info(&_igvn, alloc); |
| 1396 | if (expand_fast_path) { |
| 1397 | call->set_cnt(PROB_UNLIKELY_MAG(4)(1e-4f)); // Same effect as RC_UNCOMMON. |
| 1398 | } else { |
| 1399 | // Hook i_o projection to avoid its elimination during allocation |
| 1400 | // replacement (when only a slow call is generated). |
| 1401 | call->set_req(TypeFunc::I_O, result_phi_i_o); |
| 1402 | } |
| 1403 | _igvn.replace_node(alloc, call); |
| 1404 | transform_later(call); |
| 1405 | |
| 1406 | // Identify the output projections from the allocate node and |
| 1407 | // adjust any references to them. |
| 1408 | // The control and io projections look like: |
| 1409 | // |
| 1410 | // v---Proj(ctrl) <-----+ v---CatchProj(ctrl) |
| 1411 | // Allocate Catch |
| 1412 | // ^---Proj(io) <-------+ ^---CatchProj(io) |
| 1413 | // |
| 1414 | // We are interested in the CatchProj nodes. |
| 1415 | // |
| 1416 | call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 1417 | |
| 1418 | // An allocate node has separate memory projections for the uses on |
| 1419 | // the control and i_o paths. Replace the control memory projection with |
| 1420 | // result_phi_rawmem (unless we are only generating a slow call when |
| 1421 | // both memory projections are combined) |
| 1422 | if (expand_fast_path && _callprojs.fallthrough_memproj != NULL__null) { |
| 1423 | migrate_outs(_callprojs.fallthrough_memproj, result_phi_rawmem); |
| 1424 | } |
| 1425 | // Now change uses of catchall_memproj to use fallthrough_memproj and delete |
| 1426 | // catchall_memproj so we end up with a call that has only 1 memory projection. |
| 1427 | if (_callprojs.catchall_memproj != NULL__null ) { |
| 1428 | if (_callprojs.fallthrough_memproj == NULL__null) { |
| 1429 | _callprojs.fallthrough_memproj = new ProjNode(call, TypeFunc::Memory); |
| 1430 | transform_later(_callprojs.fallthrough_memproj); |
| 1431 | } |
| 1432 | migrate_outs(_callprojs.catchall_memproj, _callprojs.fallthrough_memproj); |
| 1433 | _igvn.remove_dead_node(_callprojs.catchall_memproj); |
| 1434 | } |
| 1435 | |
| 1436 | // An allocate node has separate i_o projections for the uses on the control |
| 1437 | // and i_o paths. Always replace the control i_o projection with result i_o |
| 1438 | // otherwise incoming i_o become dead when only a slow call is generated |
| 1439 | // (it is different from memory projections where both projections are |
| 1440 | // combined in such case). |
| 1441 | if (_callprojs.fallthrough_ioproj != NULL__null) { |
| 1442 | migrate_outs(_callprojs.fallthrough_ioproj, result_phi_i_o); |
| 1443 | } |
| 1444 | // Now change uses of catchall_ioproj to use fallthrough_ioproj and delete |
| 1445 | // catchall_ioproj so we end up with a call that has only 1 i_o projection. |
| 1446 | if (_callprojs.catchall_ioproj != NULL__null ) { |
| 1447 | if (_callprojs.fallthrough_ioproj == NULL__null) { |
| 1448 | _callprojs.fallthrough_ioproj = new ProjNode(call, TypeFunc::I_O); |
| 1449 | transform_later(_callprojs.fallthrough_ioproj); |
| 1450 | } |
| 1451 | migrate_outs(_callprojs.catchall_ioproj, _callprojs.fallthrough_ioproj); |
| 1452 | _igvn.remove_dead_node(_callprojs.catchall_ioproj); |
| 1453 | } |
| 1454 | |
| 1455 | // if we generated only a slow call, we are done |
| 1456 | if (!expand_fast_path) { |
| 1457 | // Now we can unhook i_o. |
| 1458 | if (result_phi_i_o->outcnt() > 1) { |
| 1459 | call->set_req(TypeFunc::I_O, top()); |
| 1460 | } else { |
| 1461 | assert(result_phi_i_o->unique_ctrl_out() == call, "sanity")do { if (!(result_phi_i_o->unique_ctrl_out() == call)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1461, "assert(" "result_phi_i_o->unique_ctrl_out() == call" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 1462 | // Case of new array with negative size known during compilation. |
| 1463 | // AllocateArrayNode::Ideal() optimization disconnect unreachable |
| 1464 | // following code since call to runtime will throw exception. |
| 1465 | // As result there will be no users of i_o after the call. |
| 1466 | // Leave i_o attached to this call to avoid problems in preceding graph. |
| 1467 | } |
| 1468 | return; |
| 1469 | } |
| 1470 | |
| 1471 | if (_callprojs.fallthrough_catchproj != NULL__null) { |
| 1472 | ctrl = _callprojs.fallthrough_catchproj->clone(); |
| 1473 | transform_later(ctrl); |
| 1474 | _igvn.replace_node(_callprojs.fallthrough_catchproj, result_region); |
| 1475 | } else { |
| 1476 | ctrl = top(); |
| 1477 | } |
| 1478 | Node *slow_result; |
| 1479 | if (_callprojs.resproj == NULL__null) { |
| 1480 | // no uses of the allocation result |
| 1481 | slow_result = top(); |
| 1482 | } else { |
| 1483 | slow_result = _callprojs.resproj->clone(); |
| 1484 | transform_later(slow_result); |
| 1485 | _igvn.replace_node(_callprojs.resproj, result_phi_rawoop); |
| 1486 | } |
| 1487 | |
| 1488 | // Plug slow-path into result merge point |
| 1489 | result_region->init_req( slow_result_path, ctrl); |
| 1490 | transform_later(result_region); |
| 1491 | if (allocation_has_use) { |
| 1492 | result_phi_rawoop->init_req(slow_result_path, slow_result); |
| 1493 | transform_later(result_phi_rawoop); |
| 1494 | } |
| 1495 | result_phi_rawmem->init_req(slow_result_path, _callprojs.fallthrough_memproj); |
| 1496 | transform_later(result_phi_rawmem); |
| 1497 | transform_later(result_phi_i_o); |
| 1498 | // This completes all paths into the result merge point |
| 1499 | } |
| 1500 | |
| 1501 | // Remove alloc node that has no uses. |
| 1502 | void PhaseMacroExpand::yank_alloc_node(AllocateNode* alloc) { |
| 1503 | Node* ctrl = alloc->in(TypeFunc::Control); |
| 1504 | Node* mem = alloc->in(TypeFunc::Memory); |
| 1505 | Node* i_o = alloc->in(TypeFunc::I_O); |
| 1506 | |
| 1507 | alloc->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 1508 | if (_callprojs.resproj != NULL__null) { |
| 1509 | for (DUIterator_Fast imax, i = _callprojs.resproj->fast_outs(imax); i < imax; i++) { |
| 1510 | Node* use = _callprojs.resproj->fast_out(i); |
| 1511 | use->isa_MemBar()->remove(&_igvn); |
| 1512 | --imax; |
| 1513 | --i; // back up iterator |
| 1514 | } |
| 1515 | assert(_callprojs.resproj->outcnt() == 0, "all uses must be deleted")do { if (!(_callprojs.resproj->outcnt() == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1515, "assert(" "_callprojs.resproj->outcnt() == 0" ") failed" , "all uses must be deleted"); ::breakpoint(); } } while (0); |
| 1516 | _igvn.remove_dead_node(_callprojs.resproj); |
| 1517 | } |
| 1518 | if (_callprojs.fallthrough_catchproj != NULL__null) { |
| 1519 | migrate_outs(_callprojs.fallthrough_catchproj, ctrl); |
| 1520 | _igvn.remove_dead_node(_callprojs.fallthrough_catchproj); |
| 1521 | } |
| 1522 | if (_callprojs.catchall_catchproj != NULL__null) { |
| 1523 | _igvn.rehash_node_delayed(_callprojs.catchall_catchproj); |
| 1524 | _callprojs.catchall_catchproj->set_req(0, top()); |
| 1525 | } |
| 1526 | if (_callprojs.fallthrough_proj != NULL__null) { |
| 1527 | Node* catchnode = _callprojs.fallthrough_proj->unique_ctrl_out(); |
| 1528 | _igvn.remove_dead_node(catchnode); |
| 1529 | _igvn.remove_dead_node(_callprojs.fallthrough_proj); |
| 1530 | } |
| 1531 | if (_callprojs.fallthrough_memproj != NULL__null) { |
| 1532 | migrate_outs(_callprojs.fallthrough_memproj, mem); |
| 1533 | _igvn.remove_dead_node(_callprojs.fallthrough_memproj); |
| 1534 | } |
| 1535 | if (_callprojs.fallthrough_ioproj != NULL__null) { |
| 1536 | migrate_outs(_callprojs.fallthrough_ioproj, i_o); |
| 1537 | _igvn.remove_dead_node(_callprojs.fallthrough_ioproj); |
| 1538 | } |
| 1539 | if (_callprojs.catchall_memproj != NULL__null) { |
| 1540 | _igvn.rehash_node_delayed(_callprojs.catchall_memproj); |
| 1541 | _callprojs.catchall_memproj->set_req(0, top()); |
| 1542 | } |
| 1543 | if (_callprojs.catchall_ioproj != NULL__null) { |
| 1544 | _igvn.rehash_node_delayed(_callprojs.catchall_ioproj); |
| 1545 | _callprojs.catchall_ioproj->set_req(0, top()); |
| 1546 | } |
| 1547 | #ifndef PRODUCT |
| 1548 | if (PrintEliminateAllocations) { |
| 1549 | if (alloc->is_AllocateArray()) { |
| 1550 | tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx); |
| 1551 | } else { |
| 1552 | tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx); |
| 1553 | } |
| 1554 | } |
| 1555 | #endif |
| 1556 | _igvn.remove_dead_node(alloc); |
| 1557 | } |
| 1558 | |
| 1559 | void PhaseMacroExpand::expand_initialize_membar(AllocateNode* alloc, InitializeNode* init, |
| 1560 | Node*& fast_oop_ctrl, Node*& fast_oop_rawmem) { |
| 1561 | // If initialization is performed by an array copy, any required |
| 1562 | // MemBarStoreStore was already added. If the object does not |
| 1563 | // escape no need for a MemBarStoreStore. If the object does not |
| 1564 | // escape in its initializer and memory barrier (MemBarStoreStore or |
| 1565 | // stronger) is already added at exit of initializer, also no need |
| 1566 | // for a MemBarStoreStore. Otherwise we need a MemBarStoreStore |
| 1567 | // so that stores that initialize this object can't be reordered |
| 1568 | // with a subsequent store that makes this object accessible by |
| 1569 | // other threads. |
| 1570 | // Other threads include java threads and JVM internal threads |
| 1571 | // (for example concurrent GC threads). Current concurrent GC |
| 1572 | // implementation: G1 will not scan newly created object, |
| 1573 | // so it's safe to skip storestore barrier when allocation does |
| 1574 | // not escape. |
| 1575 | if (!alloc->does_not_escape_thread() && |
| 1576 | !alloc->is_allocation_MemBar_redundant() && |
| 1577 | (init == NULL__null || !init->is_complete_with_arraycopy())) { |
| 1578 | if (init == NULL__null || init->req() < InitializeNode::RawStores) { |
| 1579 | // No InitializeNode or no stores captured by zeroing |
| 1580 | // elimination. Simply add the MemBarStoreStore after object |
| 1581 | // initialization. |
| 1582 | MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot); |
| 1583 | transform_later(mb); |
| 1584 | |
| 1585 | mb->init_req(TypeFunc::Memory, fast_oop_rawmem); |
| 1586 | mb->init_req(TypeFunc::Control, fast_oop_ctrl); |
| 1587 | fast_oop_ctrl = new ProjNode(mb, TypeFunc::Control); |
| 1588 | transform_later(fast_oop_ctrl); |
| 1589 | fast_oop_rawmem = new ProjNode(mb, TypeFunc::Memory); |
| 1590 | transform_later(fast_oop_rawmem); |
| 1591 | } else { |
| 1592 | // Add the MemBarStoreStore after the InitializeNode so that |
| 1593 | // all stores performing the initialization that were moved |
| 1594 | // before the InitializeNode happen before the storestore |
| 1595 | // barrier. |
| 1596 | |
| 1597 | Node* init_ctrl = init->proj_out_or_null(TypeFunc::Control); |
| 1598 | Node* init_mem = init->proj_out_or_null(TypeFunc::Memory); |
| 1599 | |
| 1600 | MemBarNode* mb = MemBarNode::make(C, Op_MemBarStoreStore, Compile::AliasIdxBot); |
| 1601 | transform_later(mb); |
| 1602 | |
| 1603 | Node* ctrl = new ProjNode(init, TypeFunc::Control); |
| 1604 | transform_later(ctrl); |
| 1605 | Node* mem = new ProjNode(init, TypeFunc::Memory); |
| 1606 | transform_later(mem); |
| 1607 | |
| 1608 | // The MemBarStoreStore depends on control and memory coming |
| 1609 | // from the InitializeNode |
| 1610 | mb->init_req(TypeFunc::Memory, mem); |
| 1611 | mb->init_req(TypeFunc::Control, ctrl); |
| 1612 | |
| 1613 | ctrl = new ProjNode(mb, TypeFunc::Control); |
| 1614 | transform_later(ctrl); |
| 1615 | mem = new ProjNode(mb, TypeFunc::Memory); |
| 1616 | transform_later(mem); |
| 1617 | |
| 1618 | // All nodes that depended on the InitializeNode for control |
| 1619 | // and memory must now depend on the MemBarNode that itself |
| 1620 | // depends on the InitializeNode |
| 1621 | if (init_ctrl != NULL__null) { |
| 1622 | _igvn.replace_node(init_ctrl, ctrl); |
| 1623 | } |
| 1624 | if (init_mem != NULL__null) { |
| 1625 | _igvn.replace_node(init_mem, mem); |
| 1626 | } |
| 1627 | } |
| 1628 | } |
| 1629 | } |
| 1630 | |
| 1631 | void PhaseMacroExpand::expand_dtrace_alloc_probe(AllocateNode* alloc, Node* oop, |
| 1632 | Node*& ctrl, Node*& rawmem) { |
| 1633 | if (C->env()->dtrace_alloc_probes()) { |
| 1634 | // Slow-path call |
| 1635 | int size = TypeFunc::Parms + 2; |
| 1636 | CallLeafNode *call = new CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(), |
| 1637 | CAST_FROM_FN_PTR(address,((address)((address_word)(static_cast<int (*)(Thread*, oopDesc *)>(SharedRuntime::dtrace_object_alloc)))) |
| 1638 | static_cast<int (*)(Thread*, oopDesc*)>(SharedRuntime::dtrace_object_alloc))((address)((address_word)(static_cast<int (*)(Thread*, oopDesc *)>(SharedRuntime::dtrace_object_alloc)))), |
| 1639 | "dtrace_object_alloc", |
| 1640 | TypeRawPtr::BOTTOM); |
| 1641 | |
| 1642 | // Get base of thread-local storage area |
| 1643 | Node* thread = new ThreadLocalNode(); |
| 1644 | transform_later(thread); |
| 1645 | |
| 1646 | call->init_req(TypeFunc::Parms + 0, thread); |
| 1647 | call->init_req(TypeFunc::Parms + 1, oop); |
| 1648 | call->init_req(TypeFunc::Control, ctrl); |
| 1649 | call->init_req(TypeFunc::I_O , top()); // does no i/o |
| 1650 | call->init_req(TypeFunc::Memory , rawmem); |
| 1651 | call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr)); |
| 1652 | call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr)); |
| 1653 | transform_later(call); |
| 1654 | ctrl = new ProjNode(call, TypeFunc::Control); |
| 1655 | transform_later(ctrl); |
| 1656 | rawmem = new ProjNode(call, TypeFunc::Memory); |
| 1657 | transform_later(rawmem); |
| 1658 | } |
| 1659 | } |
| 1660 | |
| 1661 | // Helper for PhaseMacroExpand::expand_allocate_common. |
| 1662 | // Initializes the newly-allocated storage. |
| 1663 | Node* |
| 1664 | PhaseMacroExpand::initialize_object(AllocateNode* alloc, |
| 1665 | Node* control, Node* rawmem, Node* object, |
| 1666 | Node* klass_node, Node* length, |
| 1667 | Node* size_in_bytes) { |
| 1668 | InitializeNode* init = alloc->initialization(); |
| 1669 | // Store the klass & mark bits |
| 1670 | Node* mark_node = alloc->make_ideal_mark(&_igvn, object, control, rawmem); |
| 1671 | if (!mark_node->is_Con()) { |
| 1672 | transform_later(mark_node); |
| 1673 | } |
| 1674 | rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_XTypeLong::LONG->basic_type()); |
| 1675 | |
| 1676 | rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA); |
| 1677 | int header_size = alloc->minimum_header_size(); // conservatively small |
| 1678 | |
| 1679 | // Array length |
| 1680 | if (length != NULL__null) { // Arrays need length field |
| 1681 | rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT); |
| 1682 | // conservatively small header size: |
| 1683 | header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE); |
| 1684 | ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); |
| 1685 | if (k->is_array_klass()) // we know the exact header size in most cases: |
| 1686 | header_size = Klass::layout_helper_header_size(k->layout_helper()); |
| 1687 | } |
| 1688 | |
| 1689 | // Clear the object body, if necessary. |
| 1690 | if (init == NULL__null) { |
| 1691 | // The init has somehow disappeared; be cautious and clear everything. |
| 1692 | // |
| 1693 | // This can happen if a node is allocated but an uncommon trap occurs |
| 1694 | // immediately. In this case, the Initialize gets associated with the |
| 1695 | // trap, and may be placed in a different (outer) loop, if the Allocate |
| 1696 | // is in a loop. If (this is rare) the inner loop gets unrolled, then |
| 1697 | // there can be two Allocates to one Initialize. The answer in all these |
| 1698 | // edge cases is safety first. It is always safe to clear immediately |
| 1699 | // within an Allocate, and then (maybe or maybe not) clear some more later. |
| 1700 | if (!(UseTLAB && ZeroTLAB)) { |
| 1701 | rawmem = ClearArrayNode::clear_memory(control, rawmem, object, |
| 1702 | header_size, size_in_bytes, |
| 1703 | &_igvn); |
| 1704 | } |
| 1705 | } else { |
| 1706 | if (!init->is_complete()) { |
| 1707 | // Try to win by zeroing only what the init does not store. |
| 1708 | // We can also try to do some peephole optimizations, |
| 1709 | // such as combining some adjacent subword stores. |
| 1710 | rawmem = init->complete_stores(control, rawmem, object, |
| 1711 | header_size, size_in_bytes, &_igvn); |
| 1712 | } |
| 1713 | // We have no more use for this link, since the AllocateNode goes away: |
| 1714 | init->set_req(InitializeNode::RawAddress, top()); |
| 1715 | // (If we keep the link, it just confuses the register allocator, |
| 1716 | // who thinks he sees a real use of the address by the membar.) |
| 1717 | } |
| 1718 | |
| 1719 | return rawmem; |
| 1720 | } |
| 1721 | |
| 1722 | // Generate prefetch instructions for next allocations. |
| 1723 | Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false, |
| 1724 | Node*& contended_phi_rawmem, |
| 1725 | Node* old_eden_top, Node* new_eden_top, |
| 1726 | intx lines) { |
| 1727 | enum { fall_in_path = 1, pf_path = 2 }; |
| 1728 | if( UseTLAB && AllocatePrefetchStyle == 2 ) { |
| 1729 | // Generate prefetch allocation with watermark check. |
| 1730 | // As an allocation hits the watermark, we will prefetch starting |
| 1731 | // at a "distance" away from watermark. |
| 1732 | |
| 1733 | Node *pf_region = new RegionNode(3); |
| 1734 | Node *pf_phi_rawmem = new PhiNode( pf_region, Type::MEMORY, |
| 1735 | TypeRawPtr::BOTTOM ); |
| 1736 | // I/O is used for Prefetch |
| 1737 | Node *pf_phi_abio = new PhiNode( pf_region, Type::ABIO ); |
| 1738 | |
| 1739 | Node *thread = new ThreadLocalNode(); |
| 1740 | transform_later(thread); |
| 1741 | |
| 1742 | Node *eden_pf_adr = new AddPNode( top()/*not oop*/, thread, |
| 1743 | _igvn.MakeConXlongcon(in_bytes(JavaThread::tlab_pf_top_offset())) ); |
| 1744 | transform_later(eden_pf_adr); |
| 1745 | |
| 1746 | Node *old_pf_wm = new LoadPNode(needgc_false, |
| 1747 | contended_phi_rawmem, eden_pf_adr, |
| 1748 | TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, |
| 1749 | MemNode::unordered); |
| 1750 | transform_later(old_pf_wm); |
| 1751 | |
| 1752 | // check against new_eden_top |
| 1753 | Node *need_pf_cmp = new CmpPNode( new_eden_top, old_pf_wm ); |
| 1754 | transform_later(need_pf_cmp); |
| 1755 | Node *need_pf_bol = new BoolNode( need_pf_cmp, BoolTest::ge ); |
| 1756 | transform_later(need_pf_bol); |
| 1757 | IfNode *need_pf_iff = new IfNode( needgc_false, need_pf_bol, |
| 1758 | PROB_UNLIKELY_MAG(4)(1e-4f), COUNT_UNKNOWN(-1.0f) ); |
| 1759 | transform_later(need_pf_iff); |
| 1760 | |
| 1761 | // true node, add prefetchdistance |
| 1762 | Node *need_pf_true = new IfTrueNode( need_pf_iff ); |
| 1763 | transform_later(need_pf_true); |
| 1764 | |
| 1765 | Node *need_pf_false = new IfFalseNode( need_pf_iff ); |
| 1766 | transform_later(need_pf_false); |
| 1767 | |
| 1768 | Node *new_pf_wmt = new AddPNode( top(), old_pf_wm, |
| 1769 | _igvn.MakeConXlongcon(AllocatePrefetchDistance) ); |
| 1770 | transform_later(new_pf_wmt ); |
| 1771 | new_pf_wmt->set_req(0, need_pf_true); |
| 1772 | |
| 1773 | Node *store_new_wmt = new StorePNode(need_pf_true, |
| 1774 | contended_phi_rawmem, eden_pf_adr, |
| 1775 | TypeRawPtr::BOTTOM, new_pf_wmt, |
| 1776 | MemNode::unordered); |
| 1777 | transform_later(store_new_wmt); |
| 1778 | |
| 1779 | // adding prefetches |
| 1780 | pf_phi_abio->init_req( fall_in_path, i_o ); |
| 1781 | |
| 1782 | Node *prefetch_adr; |
| 1783 | Node *prefetch; |
| 1784 | uint step_size = AllocatePrefetchStepSize; |
| 1785 | uint distance = 0; |
| 1786 | |
| 1787 | for ( intx i = 0; i < lines; i++ ) { |
| 1788 | prefetch_adr = new AddPNode( old_pf_wm, new_pf_wmt, |
| 1789 | _igvn.MakeConXlongcon(distance) ); |
| 1790 | transform_later(prefetch_adr); |
| 1791 | prefetch = new PrefetchAllocationNode( i_o, prefetch_adr ); |
| 1792 | transform_later(prefetch); |
| 1793 | distance += step_size; |
| 1794 | i_o = prefetch; |
| 1795 | } |
| 1796 | pf_phi_abio->set_req( pf_path, i_o ); |
| 1797 | |
| 1798 | pf_region->init_req( fall_in_path, need_pf_false ); |
| 1799 | pf_region->init_req( pf_path, need_pf_true ); |
| 1800 | |
| 1801 | pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem ); |
| 1802 | pf_phi_rawmem->init_req( pf_path, store_new_wmt ); |
| 1803 | |
| 1804 | transform_later(pf_region); |
| 1805 | transform_later(pf_phi_rawmem); |
| 1806 | transform_later(pf_phi_abio); |
| 1807 | |
| 1808 | needgc_false = pf_region; |
| 1809 | contended_phi_rawmem = pf_phi_rawmem; |
| 1810 | i_o = pf_phi_abio; |
| 1811 | } else if( UseTLAB && AllocatePrefetchStyle == 3 ) { |
| 1812 | // Insert a prefetch instruction for each allocation. |
| 1813 | // This code is used to generate 1 prefetch instruction per cache line. |
| 1814 | |
| 1815 | // Generate several prefetch instructions. |
| 1816 | uint step_size = AllocatePrefetchStepSize; |
| 1817 | uint distance = AllocatePrefetchDistance; |
| 1818 | |
| 1819 | // Next cache address. |
| 1820 | Node *cache_adr = new AddPNode(old_eden_top, old_eden_top, |
| 1821 | _igvn.MakeConXlongcon(step_size + distance)); |
| 1822 | transform_later(cache_adr); |
| 1823 | cache_adr = new CastP2XNode(needgc_false, cache_adr); |
| 1824 | transform_later(cache_adr); |
| 1825 | // Address is aligned to execute prefetch to the beginning of cache line size |
| 1826 | // (it is important when BIS instruction is used on SPARC as prefetch). |
| 1827 | Node* mask = _igvn.MakeConXlongcon(~(intptr_t)(step_size-1)); |
| 1828 | cache_adr = new AndXNodeAndLNode(cache_adr, mask); |
| 1829 | transform_later(cache_adr); |
| 1830 | cache_adr = new CastX2PNode(cache_adr); |
| 1831 | transform_later(cache_adr); |
| 1832 | |
| 1833 | // Prefetch |
| 1834 | Node *prefetch = new PrefetchAllocationNode( contended_phi_rawmem, cache_adr ); |
| 1835 | prefetch->set_req(0, needgc_false); |
| 1836 | transform_later(prefetch); |
| 1837 | contended_phi_rawmem = prefetch; |
| 1838 | Node *prefetch_adr; |
| 1839 | distance = step_size; |
| 1840 | for ( intx i = 1; i < lines; i++ ) { |
| 1841 | prefetch_adr = new AddPNode( cache_adr, cache_adr, |
| 1842 | _igvn.MakeConXlongcon(distance) ); |
| 1843 | transform_later(prefetch_adr); |
| 1844 | prefetch = new PrefetchAllocationNode( contended_phi_rawmem, prefetch_adr ); |
| 1845 | transform_later(prefetch); |
| 1846 | distance += step_size; |
| 1847 | contended_phi_rawmem = prefetch; |
| 1848 | } |
| 1849 | } else if( AllocatePrefetchStyle > 0 ) { |
| 1850 | // Insert a prefetch for each allocation only on the fast-path |
| 1851 | Node *prefetch_adr; |
| 1852 | Node *prefetch; |
| 1853 | // Generate several prefetch instructions. |
| 1854 | uint step_size = AllocatePrefetchStepSize; |
| 1855 | uint distance = AllocatePrefetchDistance; |
| 1856 | for ( intx i = 0; i < lines; i++ ) { |
| 1857 | prefetch_adr = new AddPNode( old_eden_top, new_eden_top, |
| 1858 | _igvn.MakeConXlongcon(distance) ); |
| 1859 | transform_later(prefetch_adr); |
| 1860 | prefetch = new PrefetchAllocationNode( i_o, prefetch_adr ); |
| 1861 | // Do not let it float too high, since if eden_top == eden_end, |
| 1862 | // both might be null. |
| 1863 | if( i == 0 ) { // Set control for first prefetch, next follows it |
| 1864 | prefetch->init_req(0, needgc_false); |
| 1865 | } |
| 1866 | transform_later(prefetch); |
| 1867 | distance += step_size; |
| 1868 | i_o = prefetch; |
| 1869 | } |
| 1870 | } |
| 1871 | return i_o; |
| 1872 | } |
| 1873 | |
| 1874 | |
| 1875 | void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { |
| 1876 | expand_allocate_common(alloc, NULL__null, |
| 1877 | OptoRuntime::new_instance_Type(), |
| 1878 | OptoRuntime::new_instance_Java()); |
| 1879 | } |
| 1880 | |
| 1881 | void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { |
| 1882 | Node* length = alloc->in(AllocateNode::ALength); |
| 1883 | InitializeNode* init = alloc->initialization(); |
| 1884 | Node* klass_node = alloc->in(AllocateNode::KlassNode); |
| 1885 | ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); |
| 1886 | address slow_call_address; // Address of slow call |
| 1887 | if (init != NULL__null && init->is_complete_with_arraycopy() && |
| 1888 | k->is_type_array_klass()) { |
| 1889 | // Don't zero type array during slow allocation in VM since |
| 1890 | // it will be initialized later by arraycopy in compiled code. |
| 1891 | slow_call_address = OptoRuntime::new_array_nozero_Java(); |
| 1892 | } else { |
| 1893 | slow_call_address = OptoRuntime::new_array_Java(); |
| 1894 | } |
| 1895 | expand_allocate_common(alloc, length, |
| 1896 | OptoRuntime::new_array_Type(), |
| 1897 | slow_call_address); |
| 1898 | } |
| 1899 | |
| 1900 | //-------------------mark_eliminated_box---------------------------------- |
| 1901 | // |
| 1902 | // During EA obj may point to several objects but after few ideal graph |
| 1903 | // transformations (CCP) it may point to only one non escaping object |
| 1904 | // (but still using phi), corresponding locks and unlocks will be marked |
| 1905 | // for elimination. Later obj could be replaced with a new node (new phi) |
| 1906 | // and which does not have escape information. And later after some graph |
| 1907 | // reshape other locks and unlocks (which were not marked for elimination |
| 1908 | // before) are connected to this new obj (phi) but they still will not be |
| 1909 | // marked for elimination since new obj has no escape information. |
| 1910 | // Mark all associated (same box and obj) lock and unlock nodes for |
| 1911 | // elimination if some of them marked already. |
| 1912 | void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) { |
| 1913 | if (oldbox->as_BoxLock()->is_eliminated()) { |
| 1914 | return; // This BoxLock node was processed already. |
| 1915 | } |
| 1916 | // New implementation (EliminateNestedLocks) has separate BoxLock |
| 1917 | // node for each locked region so mark all associated locks/unlocks as |
| 1918 | // eliminated even if different objects are referenced in one locked region |
| 1919 | // (for example, OSR compilation of nested loop inside locked scope). |
| 1920 | if (EliminateNestedLocks || |
| 1921 | oldbox->as_BoxLock()->is_simple_lock_region(NULL__null, obj, NULL__null)) { |
| 1922 | // Box is used only in one lock region. Mark this box as eliminated. |
| 1923 | _igvn.hash_delete(oldbox); |
| 1924 | oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value |
| 1925 | _igvn.hash_insert(oldbox); |
| 1926 | |
| 1927 | for (uint i = 0; i < oldbox->outcnt(); i++) { |
| 1928 | Node* u = oldbox->raw_out(i); |
| 1929 | if (u->is_AbstractLock() && !u->as_AbstractLock()->is_non_esc_obj()) { |
| 1930 | AbstractLockNode* alock = u->as_AbstractLock(); |
| 1931 | // Check lock's box since box could be referenced by Lock's debug info. |
| 1932 | if (alock->box_node() == oldbox) { |
| 1933 | // Mark eliminated all related locks and unlocks. |
| 1934 | #ifdef ASSERT1 |
| 1935 | alock->log_lock_optimization(C, "eliminate_lock_set_non_esc4"); |
| 1936 | #endif |
| 1937 | alock->set_non_esc_obj(); |
| 1938 | } |
| 1939 | } |
| 1940 | } |
| 1941 | return; |
| 1942 | } |
| 1943 | |
| 1944 | // Create new "eliminated" BoxLock node and use it in monitor debug info |
| 1945 | // instead of oldbox for the same object. |
| 1946 | BoxLockNode* newbox = oldbox->clone()->as_BoxLock(); |
| 1947 | |
| 1948 | // Note: BoxLock node is marked eliminated only here and it is used |
| 1949 | // to indicate that all associated lock and unlock nodes are marked |
| 1950 | // for elimination. |
| 1951 | newbox->set_eliminated(); |
| 1952 | transform_later(newbox); |
| 1953 | |
| 1954 | // Replace old box node with new box for all users of the same object. |
| 1955 | for (uint i = 0; i < oldbox->outcnt();) { |
| 1956 | bool next_edge = true; |
| 1957 | |
| 1958 | Node* u = oldbox->raw_out(i); |
| 1959 | if (u->is_AbstractLock()) { |
| 1960 | AbstractLockNode* alock = u->as_AbstractLock(); |
| 1961 | if (alock->box_node() == oldbox && alock->obj_node()->eqv_uncast(obj)) { |
| 1962 | // Replace Box and mark eliminated all related locks and unlocks. |
| 1963 | #ifdef ASSERT1 |
| 1964 | alock->log_lock_optimization(C, "eliminate_lock_set_non_esc5"); |
| 1965 | #endif |
| 1966 | alock->set_non_esc_obj(); |
| 1967 | _igvn.rehash_node_delayed(alock); |
| 1968 | alock->set_box_node(newbox); |
| 1969 | next_edge = false; |
| 1970 | } |
| 1971 | } |
| 1972 | if (u->is_FastLock() && u->as_FastLock()->obj_node()->eqv_uncast(obj)) { |
| 1973 | FastLockNode* flock = u->as_FastLock(); |
| 1974 | assert(flock->box_node() == oldbox, "sanity")do { if (!(flock->box_node() == oldbox)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 1974, "assert(" "flock->box_node() == oldbox" ") failed" , "sanity"); ::breakpoint(); } } while (0); |
| 1975 | _igvn.rehash_node_delayed(flock); |
| 1976 | flock->set_box_node(newbox); |
| 1977 | next_edge = false; |
| 1978 | } |
| 1979 | |
| 1980 | // Replace old box in monitor debug info. |
| 1981 | if (u->is_SafePoint() && u->as_SafePoint()->jvms()) { |
| 1982 | SafePointNode* sfn = u->as_SafePoint(); |
| 1983 | JVMState* youngest_jvms = sfn->jvms(); |
| 1984 | int max_depth = youngest_jvms->depth(); |
| 1985 | for (int depth = 1; depth <= max_depth; depth++) { |
| 1986 | JVMState* jvms = youngest_jvms->of_depth(depth); |
| 1987 | int num_mon = jvms->nof_monitors(); |
| 1988 | // Loop over monitors |
| 1989 | for (int idx = 0; idx < num_mon; idx++) { |
| 1990 | Node* obj_node = sfn->monitor_obj(jvms, idx); |
| 1991 | Node* box_node = sfn->monitor_box(jvms, idx); |
| 1992 | if (box_node == oldbox && obj_node->eqv_uncast(obj)) { |
| 1993 | int j = jvms->monitor_box_offset(idx); |
| 1994 | _igvn.replace_input_of(u, j, newbox); |
| 1995 | next_edge = false; |
| 1996 | } |
| 1997 | } |
| 1998 | } |
| 1999 | } |
| 2000 | if (next_edge) i++; |
| 2001 | } |
| 2002 | } |
| 2003 | |
| 2004 | //-----------------------mark_eliminated_locking_nodes----------------------- |
| 2005 | void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) { |
| 2006 | if (EliminateNestedLocks) { |
| 2007 | if (alock->is_nested()) { |
| 2008 | assert(alock->box_node()->as_BoxLock()->is_eliminated(), "sanity")do { if (!(alock->box_node()->as_BoxLock()->is_eliminated ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2008, "assert(" "alock->box_node()->as_BoxLock()->is_eliminated()" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 2009 | return; |
| 2010 | } else if (!alock->is_non_esc_obj()) { // Not eliminated or coarsened |
| 2011 | // Only Lock node has JVMState needed here. |
| 2012 | // Not that preceding claim is documented anywhere else. |
| 2013 | if (alock->jvms() != NULL__null) { |
| 2014 | if (alock->as_Lock()->is_nested_lock_region()) { |
| 2015 | // Mark eliminated related nested locks and unlocks. |
| 2016 | Node* obj = alock->obj_node(); |
| 2017 | BoxLockNode* box_node = alock->box_node()->as_BoxLock(); |
| 2018 | assert(!box_node->is_eliminated(), "should not be marked yet")do { if (!(!box_node->is_eliminated())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2018, "assert(" "!box_node->is_eliminated()" ") failed", "should not be marked yet"); ::breakpoint(); } } while (0); |
| 2019 | // Note: BoxLock node is marked eliminated only here |
| 2020 | // and it is used to indicate that all associated lock |
| 2021 | // and unlock nodes are marked for elimination. |
| 2022 | box_node->set_eliminated(); // Box's hash is always NO_HASH here |
| 2023 | for (uint i = 0; i < box_node->outcnt(); i++) { |
| 2024 | Node* u = box_node->raw_out(i); |
| 2025 | if (u->is_AbstractLock()) { |
| 2026 | alock = u->as_AbstractLock(); |
| 2027 | if (alock->box_node() == box_node) { |
| 2028 | // Verify that this Box is referenced only by related locks. |
| 2029 | assert(alock->obj_node()->eqv_uncast(obj), "")do { if (!(alock->obj_node()->eqv_uncast(obj))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2029, "assert(" "alock->obj_node()->eqv_uncast(obj)" ") failed" , ""); ::breakpoint(); } } while (0); |
| 2030 | // Mark all related locks and unlocks. |
| 2031 | #ifdef ASSERT1 |
| 2032 | alock->log_lock_optimization(C, "eliminate_lock_set_nested"); |
| 2033 | #endif |
| 2034 | alock->set_nested(); |
| 2035 | } |
| 2036 | } |
| 2037 | } |
| 2038 | } else { |
| 2039 | #ifdef ASSERT1 |
| 2040 | alock->log_lock_optimization(C, "eliminate_lock_NOT_nested_lock_region"); |
| 2041 | if (C->log() != NULL__null) |
| 2042 | alock->as_Lock()->is_nested_lock_region(C); // rerun for debugging output |
| 2043 | #endif |
| 2044 | } |
| 2045 | } |
| 2046 | return; |
| 2047 | } |
| 2048 | // Process locks for non escaping object |
| 2049 | assert(alock->is_non_esc_obj(), "")do { if (!(alock->is_non_esc_obj())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2049, "assert(" "alock->is_non_esc_obj()" ") failed", "" ); ::breakpoint(); } } while (0); |
| 2050 | } // EliminateNestedLocks |
| 2051 | |
| 2052 | if (alock->is_non_esc_obj()) { // Lock is used for non escaping object |
| 2053 | // Look for all locks of this object and mark them and |
| 2054 | // corresponding BoxLock nodes as eliminated. |
| 2055 | Node* obj = alock->obj_node(); |
| 2056 | for (uint j = 0; j < obj->outcnt(); j++) { |
| 2057 | Node* o = obj->raw_out(j); |
| 2058 | if (o->is_AbstractLock() && |
| 2059 | o->as_AbstractLock()->obj_node()->eqv_uncast(obj)) { |
| 2060 | alock = o->as_AbstractLock(); |
| 2061 | Node* box = alock->box_node(); |
| 2062 | // Replace old box node with new eliminated box for all users |
| 2063 | // of the same object and mark related locks as eliminated. |
| 2064 | mark_eliminated_box(box, obj); |
| 2065 | } |
| 2066 | } |
| 2067 | } |
| 2068 | } |
| 2069 | |
| 2070 | // we have determined that this lock/unlock can be eliminated, we simply |
| 2071 | // eliminate the node without expanding it. |
| 2072 | // |
| 2073 | // Note: The membar's associated with the lock/unlock are currently not |
| 2074 | // eliminated. This should be investigated as a future enhancement. |
| 2075 | // |
| 2076 | bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) { |
| 2077 | |
| 2078 | if (!alock->is_eliminated()) { |
| 2079 | return false; |
| 2080 | } |
| 2081 | #ifdef ASSERT1 |
| 2082 | if (!alock->is_coarsened()) { |
| 2083 | // Check that new "eliminated" BoxLock node is created. |
| 2084 | BoxLockNode* oldbox = alock->box_node()->as_BoxLock(); |
| 2085 | assert(oldbox->is_eliminated(), "should be done already")do { if (!(oldbox->is_eliminated())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2085, "assert(" "oldbox->is_eliminated()" ") failed", "should be done already" ); ::breakpoint(); } } while (0); |
| 2086 | } |
| 2087 | #endif |
| 2088 | |
| 2089 | alock->log_lock_optimization(C, "eliminate_lock"); |
| 2090 | |
| 2091 | #ifndef PRODUCT |
| 2092 | if (PrintEliminateLocks) { |
| 2093 | tty->print_cr("++++ Eliminated: %d %s '%s'", alock->_idx, (alock->is_Lock() ? "Lock" : "Unlock"), alock->kind_as_string()); |
| 2094 | } |
| 2095 | #endif |
| 2096 | |
| 2097 | Node* mem = alock->in(TypeFunc::Memory); |
| 2098 | Node* ctrl = alock->in(TypeFunc::Control); |
| 2099 | guarantee(ctrl != NULL, "missing control projection, cannot replace_node() with NULL")do { if (!(ctrl != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2099, "guarantee(" "ctrl != NULL" ") failed", "missing control projection, cannot replace_node() with NULL" ); ::breakpoint(); } } while (0); |
| 2100 | |
| 2101 | alock->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 2102 | // There are 2 projections from the lock. The lock node will |
| 2103 | // be deleted when its last use is subsumed below. |
| 2104 | assert(alock->outcnt() == 2 &&do { if (!(alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2107, "assert(" "alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null" ") failed", "Unexpected projections from Lock/Unlock"); ::breakpoint (); } } while (0) |
| 2105 | _callprojs.fallthrough_proj != NULL &&do { if (!(alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2107, "assert(" "alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null" ") failed", "Unexpected projections from Lock/Unlock"); ::breakpoint (); } } while (0) |
| 2106 | _callprojs.fallthrough_memproj != NULL,do { if (!(alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2107, "assert(" "alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null" ") failed", "Unexpected projections from Lock/Unlock"); ::breakpoint (); } } while (0) |
| 2107 | "Unexpected projections from Lock/Unlock")do { if (!(alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2107, "assert(" "alock->outcnt() == 2 && _callprojs.fallthrough_proj != __null && _callprojs.fallthrough_memproj != __null" ") failed", "Unexpected projections from Lock/Unlock"); ::breakpoint (); } } while (0); |
| 2108 | |
| 2109 | Node* fallthroughproj = _callprojs.fallthrough_proj; |
| 2110 | Node* memproj_fallthrough = _callprojs.fallthrough_memproj; |
| 2111 | |
| 2112 | // The memory projection from a lock/unlock is RawMem |
| 2113 | // The input to a Lock is merged memory, so extract its RawMem input |
| 2114 | // (unless the MergeMem has been optimized away.) |
| 2115 | if (alock->is_Lock()) { |
| 2116 | // Seach for MemBarAcquireLock node and delete it also. |
| 2117 | MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar(); |
| 2118 | assert(membar != NULL && membar->Opcode() == Op_MemBarAcquireLock, "")do { if (!(membar != __null && membar->Opcode() == Op_MemBarAcquireLock)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2118, "assert(" "membar != __null && membar->Opcode() == Op_MemBarAcquireLock" ") failed", ""); ::breakpoint(); } } while (0); |
| 2119 | Node* ctrlproj = membar->proj_out(TypeFunc::Control); |
| 2120 | Node* memproj = membar->proj_out(TypeFunc::Memory); |
| 2121 | _igvn.replace_node(ctrlproj, fallthroughproj); |
| 2122 | _igvn.replace_node(memproj, memproj_fallthrough); |
| 2123 | |
| 2124 | // Delete FastLock node also if this Lock node is unique user |
| 2125 | // (a loop peeling may clone a Lock node). |
| 2126 | Node* flock = alock->as_Lock()->fastlock_node(); |
| 2127 | if (flock->outcnt() == 1) { |
| 2128 | assert(flock->unique_out() == alock, "sanity")do { if (!(flock->unique_out() == alock)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2128, "assert(" "flock->unique_out() == alock" ") failed" , "sanity"); ::breakpoint(); } } while (0); |
| 2129 | _igvn.replace_node(flock, top()); |
| 2130 | } |
| 2131 | } |
| 2132 | |
| 2133 | // Seach for MemBarReleaseLock node and delete it also. |
| 2134 | if (alock->is_Unlock() && ctrl->is_Proj() && ctrl->in(0)->is_MemBar()) { |
| 2135 | MemBarNode* membar = ctrl->in(0)->as_MemBar(); |
| 2136 | assert(membar->Opcode() == Op_MemBarReleaseLock &&do { if (!(membar->Opcode() == Op_MemBarReleaseLock && mem->is_Proj() && membar == mem->in(0))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2137, "assert(" "membar->Opcode() == Op_MemBarReleaseLock && mem->is_Proj() && membar == mem->in(0)" ") failed", ""); ::breakpoint(); } } while (0) |
| 2137 | mem->is_Proj() && membar == mem->in(0), "")do { if (!(membar->Opcode() == Op_MemBarReleaseLock && mem->is_Proj() && membar == mem->in(0))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2137, "assert(" "membar->Opcode() == Op_MemBarReleaseLock && mem->is_Proj() && membar == mem->in(0)" ") failed", ""); ::breakpoint(); } } while (0); |
| 2138 | _igvn.replace_node(fallthroughproj, ctrl); |
| 2139 | _igvn.replace_node(memproj_fallthrough, mem); |
| 2140 | fallthroughproj = ctrl; |
| 2141 | memproj_fallthrough = mem; |
| 2142 | ctrl = membar->in(TypeFunc::Control); |
| 2143 | mem = membar->in(TypeFunc::Memory); |
| 2144 | } |
| 2145 | |
| 2146 | _igvn.replace_node(fallthroughproj, ctrl); |
| 2147 | _igvn.replace_node(memproj_fallthrough, mem); |
| 2148 | return true; |
| 2149 | } |
| 2150 | |
| 2151 | |
| 2152 | //------------------------------expand_lock_node---------------------- |
| 2153 | void PhaseMacroExpand::expand_lock_node(LockNode *lock) { |
| 2154 | |
| 2155 | Node* ctrl = lock->in(TypeFunc::Control); |
| 2156 | Node* mem = lock->in(TypeFunc::Memory); |
| 2157 | Node* obj = lock->obj_node(); |
| 2158 | Node* box = lock->box_node(); |
| 2159 | Node* flock = lock->fastlock_node(); |
| 2160 | |
| 2161 | assert(!box->as_BoxLock()->is_eliminated(), "sanity")do { if (!(!box->as_BoxLock()->is_eliminated())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2161, "assert(" "!box->as_BoxLock()->is_eliminated()" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 2162 | |
| 2163 | // Make the merge point |
| 2164 | Node *region; |
| 2165 | Node *mem_phi; |
| 2166 | Node *slow_path; |
| 2167 | |
| 2168 | region = new RegionNode(3); |
| 2169 | // create a Phi for the memory state |
| 2170 | mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); |
| 2171 | |
| 2172 | // Optimize test; set region slot 2 |
| 2173 | slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0); |
| 2174 | mem_phi->init_req(2, mem); |
| 2175 | |
| 2176 | // Make slow path call |
| 2177 | CallNode *call = make_slow_call((CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), |
| 2178 | OptoRuntime::complete_monitor_locking_Java(), NULL__null, slow_path, |
| 2179 | obj, box, NULL__null); |
| 2180 | |
| 2181 | call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 2182 | |
| 2183 | // Slow path can only throw asynchronous exceptions, which are always |
| 2184 | // de-opted. So the compiler thinks the slow-call can never throw an |
| 2185 | // exception. If it DOES throw an exception we would need the debug |
| 2186 | // info removed first (since if it throws there is no monitor). |
| 2187 | assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL &&do { if (!(_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2188, "assert(" "_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null" ") failed", "Unexpected projection from Lock"); ::breakpoint (); } } while (0) |
| 2188 | _callprojs.catchall_memproj == NULL && _callprojs.catchall_catchproj == NULL, "Unexpected projection from Lock")do { if (!(_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2188, "assert(" "_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null" ") failed", "Unexpected projection from Lock"); ::breakpoint (); } } while (0); |
| 2189 | |
| 2190 | // Capture slow path |
| 2191 | // disconnect fall-through projection from call and create a new one |
| 2192 | // hook up users of fall-through projection to region |
| 2193 | Node *slow_ctrl = _callprojs.fallthrough_proj->clone(); |
| 2194 | transform_later(slow_ctrl); |
| 2195 | _igvn.hash_delete(_callprojs.fallthrough_proj); |
| 2196 | _callprojs.fallthrough_proj->disconnect_inputs(C); |
| 2197 | region->init_req(1, slow_ctrl); |
| 2198 | // region inputs are now complete |
| 2199 | transform_later(region); |
| 2200 | _igvn.replace_node(_callprojs.fallthrough_proj, region); |
| 2201 | |
| 2202 | Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory)); |
| 2203 | mem_phi->init_req(1, memproj ); |
| 2204 | transform_later(mem_phi); |
| 2205 | _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi); |
| 2206 | } |
| 2207 | |
| 2208 | //------------------------------expand_unlock_node---------------------- |
| 2209 | void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { |
| 2210 | |
| 2211 | Node* ctrl = unlock->in(TypeFunc::Control); |
| 2212 | Node* mem = unlock->in(TypeFunc::Memory); |
| 2213 | Node* obj = unlock->obj_node(); |
| 2214 | Node* box = unlock->box_node(); |
| 2215 | |
| 2216 | assert(!box->as_BoxLock()->is_eliminated(), "sanity")do { if (!(!box->as_BoxLock()->is_eliminated())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2216, "assert(" "!box->as_BoxLock()->is_eliminated()" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 2217 | |
| 2218 | // No need for a null check on unlock |
| 2219 | |
| 2220 | // Make the merge point |
| 2221 | Node *region; |
| 2222 | Node *mem_phi; |
| 2223 | |
| 2224 | region = new RegionNode(3); |
| 2225 | // create a Phi for the memory state |
| 2226 | mem_phi = new PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); |
| 2227 | |
| 2228 | FastUnlockNode *funlock = new FastUnlockNode( ctrl, obj, box ); |
| 2229 | funlock = transform_later( funlock )->as_FastUnlock(); |
| 2230 | // Optimize test; set region slot 2 |
| 2231 | Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0); |
| 2232 | Node *thread = transform_later(new ThreadLocalNode()); |
| 2233 | |
| 2234 | CallNode *call = make_slow_call((CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), |
| 2235 | CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C)((address)((address_word)(SharedRuntime::complete_monitor_unlocking_C ))), |
| 2236 | "complete_monitor_unlocking_C", slow_path, obj, box, thread); |
| 2237 | |
| 2238 | call->extract_projections(&_callprojs, false /*separate_io_proj*/, false /*do_asserts*/); |
| 2239 | assert(_callprojs.fallthrough_ioproj == NULL && _callprojs.catchall_ioproj == NULL &&do { if (!(_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2240, "assert(" "_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null" ") failed", "Unexpected projection from Lock"); ::breakpoint (); } } while (0) |
| 2240 | _callprojs.catchall_memproj == NULL && _callprojs.catchall_catchproj == NULL, "Unexpected projection from Lock")do { if (!(_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2240, "assert(" "_callprojs.fallthrough_ioproj == __null && _callprojs.catchall_ioproj == __null && _callprojs.catchall_memproj == __null && _callprojs.catchall_catchproj == __null" ") failed", "Unexpected projection from Lock"); ::breakpoint (); } } while (0); |
| 2241 | |
| 2242 | // No exceptions for unlocking |
| 2243 | // Capture slow path |
| 2244 | // disconnect fall-through projection from call and create a new one |
| 2245 | // hook up users of fall-through projection to region |
| 2246 | Node *slow_ctrl = _callprojs.fallthrough_proj->clone(); |
| 2247 | transform_later(slow_ctrl); |
| 2248 | _igvn.hash_delete(_callprojs.fallthrough_proj); |
| 2249 | _callprojs.fallthrough_proj->disconnect_inputs(C); |
| 2250 | region->init_req(1, slow_ctrl); |
| 2251 | // region inputs are now complete |
| 2252 | transform_later(region); |
| 2253 | _igvn.replace_node(_callprojs.fallthrough_proj, region); |
| 2254 | |
| 2255 | Node *memproj = transform_later(new ProjNode(call, TypeFunc::Memory) ); |
| 2256 | mem_phi->init_req(1, memproj ); |
| 2257 | mem_phi->init_req(2, mem); |
| 2258 | transform_later(mem_phi); |
| 2259 | _igvn.replace_node(_callprojs.fallthrough_memproj, mem_phi); |
| 2260 | } |
| 2261 | |
| 2262 | void PhaseMacroExpand::expand_subtypecheck_node(SubTypeCheckNode *check) { |
| 2263 | assert(check->in(SubTypeCheckNode::Control) == NULL, "should be pinned")do { if (!(check->in(SubTypeCheckNode::Control) == __null) ) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2263, "assert(" "check->in(SubTypeCheckNode::Control) == __null" ") failed", "should be pinned"); ::breakpoint(); } } while ( 0); |
| 2264 | Node* bol = check->unique_out(); |
| 2265 | Node* obj_or_subklass = check->in(SubTypeCheckNode::ObjOrSubKlass); |
| 2266 | Node* superklass = check->in(SubTypeCheckNode::SuperKlass); |
| 2267 | assert(bol->is_Bool() && bol->as_Bool()->_test._test == BoolTest::ne, "unexpected bool node")do { if (!(bol->is_Bool() && bol->as_Bool()-> _test._test == BoolTest::ne)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2267, "assert(" "bol->is_Bool() && bol->as_Bool()->_test._test == BoolTest::ne" ") failed", "unexpected bool node"); ::breakpoint(); } } while (0); |
| 2268 | |
| 2269 | for (DUIterator_Last imin, i = bol->last_outs(imin); i >= imin; --i) { |
| 2270 | Node* iff = bol->last_out(i); |
| 2271 | assert(iff->is_If(), "where's the if?")do { if (!(iff->is_If())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2271, "assert(" "iff->is_If()" ") failed", "where's the if?" ); ::breakpoint(); } } while (0); |
| 2272 | |
| 2273 | if (iff->in(0)->is_top()) { |
| 2274 | _igvn.replace_input_of(iff, 1, C->top()); |
| 2275 | continue; |
| 2276 | } |
| 2277 | |
| 2278 | Node* iftrue = iff->as_If()->proj_out(1); |
| 2279 | Node* iffalse = iff->as_If()->proj_out(0); |
| 2280 | Node* ctrl = iff->in(0); |
| 2281 | |
| 2282 | Node* subklass = NULL__null; |
| 2283 | if (_igvn.type(obj_or_subklass)->isa_klassptr()) { |
| 2284 | subklass = obj_or_subklass; |
| 2285 | } else { |
| 2286 | Node* k_adr = basic_plus_adr(obj_or_subklass, oopDesc::klass_offset_in_bytes()); |
| 2287 | subklass = _igvn.transform(LoadKlassNode::make(_igvn, NULL__null, C->immutable_memory(), k_adr, TypeInstPtr::KLASS)); |
| 2288 | } |
| 2289 | |
| 2290 | Node* not_subtype_ctrl = Phase::gen_subtype_check(subklass, superklass, &ctrl, NULL__null, _igvn); |
| 2291 | |
| 2292 | _igvn.replace_input_of(iff, 0, C->top()); |
| 2293 | _igvn.replace_node(iftrue, not_subtype_ctrl); |
| 2294 | _igvn.replace_node(iffalse, ctrl); |
| 2295 | } |
| 2296 | _igvn.replace_node(check, C->top()); |
| 2297 | } |
| 2298 | |
| 2299 | //---------------------------eliminate_macro_nodes---------------------- |
| 2300 | // Eliminate scalar replaced allocations and associated locks. |
| 2301 | void PhaseMacroExpand::eliminate_macro_nodes() { |
| 2302 | if (C->macro_count() == 0) |
| 2303 | return; |
| 2304 | |
| 2305 | // Before elimination may re-mark (change to Nested or NonEscObj) |
| 2306 | // all associated (same box and obj) lock and unlock nodes. |
| 2307 | int cnt = C->macro_count(); |
| 2308 | for (int i=0; i < cnt; i++) { |
| 2309 | Node *n = C->macro_node(i); |
| 2310 | if (n->is_AbstractLock()) { // Lock and Unlock nodes |
| 2311 | mark_eliminated_locking_nodes(n->as_AbstractLock()); |
| 2312 | } |
| 2313 | } |
| 2314 | // Re-marking may break consistency of Coarsened locks. |
| 2315 | if (!C->coarsened_locks_consistent()) { |
| 2316 | return; // recompile without Coarsened locks if broken |
| 2317 | } |
| 2318 | |
| 2319 | // First, attempt to eliminate locks |
| 2320 | bool progress = true; |
| 2321 | while (progress) { |
| 2322 | progress = false; |
| 2323 | for (int i = C->macro_count(); i > 0; i = MIN2(i - 1, C->macro_count())) { // more than 1 element can be eliminated at once |
| 2324 | Node* n = C->macro_node(i - 1); |
| 2325 | bool success = false; |
| 2326 | DEBUG_ONLY(int old_macro_count = C->macro_count();)int old_macro_count = C->macro_count(); |
| 2327 | if (n->is_AbstractLock()) { |
| 2328 | success = eliminate_locking_node(n->as_AbstractLock()); |
| 2329 | } |
| 2330 | assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count")do { if (!(success == (C->macro_count() < old_macro_count ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2330, "assert(" "success == (C->macro_count() < old_macro_count)" ") failed", "elimination reduces macro count"); ::breakpoint (); } } while (0); |
| 2331 | progress = progress || success; |
| 2332 | } |
| 2333 | } |
| 2334 | // Next, attempt to eliminate allocations |
| 2335 | _has_locks = false; |
| 2336 | progress = true; |
| 2337 | while (progress) { |
| 2338 | progress = false; |
| 2339 | for (int i = C->macro_count(); i > 0; i = MIN2(i - 1, C->macro_count())) { // more than 1 element can be eliminated at once |
| 2340 | Node* n = C->macro_node(i - 1); |
| 2341 | bool success = false; |
| 2342 | DEBUG_ONLY(int old_macro_count = C->macro_count();)int old_macro_count = C->macro_count(); |
| 2343 | switch (n->class_id()) { |
| 2344 | case Node::Class_Allocate: |
| 2345 | case Node::Class_AllocateArray: |
| 2346 | success = eliminate_allocate_node(n->as_Allocate()); |
| 2347 | break; |
| 2348 | case Node::Class_CallStaticJava: |
| 2349 | success = eliminate_boxing_node(n->as_CallStaticJava()); |
| 2350 | break; |
| 2351 | case Node::Class_Lock: |
| 2352 | case Node::Class_Unlock: |
| 2353 | assert(!n->as_AbstractLock()->is_eliminated(), "sanity")do { if (!(!n->as_AbstractLock()->is_eliminated())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2353, "assert(" "!n->as_AbstractLock()->is_eliminated()" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 2354 | _has_locks = true; |
| 2355 | break; |
| 2356 | case Node::Class_ArrayCopy: |
| 2357 | break; |
| 2358 | case Node::Class_OuterStripMinedLoop: |
| 2359 | break; |
| 2360 | case Node::Class_SubTypeCheck: |
| 2361 | break; |
| 2362 | case Node::Class_Opaque1: |
| 2363 | break; |
| 2364 | default: |
| 2365 | assert(n->Opcode() == Op_LoopLimit ||do { if (!(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2 ()->is_gc_barrier_node(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2370, "assert(" "n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n)" ") failed", "unknown node type in macro list"); ::breakpoint (); } } while (0) |
| 2366 | n->Opcode() == Op_Opaque2 ||do { if (!(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2 ()->is_gc_barrier_node(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2370, "assert(" "n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n)" ") failed", "unknown node type in macro list"); ::breakpoint (); } } while (0) |
| 2367 | n->Opcode() == Op_Opaque3 ||do { if (!(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2 ()->is_gc_barrier_node(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2370, "assert(" "n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n)" ") failed", "unknown node type in macro list"); ::breakpoint (); } } while (0) |
| 2368 | n->Opcode() == Op_Opaque4 ||do { if (!(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2 ()->is_gc_barrier_node(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2370, "assert(" "n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n)" ") failed", "unknown node type in macro list"); ::breakpoint (); } } while (0) |
| 2369 | BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n),do { if (!(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2 ()->is_gc_barrier_node(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2370, "assert(" "n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n)" ") failed", "unknown node type in macro list"); ::breakpoint (); } } while (0) |
| 2370 | "unknown node type in macro list")do { if (!(n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2 ()->is_gc_barrier_node(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2370, "assert(" "n->Opcode() == Op_LoopLimit || n->Opcode() == Op_Opaque2 || n->Opcode() == Op_Opaque3 || n->Opcode() == Op_Opaque4 || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(n)" ") failed", "unknown node type in macro list"); ::breakpoint (); } } while (0); |
| 2371 | } |
| 2372 | assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count")do { if (!(success == (C->macro_count() < old_macro_count ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2372, "assert(" "success == (C->macro_count() < old_macro_count)" ") failed", "elimination reduces macro count"); ::breakpoint (); } } while (0); |
| 2373 | progress = progress || success; |
| 2374 | } |
| 2375 | } |
| 2376 | } |
| 2377 | |
| 2378 | //------------------------------expand_macro_nodes---------------------- |
| 2379 | // Returns true if a failure occurred. |
| 2380 | bool PhaseMacroExpand::expand_macro_nodes() { |
| 2381 | // Last attempt to eliminate macro nodes. |
| 2382 | eliminate_macro_nodes(); |
| 2383 | if (C->failing()) return true; |
| 2384 | |
| 2385 | // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations. |
| 2386 | bool progress = true; |
| 2387 | while (progress) { |
| 2388 | progress = false; |
| 2389 | for (int i = C->macro_count(); i > 0; i--) { |
| 2390 | Node* n = C->macro_node(i-1); |
| 2391 | bool success = false; |
| 2392 | DEBUG_ONLY(int old_macro_count = C->macro_count();)int old_macro_count = C->macro_count(); |
| 2393 | if (n->Opcode() == Op_LoopLimit) { |
| 2394 | // Remove it from macro list and put on IGVN worklist to optimize. |
| 2395 | C->remove_macro_node(n); |
| 2396 | _igvn._worklist.push(n); |
| 2397 | success = true; |
| 2398 | } else if (n->Opcode() == Op_CallStaticJava) { |
| 2399 | // Remove it from macro list and put on IGVN worklist to optimize. |
| 2400 | C->remove_macro_node(n); |
| 2401 | _igvn._worklist.push(n); |
| 2402 | success = true; |
| 2403 | } else if (n->is_Opaque1() || n->Opcode() == Op_Opaque2) { |
| 2404 | _igvn.replace_node(n, n->in(1)); |
| 2405 | success = true; |
| 2406 | #if INCLUDE_RTM_OPT1 |
| 2407 | } else if ((n->Opcode() == Op_Opaque3) && ((Opaque3Node*)n)->rtm_opt()) { |
| 2408 | assert(C->profile_rtm(), "should be used only in rtm deoptimization code")do { if (!(C->profile_rtm())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2408, "assert(" "C->profile_rtm()" ") failed", "should be used only in rtm deoptimization code" ); ::breakpoint(); } } while (0); |
| 2409 | assert((n->outcnt() == 1) && n->unique_out()->is_Cmp(), "")do { if (!((n->outcnt() == 1) && n->unique_out( )->is_Cmp())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2409, "assert(" "(n->outcnt() == 1) && n->unique_out()->is_Cmp()" ") failed", ""); ::breakpoint(); } } while (0); |
| 2410 | Node* cmp = n->unique_out(); |
| 2411 | #ifdef ASSERT1 |
| 2412 | // Validate graph. |
| 2413 | assert((cmp->outcnt() == 1) && cmp->unique_out()->is_Bool(), "")do { if (!((cmp->outcnt() == 1) && cmp->unique_out ()->is_Bool())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2413, "assert(" "(cmp->outcnt() == 1) && cmp->unique_out()->is_Bool()" ") failed", ""); ::breakpoint(); } } while (0); |
| 2414 | BoolNode* bol = cmp->unique_out()->as_Bool(); |
| 2415 | assert((bol->outcnt() == 1) && bol->unique_out()->is_If() &&do { if (!((bol->outcnt() == 1) && bol->unique_out ()->is_If() && (bol->_test._test == BoolTest::ne ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2416, "assert(" "(bol->outcnt() == 1) && bol->unique_out()->is_If() && (bol->_test._test == BoolTest::ne)" ") failed", ""); ::breakpoint(); } } while (0) |
| 2416 | (bol->_test._test == BoolTest::ne), "")do { if (!((bol->outcnt() == 1) && bol->unique_out ()->is_If() && (bol->_test._test == BoolTest::ne ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2416, "assert(" "(bol->outcnt() == 1) && bol->unique_out()->is_If() && (bol->_test._test == BoolTest::ne)" ") failed", ""); ::breakpoint(); } } while (0); |
| 2417 | IfNode* ifn = bol->unique_out()->as_If(); |
| 2418 | assert((ifn->outcnt() == 2) &&do { if (!((ifn->outcnt() == 2) && ifn->proj_out (1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change ) != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2419, "assert(" "(ifn->outcnt() == 2) && ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != __null" ") failed", ""); ::breakpoint(); } } while (0) |
| 2419 | ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != NULL, "")do { if (!((ifn->outcnt() == 2) && ifn->proj_out (1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change ) != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2419, "assert(" "(ifn->outcnt() == 2) && ifn->proj_out(1)->is_uncommon_trap_proj(Deoptimization::Reason_rtm_state_change) != __null" ") failed", ""); ::breakpoint(); } } while (0); |
| 2420 | #endif |
| 2421 | Node* repl = n->in(1); |
| 2422 | if (!_has_locks) { |
| 2423 | // Remove RTM state check if there are no locks in the code. |
| 2424 | // Replace input to compare the same value. |
| 2425 | repl = (cmp->in(1) == n) ? cmp->in(2) : cmp->in(1); |
| 2426 | } |
| 2427 | _igvn.replace_node(n, repl); |
| 2428 | success = true; |
| 2429 | #endif |
| 2430 | } else if (n->Opcode() == Op_Opaque4) { |
| 2431 | // With Opaque4 nodes, the expectation is that the test of input 1 |
| 2432 | // is always equal to the constant value of input 2. So we can |
| 2433 | // remove the Opaque4 and replace it by input 2. In debug builds, |
| 2434 | // leave the non constant test in instead to sanity check that it |
| 2435 | // never fails (if it does, that subgraph was constructed so, at |
| 2436 | // runtime, a Halt node is executed). |
| 2437 | #ifdef ASSERT1 |
| 2438 | _igvn.replace_node(n, n->in(1)); |
| 2439 | #else |
| 2440 | _igvn.replace_node(n, n->in(2)); |
| 2441 | #endif |
| 2442 | success = true; |
| 2443 | } else if (n->Opcode() == Op_OuterStripMinedLoop) { |
| 2444 | n->as_OuterStripMinedLoop()->adjust_strip_mined_loop(&_igvn); |
| 2445 | C->remove_macro_node(n); |
| 2446 | success = true; |
| 2447 | } |
| 2448 | assert(!success || (C->macro_count() == (old_macro_count - 1)), "elimination must have deleted one node from macro list")do { if (!(!success || (C->macro_count() == (old_macro_count - 1)))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2448, "assert(" "!success || (C->macro_count() == (old_macro_count - 1))" ") failed", "elimination must have deleted one node from macro list" ); ::breakpoint(); } } while (0); |
| 2449 | progress = progress || success; |
| 2450 | } |
| 2451 | } |
| 2452 | |
| 2453 | // Clean up the graph so we're less likely to hit the maximum node |
| 2454 | // limit |
| 2455 | _igvn.set_delay_transform(false); |
| 2456 | _igvn.optimize(); |
| 2457 | if (C->failing()) return true; |
| 2458 | _igvn.set_delay_transform(true); |
| 2459 | |
| 2460 | |
| 2461 | // Because we run IGVN after each expansion, some macro nodes may go |
| 2462 | // dead and be removed from the list as we iterate over it. Move |
| 2463 | // Allocate nodes (processed in a second pass) at the beginning of |
| 2464 | // the list and then iterate from the last element of the list until |
| 2465 | // an Allocate node is seen. This is robust to random deletion in |
| 2466 | // the list due to nodes going dead. |
| 2467 | C->sort_macro_nodes(); |
| 2468 | |
| 2469 | // expand arraycopy "macro" nodes first |
| 2470 | // For ReduceBulkZeroing, we must first process all arraycopy nodes |
| 2471 | // before the allocate nodes are expanded. |
| 2472 | while (C->macro_count() > 0) { |
| 2473 | int macro_count = C->macro_count(); |
| 2474 | Node * n = C->macro_node(macro_count-1); |
| 2475 | assert(n->is_macro(), "only macro nodes expected here")do { if (!(n->is_macro())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2475, "assert(" "n->is_macro()" ") failed", "only macro nodes expected here" ); ::breakpoint(); } } while (0); |
| 2476 | if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL__null && n->in(0)->is_top())) { |
| 2477 | // node is unreachable, so don't try to expand it |
| 2478 | C->remove_macro_node(n); |
| 2479 | continue; |
| 2480 | } |
| 2481 | if (n->is_Allocate()) { |
| 2482 | break; |
| 2483 | } |
| 2484 | // Make sure expansion will not cause node limit to be exceeded. |
| 2485 | // Worst case is a macro node gets expanded into about 200 nodes. |
| 2486 | // Allow 50% more for optimization. |
| 2487 | if (C->check_node_count(300, "out of nodes before macro expansion")) { |
| 2488 | return true; |
| 2489 | } |
| 2490 | |
| 2491 | DEBUG_ONLY(int old_macro_count = C->macro_count();)int old_macro_count = C->macro_count(); |
| 2492 | switch (n->class_id()) { |
| 2493 | case Node::Class_Lock: |
| 2494 | expand_lock_node(n->as_Lock()); |
| 2495 | break; |
| 2496 | case Node::Class_Unlock: |
| 2497 | expand_unlock_node(n->as_Unlock()); |
| 2498 | break; |
| 2499 | case Node::Class_ArrayCopy: |
| 2500 | expand_arraycopy_node(n->as_ArrayCopy()); |
| 2501 | break; |
| 2502 | case Node::Class_SubTypeCheck: |
| 2503 | expand_subtypecheck_node(n->as_SubTypeCheck()); |
| 2504 | break; |
| 2505 | default: |
| 2506 | assert(false, "unknown node type in macro list")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2506, "assert(" "false" ") failed", "unknown node type in macro list" ); ::breakpoint(); } } while (0); |
| 2507 | } |
| 2508 | assert(C->macro_count() == (old_macro_count - 1), "expansion must have deleted one node from macro list")do { if (!(C->macro_count() == (old_macro_count - 1))) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2508, "assert(" "C->macro_count() == (old_macro_count - 1)" ") failed", "expansion must have deleted one node from macro list" ); ::breakpoint(); } } while (0); |
| 2509 | if (C->failing()) return true; |
| 2510 | |
| 2511 | // Clean up the graph so we're less likely to hit the maximum node |
| 2512 | // limit |
| 2513 | _igvn.set_delay_transform(false); |
| 2514 | _igvn.optimize(); |
| 2515 | if (C->failing()) return true; |
| 2516 | _igvn.set_delay_transform(true); |
| 2517 | } |
| 2518 | |
| 2519 | // All nodes except Allocate nodes are expanded now. There could be |
| 2520 | // new optimization opportunities (such as folding newly created |
| 2521 | // load from a just allocated object). Run IGVN. |
| 2522 | |
| 2523 | // expand "macro" nodes |
| 2524 | // nodes are removed from the macro list as they are processed |
| 2525 | while (C->macro_count() > 0) { |
| 2526 | int macro_count = C->macro_count(); |
| 2527 | Node * n = C->macro_node(macro_count-1); |
| 2528 | assert(n->is_macro(), "only macro nodes expected here")do { if (!(n->is_macro())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2528, "assert(" "n->is_macro()" ") failed", "only macro nodes expected here" ); ::breakpoint(); } } while (0); |
| 2529 | if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL__null && n->in(0)->is_top())) { |
| 2530 | // node is unreachable, so don't try to expand it |
| 2531 | C->remove_macro_node(n); |
| 2532 | continue; |
| 2533 | } |
| 2534 | // Make sure expansion will not cause node limit to be exceeded. |
| 2535 | // Worst case is a macro node gets expanded into about 200 nodes. |
| 2536 | // Allow 50% more for optimization. |
| 2537 | if (C->check_node_count(300, "out of nodes before macro expansion")) { |
| 2538 | return true; |
| 2539 | } |
| 2540 | switch (n->class_id()) { |
| 2541 | case Node::Class_Allocate: |
| 2542 | expand_allocate(n->as_Allocate()); |
| 2543 | break; |
| 2544 | case Node::Class_AllocateArray: |
| 2545 | expand_allocate_array(n->as_AllocateArray()); |
| 2546 | break; |
| 2547 | default: |
| 2548 | assert(false, "unknown node type in macro list")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2548, "assert(" "false" ") failed", "unknown node type in macro list" ); ::breakpoint(); } } while (0); |
| 2549 | } |
| 2550 | assert(C->macro_count() < macro_count, "must have deleted a node from macro list")do { if (!(C->macro_count() < macro_count)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/macro.cpp" , 2550, "assert(" "C->macro_count() < macro_count" ") failed" , "must have deleted a node from macro list"); ::breakpoint() ; } } while (0); |
| 2551 | if (C->failing()) return true; |
| 2552 | |
| 2553 | // Clean up the graph so we're less likely to hit the maximum node |
| 2554 | // limit |
| 2555 | _igvn.set_delay_transform(false); |
| 2556 | _igvn.optimize(); |
| 2557 | if (C->failing()) return true; |
| 2558 | _igvn.set_delay_transform(true); |
| 2559 | } |
| 2560 | |
| 2561 | _igvn.set_delay_transform(false); |
| 2562 | return false; |
| 2563 | } |