| File: | jdk/src/hotspot/share/opto/loopnode.cpp |
| Warning: | line 577, column 15 Value stored to 'jvms' during its initialization is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* |
| 2 | * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #include "precompiled.hpp" |
| 26 | #include "ci/ciMethodData.hpp" |
| 27 | #include "compiler/compileLog.hpp" |
| 28 | #include "gc/shared/barrierSet.hpp" |
| 29 | #include "gc/shared/c2/barrierSetC2.hpp" |
| 30 | #include "libadt/vectset.hpp" |
| 31 | #include "memory/allocation.inline.hpp" |
| 32 | #include "memory/resourceArea.hpp" |
| 33 | #include "opto/addnode.hpp" |
| 34 | #include "opto/arraycopynode.hpp" |
| 35 | #include "opto/callnode.hpp" |
| 36 | #include "opto/castnode.hpp" |
| 37 | #include "opto/connode.hpp" |
| 38 | #include "opto/convertnode.hpp" |
| 39 | #include "opto/divnode.hpp" |
| 40 | #include "opto/idealGraphPrinter.hpp" |
| 41 | #include "opto/loopnode.hpp" |
| 42 | #include "opto/movenode.hpp" |
| 43 | #include "opto/mulnode.hpp" |
| 44 | #include "opto/opaquenode.hpp" |
| 45 | #include "opto/rootnode.hpp" |
| 46 | #include "opto/runtime.hpp" |
| 47 | #include "opto/superword.hpp" |
| 48 | #include "runtime/sharedRuntime.hpp" |
| 49 | #include "utilities/powerOfTwo.hpp" |
| 50 | |
| 51 | //============================================================================= |
| 52 | //--------------------------is_cloop_ind_var----------------------------------- |
| 53 | // Determine if a node is a counted loop induction variable. |
| 54 | // NOTE: The method is declared in "node.hpp". |
| 55 | bool Node::is_cloop_ind_var() const { |
| 56 | return (is_Phi() && |
| 57 | as_Phi()->region()->is_CountedLoop() && |
| 58 | as_Phi()->region()->as_CountedLoop()->phi() == this); |
| 59 | } |
| 60 | |
| 61 | //============================================================================= |
| 62 | //------------------------------dump_spec-------------------------------------- |
| 63 | // Dump special per-node info |
| 64 | #ifndef PRODUCT |
| 65 | void LoopNode::dump_spec(outputStream *st) const { |
| 66 | if (is_inner_loop()) st->print( "inner " ); |
| 67 | if (is_partial_peel_loop()) st->print( "partial_peel " ); |
| 68 | if (partial_peel_has_failed()) st->print( "partial_peel_failed " ); |
| 69 | } |
| 70 | #endif |
| 71 | |
| 72 | //------------------------------is_valid_counted_loop------------------------- |
| 73 | bool LoopNode::is_valid_counted_loop(BasicType bt) const { |
| 74 | if (is_BaseCountedLoop() && as_BaseCountedLoop()->bt() == bt) { |
| 75 | BaseCountedLoopNode* l = as_BaseCountedLoop(); |
| 76 | BaseCountedLoopEndNode* le = l->loopexit_or_null(); |
| 77 | if (le != NULL__null && |
| 78 | le->proj_out_or_null(1 /* true */) == l->in(LoopNode::LoopBackControl)) { |
| 79 | Node* phi = l->phi(); |
| 80 | Node* exit = le->proj_out_or_null(0 /* false */); |
| 81 | if (exit != NULL__null && exit->Opcode() == Op_IfFalse && |
| 82 | phi != NULL__null && phi->is_Phi() && |
| 83 | phi->in(LoopNode::LoopBackControl) == l->incr() && |
| 84 | le->loopnode() == l && le->stride_is_con()) { |
| 85 | return true; |
| 86 | } |
| 87 | } |
| 88 | } |
| 89 | return false; |
| 90 | } |
| 91 | |
| 92 | //------------------------------get_early_ctrl--------------------------------- |
| 93 | // Compute earliest legal control |
| 94 | Node *PhaseIdealLoop::get_early_ctrl( Node *n ) { |
| 95 | assert( !n->is_Phi() && !n->is_CFG(), "this code only handles data nodes" )do { if (!(!n->is_Phi() && !n->is_CFG())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 95, "assert(" "!n->is_Phi() && !n->is_CFG()" ") failed" , "this code only handles data nodes"); ::breakpoint(); } } while (0); |
| 96 | uint i; |
| 97 | Node *early; |
| 98 | if (n->in(0) && !n->is_expensive()) { |
| 99 | early = n->in(0); |
| 100 | if (!early->is_CFG()) // Might be a non-CFG multi-def |
| 101 | early = get_ctrl(early); // So treat input as a straight data input |
| 102 | i = 1; |
| 103 | } else { |
| 104 | early = get_ctrl(n->in(1)); |
| 105 | i = 2; |
| 106 | } |
| 107 | uint e_d = dom_depth(early); |
| 108 | assert( early, "" )do { if (!(early)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 108, "assert(" "early" ") failed", ""); ::breakpoint(); } } while (0); |
| 109 | for (; i < n->req(); i++) { |
| 110 | Node *cin = get_ctrl(n->in(i)); |
| 111 | assert( cin, "" )do { if (!(cin)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 111, "assert(" "cin" ") failed", ""); ::breakpoint(); } } while (0); |
| 112 | // Keep deepest dominator depth |
| 113 | uint c_d = dom_depth(cin); |
| 114 | if (c_d > e_d) { // Deeper guy? |
| 115 | early = cin; // Keep deepest found so far |
| 116 | e_d = c_d; |
| 117 | } else if (c_d == e_d && // Same depth? |
| 118 | early != cin) { // If not equal, must use slower algorithm |
| 119 | // If same depth but not equal, one _must_ dominate the other |
| 120 | // and we want the deeper (i.e., dominated) guy. |
| 121 | Node *n1 = early; |
| 122 | Node *n2 = cin; |
| 123 | while (1) { |
| 124 | n1 = idom(n1); // Walk up until break cycle |
| 125 | n2 = idom(n2); |
| 126 | if (n1 == cin || // Walked early up to cin |
| 127 | dom_depth(n2) < c_d) |
| 128 | break; // early is deeper; keep him |
| 129 | if (n2 == early || // Walked cin up to early |
| 130 | dom_depth(n1) < c_d) { |
| 131 | early = cin; // cin is deeper; keep him |
| 132 | break; |
| 133 | } |
| 134 | } |
| 135 | e_d = dom_depth(early); // Reset depth register cache |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // Return earliest legal location |
| 140 | assert(early == find_non_split_ctrl(early), "unexpected early control")do { if (!(early == find_non_split_ctrl(early))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 140, "assert(" "early == find_non_split_ctrl(early)" ") failed" , "unexpected early control"); ::breakpoint(); } } while (0); |
| 141 | |
| 142 | if (n->is_expensive() && !_verify_only && !_verify_me) { |
| 143 | assert(n->in(0), "should have control input")do { if (!(n->in(0))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 143, "assert(" "n->in(0)" ") failed", "should have control input" ); ::breakpoint(); } } while (0); |
| 144 | early = get_early_ctrl_for_expensive(n, early); |
| 145 | } |
| 146 | |
| 147 | return early; |
| 148 | } |
| 149 | |
| 150 | //------------------------------get_early_ctrl_for_expensive--------------------------------- |
| 151 | // Move node up the dominator tree as high as legal while still beneficial |
| 152 | Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { |
| 153 | assert(n->in(0) && n->is_expensive(), "expensive node with control input here")do { if (!(n->in(0) && n->is_expensive())) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 153, "assert(" "n->in(0) && n->is_expensive()" ") failed", "expensive node with control input here"); ::breakpoint (); } } while (0); |
| 154 | assert(OptimizeExpensiveOps, "optimization off?")do { if (!(OptimizeExpensiveOps)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 154, "assert(" "OptimizeExpensiveOps" ") failed", "optimization off?" ); ::breakpoint(); } } while (0); |
| 155 | |
| 156 | Node* ctl = n->in(0); |
| 157 | assert(ctl->is_CFG(), "expensive input 0 must be cfg")do { if (!(ctl->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 157, "assert(" "ctl->is_CFG()" ") failed", "expensive input 0 must be cfg" ); ::breakpoint(); } } while (0); |
| 158 | uint min_dom_depth = dom_depth(earliest); |
| 159 | #ifdef ASSERT1 |
| 160 | if (!is_dominator(ctl, earliest) && !is_dominator(earliest, ctl)) { |
| 161 | dump_bad_graph("Bad graph detected in get_early_ctrl_for_expensive", n, earliest, ctl); |
| 162 | assert(false, "Bad graph detected in get_early_ctrl_for_expensive")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 162, "assert(" "false" ") failed", "Bad graph detected in get_early_ctrl_for_expensive" ); ::breakpoint(); } } while (0); |
| 163 | } |
| 164 | #endif |
| 165 | if (dom_depth(ctl) < min_dom_depth) { |
| 166 | return earliest; |
| 167 | } |
| 168 | |
| 169 | while (1) { |
| 170 | Node *next = ctl; |
| 171 | // Moving the node out of a loop on the projection of a If |
| 172 | // confuses loop predication. So once we hit a Loop in a If branch |
| 173 | // that doesn't branch to an UNC, we stop. The code that process |
| 174 | // expensive nodes will notice the loop and skip over it to try to |
| 175 | // move the node further up. |
| 176 | if (ctl->is_CountedLoop() && ctl->in(1) != NULL__null && ctl->in(1)->in(0) != NULL__null && ctl->in(1)->in(0)->is_If()) { |
| 177 | if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { |
| 178 | break; |
| 179 | } |
| 180 | next = idom(ctl->in(1)->in(0)); |
| 181 | } else if (ctl->is_Proj()) { |
| 182 | // We only move it up along a projection if the projection is |
| 183 | // the single control projection for its parent: same code path, |
| 184 | // if it's a If with UNC or fallthrough of a call. |
| 185 | Node* parent_ctl = ctl->in(0); |
| 186 | if (parent_ctl == NULL__null) { |
| 187 | break; |
| 188 | } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != NULL__null) { |
| 189 | next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control(); |
| 190 | } else if (parent_ctl->is_If()) { |
| 191 | if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { |
| 192 | break; |
| 193 | } |
| 194 | assert(idom(ctl) == parent_ctl, "strange")do { if (!(idom(ctl) == parent_ctl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 194, "assert(" "idom(ctl) == parent_ctl" ") failed", "strange" ); ::breakpoint(); } } while (0); |
| 195 | next = idom(parent_ctl); |
| 196 | } else if (ctl->is_CatchProj()) { |
| 197 | if (ctl->as_Proj()->_con != CatchProjNode::fall_through_index) { |
| 198 | break; |
| 199 | } |
| 200 | assert(parent_ctl->in(0)->in(0)->is_Call(), "strange graph")do { if (!(parent_ctl->in(0)->in(0)->is_Call())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 200, "assert(" "parent_ctl->in(0)->in(0)->is_Call()" ") failed", "strange graph"); ::breakpoint(); } } while (0); |
| 201 | next = parent_ctl->in(0)->in(0)->in(0); |
| 202 | } else { |
| 203 | // Check if parent control has a single projection (this |
| 204 | // control is the only possible successor of the parent |
| 205 | // control). If so, we can try to move the node above the |
| 206 | // parent control. |
| 207 | int nb_ctl_proj = 0; |
| 208 | for (DUIterator_Fast imax, i = parent_ctl->fast_outs(imax); i < imax; i++) { |
| 209 | Node *p = parent_ctl->fast_out(i); |
| 210 | if (p->is_Proj() && p->is_CFG()) { |
| 211 | nb_ctl_proj++; |
| 212 | if (nb_ctl_proj > 1) { |
| 213 | break; |
| 214 | } |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | if (nb_ctl_proj > 1) { |
| 219 | break; |
| 220 | } |
| 221 | assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() ||do { if (!(parent_ctl->is_Start() || parent_ctl->is_MemBar () || parent_ctl->is_Call() || BarrierSet::barrier_set()-> barrier_set_c2()->is_gc_barrier_node(parent_ctl))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 222, "assert(" "parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl)" ") failed", "unexpected node"); ::breakpoint(); } } while (0 ) |
| 222 | BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl), "unexpected node")do { if (!(parent_ctl->is_Start() || parent_ctl->is_MemBar () || parent_ctl->is_Call() || BarrierSet::barrier_set()-> barrier_set_c2()->is_gc_barrier_node(parent_ctl))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 222, "assert(" "parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl)" ") failed", "unexpected node"); ::breakpoint(); } } while (0 ); |
| 223 | assert(idom(ctl) == parent_ctl, "strange")do { if (!(idom(ctl) == parent_ctl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 223, "assert(" "idom(ctl) == parent_ctl" ") failed", "strange" ); ::breakpoint(); } } while (0); |
| 224 | next = idom(parent_ctl); |
| 225 | } |
| 226 | } else { |
| 227 | next = idom(ctl); |
| 228 | } |
| 229 | if (next->is_Root() || next->is_Start() || dom_depth(next) < min_dom_depth) { |
| 230 | break; |
| 231 | } |
| 232 | ctl = next; |
| 233 | } |
| 234 | |
| 235 | if (ctl != n->in(0)) { |
| 236 | _igvn.replace_input_of(n, 0, ctl); |
| 237 | _igvn.hash_insert(n); |
| 238 | } |
| 239 | |
| 240 | return ctl; |
| 241 | } |
| 242 | |
| 243 | |
| 244 | //------------------------------set_early_ctrl--------------------------------- |
| 245 | // Set earliest legal control |
| 246 | void PhaseIdealLoop::set_early_ctrl(Node* n, bool update_body) { |
| 247 | Node *early = get_early_ctrl(n); |
| 248 | |
| 249 | // Record earliest legal location |
| 250 | set_ctrl(n, early); |
| 251 | IdealLoopTree *loop = get_loop(early); |
| 252 | if (update_body && loop->_child == NULL__null) { |
| 253 | loop->_body.push(n); |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | //------------------------------set_subtree_ctrl------------------------------- |
| 258 | // set missing _ctrl entries on new nodes |
| 259 | void PhaseIdealLoop::set_subtree_ctrl(Node* n, bool update_body) { |
| 260 | // Already set? Get out. |
| 261 | if (_nodes[n->_idx]) return; |
| 262 | // Recursively set _nodes array to indicate where the Node goes |
| 263 | uint i; |
| 264 | for (i = 0; i < n->req(); ++i) { |
| 265 | Node *m = n->in(i); |
| 266 | if (m && m != C->root()) { |
| 267 | set_subtree_ctrl(m, update_body); |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // Fixup self |
| 272 | set_early_ctrl(n, update_body); |
| 273 | } |
| 274 | |
| 275 | IdealLoopTree* PhaseIdealLoop::insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift) { |
| 276 | IdealLoopTree* outer_ilt = new IdealLoopTree(this, outer_l, outer_ift); |
| 277 | IdealLoopTree* parent = loop->_parent; |
| 278 | IdealLoopTree* sibling = parent->_child; |
| 279 | if (sibling == loop) { |
| 280 | parent->_child = outer_ilt; |
| 281 | } else { |
| 282 | while (sibling->_next != loop) { |
| 283 | sibling = sibling->_next; |
| 284 | } |
| 285 | sibling->_next = outer_ilt; |
| 286 | } |
| 287 | outer_ilt->_next = loop->_next; |
| 288 | outer_ilt->_parent = parent; |
| 289 | outer_ilt->_child = loop; |
| 290 | outer_ilt->_nest = loop->_nest; |
| 291 | loop->_parent = outer_ilt; |
| 292 | loop->_next = NULL__null; |
| 293 | loop->_nest++; |
| 294 | assert(loop->_nest <= SHRT_MAX, "sanity")do { if (!(loop->_nest <= 32767)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 294, "assert(" "loop->_nest <= 32767" ") failed", "sanity" ); ::breakpoint(); } } while (0); |
| 295 | return outer_ilt; |
| 296 | } |
| 297 | |
| 298 | // Create a skeleton strip mined outer loop: a Loop head before the |
| 299 | // inner strip mined loop, a safepoint and an exit condition guarded |
| 300 | // by an opaque node after the inner strip mined loop with a backedge |
| 301 | // to the loop head. The inner strip mined loop is left as it is. Only |
| 302 | // once loop optimizations are over, do we adjust the inner loop exit |
| 303 | // condition to limit its number of iterations, set the outer loop |
| 304 | // exit condition and add Phis to the outer loop head. Some loop |
| 305 | // optimizations that operate on the inner strip mined loop need to be |
| 306 | // aware of the outer strip mined loop: loop unswitching needs to |
| 307 | // clone the outer loop as well as the inner, unrolling needs to only |
| 308 | // clone the inner loop etc. No optimizations need to change the outer |
| 309 | // strip mined loop as it is only a skeleton. |
| 310 | IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control, |
| 311 | IdealLoopTree* loop, float cl_prob, float le_fcnt, |
| 312 | Node*& entry_control, Node*& iffalse) { |
| 313 | Node* outer_test = _igvn.intcon(0); |
| 314 | set_ctrl(outer_test, C->root()); |
| 315 | Node *orig = iffalse; |
| 316 | iffalse = iffalse->clone(); |
| 317 | _igvn.register_new_node_with_optimizer(iffalse); |
| 318 | set_idom(iffalse, idom(orig), dom_depth(orig)); |
| 319 | |
| 320 | IfNode *outer_le = new OuterStripMinedLoopEndNode(iffalse, outer_test, cl_prob, le_fcnt); |
| 321 | Node *outer_ift = new IfTrueNode (outer_le); |
| 322 | Node* outer_iff = orig; |
| 323 | _igvn.replace_input_of(outer_iff, 0, outer_le); |
| 324 | |
| 325 | LoopNode *outer_l = new OuterStripMinedLoopNode(C, init_control, outer_ift); |
| 326 | entry_control = outer_l; |
| 327 | |
| 328 | IdealLoopTree* outer_ilt = insert_outer_loop(loop, outer_l, outer_ift); |
| 329 | |
| 330 | set_loop(iffalse, outer_ilt); |
| 331 | // When this code runs, loop bodies have not yet been populated. |
| 332 | const bool body_populated = false; |
| 333 | register_control(outer_le, outer_ilt, iffalse, body_populated); |
| 334 | register_control(outer_ift, outer_ilt, outer_le, body_populated); |
| 335 | set_idom(outer_iff, outer_le, dom_depth(outer_le)); |
| 336 | _igvn.register_new_node_with_optimizer(outer_l); |
| 337 | set_loop(outer_l, outer_ilt); |
| 338 | set_idom(outer_l, init_control, dom_depth(init_control)+1); |
| 339 | |
| 340 | return outer_ilt; |
| 341 | } |
| 342 | |
| 343 | void PhaseIdealLoop::insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol) { |
| 344 | Node* new_predicate_proj = create_new_if_for_predicate(limit_check_proj, NULL__null, |
| 345 | Deoptimization::Reason_loop_limit_check, |
| 346 | Op_If); |
| 347 | Node* iff = new_predicate_proj->in(0); |
| 348 | assert(iff->Opcode() == Op_If, "bad graph shape")do { if (!(iff->Opcode() == Op_If)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 348, "assert(" "iff->Opcode() == Op_If" ") failed", "bad graph shape" ); ::breakpoint(); } } while (0); |
| 349 | Node* conv = iff->in(1); |
| 350 | assert(conv->Opcode() == Op_Conv2B, "bad graph shape")do { if (!(conv->Opcode() == Op_Conv2B)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 350, "assert(" "conv->Opcode() == Op_Conv2B" ") failed", "bad graph shape"); ::breakpoint(); } } while (0); |
| 351 | Node* opaq = conv->in(1); |
| 352 | assert(opaq->Opcode() == Op_Opaque1, "bad graph shape")do { if (!(opaq->Opcode() == Op_Opaque1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 352, "assert(" "opaq->Opcode() == Op_Opaque1" ") failed" , "bad graph shape"); ::breakpoint(); } } while (0); |
| 353 | cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit); |
| 354 | bol = _igvn.register_new_node_with_optimizer(bol); |
| 355 | set_subtree_ctrl(bol, false); |
| 356 | _igvn.replace_input_of(iff, 1, bol); |
| 357 | |
| 358 | #ifndef PRODUCT |
| 359 | // report that the loop predication has been actually performed |
| 360 | // for this loop |
| 361 | if (TraceLoopLimitCheck) { |
| 362 | tty->print_cr("Counted Loop Limit Check generated:"); |
| 363 | debug_only( bol->dump(2); )bol->dump(2); |
| 364 | } |
| 365 | #endif |
| 366 | } |
| 367 | |
| 368 | Node* PhaseIdealLoop::loop_exit_control(Node* x, IdealLoopTree* loop) { |
| 369 | // Counted loop head must be a good RegionNode with only 3 not NULL |
| 370 | // control input edges: Self, Entry, LoopBack. |
| 371 | if (x->in(LoopNode::Self) == NULL__null || x->req() != 3 || loop->_irreducible) { |
| 372 | return NULL__null; |
| 373 | } |
| 374 | Node *init_control = x->in(LoopNode::EntryControl); |
| 375 | Node *back_control = x->in(LoopNode::LoopBackControl); |
| 376 | if (init_control == NULL__null || back_control == NULL__null) { // Partially dead |
| 377 | return NULL__null; |
| 378 | } |
| 379 | // Must also check for TOP when looking for a dead loop |
| 380 | if (init_control->is_top() || back_control->is_top()) { |
| 381 | return NULL__null; |
| 382 | } |
| 383 | |
| 384 | // Allow funny placement of Safepoint |
| 385 | if (back_control->Opcode() == Op_SafePoint) { |
| 386 | back_control = back_control->in(TypeFunc::Control); |
| 387 | } |
| 388 | |
| 389 | // Controlling test for loop |
| 390 | Node *iftrue = back_control; |
| 391 | uint iftrue_op = iftrue->Opcode(); |
| 392 | if (iftrue_op != Op_IfTrue && |
| 393 | iftrue_op != Op_IfFalse) { |
| 394 | // I have a weird back-control. Probably the loop-exit test is in |
| 395 | // the middle of the loop and I am looking at some trailing control-flow |
| 396 | // merge point. To fix this I would have to partially peel the loop. |
| 397 | return NULL__null; // Obscure back-control |
| 398 | } |
| 399 | |
| 400 | // Get boolean guarding loop-back test |
| 401 | Node *iff = iftrue->in(0); |
| 402 | if (get_loop(iff) != loop || !iff->in(1)->is_Bool()) { |
| 403 | return NULL__null; |
| 404 | } |
| 405 | return iftrue; |
| 406 | } |
| 407 | |
| 408 | Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { |
| 409 | Node* iftrue = back_control; |
| 410 | uint iftrue_op = iftrue->Opcode(); |
| 411 | Node* iff = iftrue->in(0); |
| 412 | BoolNode* test = iff->in(1)->as_Bool(); |
| 413 | bt = test->_test._test; |
| 414 | cl_prob = iff->as_If()->_prob; |
| 415 | if (iftrue_op == Op_IfFalse) { |
| 416 | bt = BoolTest(bt).negate(); |
| 417 | cl_prob = 1.0 - cl_prob; |
| 418 | } |
| 419 | // Get backedge compare |
| 420 | Node* cmp = test->in(1); |
| 421 | if (!cmp->is_Cmp()) { |
| 422 | return NULL__null; |
| 423 | } |
| 424 | |
| 425 | // Find the trip-counter increment & limit. Limit must be loop invariant. |
| 426 | incr = cmp->in(1); |
| 427 | limit = cmp->in(2); |
| 428 | |
| 429 | // --------- |
| 430 | // need 'loop()' test to tell if limit is loop invariant |
| 431 | // --------- |
| 432 | |
| 433 | if (!is_member(loop, get_ctrl(incr))) { // Swapped trip counter and limit? |
| 434 | Node* tmp = incr; // Then reverse order into the CmpI |
| 435 | incr = limit; |
| 436 | limit = tmp; |
| 437 | bt = BoolTest(bt).commute(); // And commute the exit test |
| 438 | } |
| 439 | if (is_member(loop, get_ctrl(limit))) { // Limit must be loop-invariant |
| 440 | return NULL__null; |
| 441 | } |
| 442 | if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant |
| 443 | return NULL__null; |
| 444 | } |
| 445 | return cmp; |
| 446 | } |
| 447 | |
| 448 | Node* PhaseIdealLoop::loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr) { |
| 449 | if (incr->is_Phi()) { |
| 450 | if (incr->as_Phi()->region() != x || incr->req() != 3) { |
| 451 | return NULL__null; // Not simple trip counter expression |
| 452 | } |
| 453 | phi_incr = incr; |
| 454 | incr = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi |
| 455 | if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant |
| 456 | return NULL__null; |
| 457 | } |
| 458 | } |
| 459 | return incr; |
| 460 | } |
| 461 | |
| 462 | Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi) { |
| 463 | assert(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL, "caller resp.")do { if (!(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 463, "assert(" "incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL" ") failed", "caller resp."); ::breakpoint(); } } while (0); |
| 464 | // Get merge point |
| 465 | xphi = incr->in(1); |
| 466 | Node *stride = incr->in(2); |
| 467 | if (!stride->is_Con()) { // Oops, swap these |
| 468 | if (!xphi->is_Con()) { // Is the other guy a constant? |
| 469 | return NULL__null; // Nope, unknown stride, bail out |
| 470 | } |
| 471 | Node *tmp = xphi; // 'incr' is commutative, so ok to swap |
| 472 | xphi = stride; |
| 473 | stride = tmp; |
| 474 | } |
| 475 | return stride; |
| 476 | } |
| 477 | |
| 478 | PhiNode* PhaseIdealLoop::loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop) { |
| 479 | if (!xphi->is_Phi()) { |
| 480 | return NULL__null; // Too much math on the trip counter |
| 481 | } |
| 482 | if (phi_incr != NULL__null && phi_incr != xphi) { |
| 483 | return NULL__null; |
| 484 | } |
| 485 | PhiNode *phi = xphi->as_Phi(); |
| 486 | |
| 487 | // Phi must be of loop header; backedge must wrap to increment |
| 488 | if (phi->region() != x) { |
| 489 | return NULL__null; |
| 490 | } |
| 491 | return phi; |
| 492 | } |
| 493 | |
| 494 | static int check_stride_overflow(jlong stride_con, const TypeInteger* limit_t, BasicType bt) { |
| 495 | if (stride_con > 0) { |
| 496 | if (limit_t->lo_as_long() > (max_signed_integer(bt) - stride_con)) { |
| 497 | return -1; |
| 498 | } |
| 499 | if (limit_t->hi_as_long() > (max_signed_integer(bt) - stride_con)) { |
| 500 | return 1; |
| 501 | } |
| 502 | } else { |
| 503 | if (limit_t->hi_as_long() < (min_signed_integer(bt) - stride_con)) { |
| 504 | return -1; |
| 505 | } |
| 506 | if (limit_t->lo_as_long() < (min_signed_integer(bt) - stride_con)) { |
| 507 | return 1; |
| 508 | } |
| 509 | } |
| 510 | return 0; |
| 511 | } |
| 512 | |
| 513 | static bool condition_stride_ok(BoolTest::mask bt, jlong stride_con) { |
| 514 | // If the condition is inverted and we will be rolling |
| 515 | // through MININT to MAXINT, then bail out. |
| 516 | if (bt == BoolTest::eq || // Bail out, but this loop trips at most twice! |
| 517 | // Odd stride |
| 518 | (bt == BoolTest::ne && stride_con != 1 && stride_con != -1) || |
| 519 | // Count down loop rolls through MAXINT |
| 520 | ((bt == BoolTest::le || bt == BoolTest::lt) && stride_con < 0) || |
| 521 | // Count up loop rolls through MININT |
| 522 | ((bt == BoolTest::ge || bt == BoolTest::gt) && stride_con > 0)) { |
| 523 | return false; // Bail out |
| 524 | } |
| 525 | return true; |
| 526 | } |
| 527 | |
| 528 | Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, |
| 529 | BasicType bt) { |
| 530 | Node* iv_as_long; |
| 531 | if (bt == T_LONG) { |
| 532 | iv_as_long = new ConvI2LNode(inner_iv, TypeLong::INT); |
| 533 | register_new_node(iv_as_long, inner_head); |
| 534 | } else { |
| 535 | iv_as_long = inner_iv; |
| 536 | } |
| 537 | Node* iv_replacement = AddNode::make(outer_phi, iv_as_long, bt); |
| 538 | register_new_node(iv_replacement, inner_head); |
| 539 | for (DUIterator_Last imin, i = iv_to_replace->last_outs(imin); i >= imin;) { |
| 540 | Node* u = iv_to_replace->last_out(i); |
| 541 | #ifdef ASSERT1 |
| 542 | if (!is_dominator(inner_head, ctrl_or_self(u))) { |
| 543 | assert(u->is_Phi(), "should be a Phi")do { if (!(u->is_Phi())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 543, "assert(" "u->is_Phi()" ") failed", "should be a Phi" ); ::breakpoint(); } } while (0); |
| 544 | for (uint j = 1; j < u->req(); j++) { |
| 545 | if (u->in(j) == iv_to_replace) { |
| 546 | assert(is_dominator(inner_head, u->in(0)->in(j)), "iv use above loop?")do { if (!(is_dominator(inner_head, u->in(0)->in(j)))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 546, "assert(" "is_dominator(inner_head, u->in(0)->in(j))" ") failed", "iv use above loop?"); ::breakpoint(); } } while (0); |
| 547 | } |
| 548 | } |
| 549 | } |
| 550 | #endif |
| 551 | _igvn.rehash_node_delayed(u); |
| 552 | int nb = u->replace_edge(iv_to_replace, iv_replacement, &_igvn); |
| 553 | i -= nb; |
| 554 | } |
| 555 | return iv_replacement; |
| 556 | } |
| 557 | |
| 558 | void PhaseIdealLoop::add_empty_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt) { |
| 559 | if (!C->too_many_traps(reason)) { |
| 560 | Node *cont = _igvn.intcon(1); |
| 561 | Node* opq = new Opaque1Node(C, cont); |
| 562 | _igvn.register_new_node_with_optimizer(opq); |
| 563 | Node *bol = new Conv2BNode(opq); |
| 564 | _igvn.register_new_node_with_optimizer(bol); |
| 565 | set_subtree_ctrl(bol, false); |
| 566 | IfNode* iff = new IfNode(inner_head->in(LoopNode::EntryControl), bol, PROB_MAX(1.0f-(1e-6f)), COUNT_UNKNOWN(-1.0f)); |
| 567 | register_control(iff, loop, inner_head->in(LoopNode::EntryControl)); |
| 568 | Node* iffalse = new IfFalseNode(iff); |
| 569 | register_control(iffalse, _ltree_root, iff); |
| 570 | Node* iftrue = new IfTrueNode(iff); |
| 571 | register_control(iftrue, loop, iff); |
| 572 | C->add_predicate_opaq(opq); |
| 573 | |
| 574 | int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile); |
| 575 | address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); |
| 576 | const TypePtr* no_memory_effects = NULL__null; |
| 577 | JVMState* jvms = sfpt->jvms(); |
Value stored to 'jvms' during its initialization is never read | |
| 578 | CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap", |
| 579 | no_memory_effects); |
| 580 | |
| 581 | Node* mem = NULL__null; |
| 582 | Node* i_o = NULL__null; |
| 583 | if (sfpt->is_Call()) { |
| 584 | mem = sfpt->proj_out(TypeFunc::Memory); |
| 585 | i_o = sfpt->proj_out(TypeFunc::I_O); |
| 586 | } else { |
| 587 | mem = sfpt->memory(); |
| 588 | i_o = sfpt->i_o(); |
| 589 | } |
| 590 | |
| 591 | Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr); |
| 592 | register_new_node(frame, C->start()); |
| 593 | Node *ret = new ParmNode(C->start(), TypeFunc::ReturnAdr); |
| 594 | register_new_node(ret, C->start()); |
| 595 | |
| 596 | unc->init_req(TypeFunc::Control, iffalse); |
| 597 | unc->init_req(TypeFunc::I_O, i_o); |
| 598 | unc->init_req(TypeFunc::Memory, mem); // may gc ptrs |
| 599 | unc->init_req(TypeFunc::FramePtr, frame); |
| 600 | unc->init_req(TypeFunc::ReturnAdr, ret); |
| 601 | unc->init_req(TypeFunc::Parms+0, _igvn.intcon(trap_request)); |
| 602 | unc->set_cnt(PROB_UNLIKELY_MAG(4)(1e-4f)); |
| 603 | unc->copy_call_debug_info(&_igvn, sfpt); |
| 604 | |
| 605 | for (uint i = TypeFunc::Parms; i < unc->req(); i++) { |
| 606 | set_subtree_ctrl(unc->in(i), false); |
| 607 | } |
| 608 | register_control(unc, _ltree_root, iffalse); |
| 609 | |
| 610 | Node* ctrl = new ProjNode(unc, TypeFunc::Control); |
| 611 | register_control(ctrl, _ltree_root, unc); |
| 612 | Node* halt = new HaltNode(ctrl, frame, "uncommon trap returned which should never happen" PRODUCT_ONLY(COMMA /*reachable*/false)); |
| 613 | register_control(halt, _ltree_root, ctrl); |
| 614 | C->root()->add_req(halt); |
| 615 | |
| 616 | _igvn.replace_input_of(inner_head, LoopNode::EntryControl, iftrue); |
| 617 | set_idom(inner_head, iftrue, dom_depth(inner_head)); |
| 618 | } |
| 619 | } |
| 620 | |
| 621 | // Find a safepoint node that dominates the back edge. We need a |
| 622 | // SafePointNode so we can use its jvm state to create empty |
| 623 | // predicates. |
| 624 | static bool no_side_effect_since_safepoint(Compile* C, Node* x, Node* mem, MergeMemNode* mm) { |
| 625 | SafePointNode* safepoint = NULL__null; |
| 626 | for (DUIterator_Fast imax, i = x->fast_outs(imax); i < imax; i++) { |
| 627 | Node* u = x->fast_out(i); |
| 628 | if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { |
| 629 | Node* m = u->in(LoopNode::LoopBackControl); |
| 630 | if (u->adr_type() == TypePtr::BOTTOM) { |
| 631 | if (m->is_MergeMem() && mem->is_MergeMem()) { |
| 632 | if (m != mem DEBUG_ONLY(|| true)|| true) { |
| 633 | for (MergeMemStream mms(m->as_MergeMem(), mem->as_MergeMem()); mms.next_non_empty2(); ) { |
| 634 | if (!mms.is_empty()) { |
| 635 | if (mms.memory() != mms.memory2()) { |
| 636 | return false; |
| 637 | } |
| 638 | #ifdef ASSERT1 |
| 639 | if (mms.alias_idx() != Compile::AliasIdxBot) { |
| 640 | mm->set_memory_at(mms.alias_idx(), mem->as_MergeMem()->base_memory()); |
| 641 | } |
| 642 | #endif |
| 643 | } |
| 644 | } |
| 645 | } |
| 646 | } else if (mem->is_MergeMem()) { |
| 647 | if (m != mem->as_MergeMem()->base_memory()) { |
| 648 | return false; |
| 649 | } |
| 650 | } else { |
| 651 | return false; |
| 652 | } |
| 653 | } else { |
| 654 | if (mem->is_MergeMem()) { |
| 655 | if (m != mem->as_MergeMem()->memory_at(C->get_alias_index(u->adr_type()))) { |
| 656 | return false; |
| 657 | } |
| 658 | #ifdef ASSERT1 |
| 659 | mm->set_memory_at(C->get_alias_index(u->adr_type()), mem->as_MergeMem()->base_memory()); |
| 660 | #endif |
| 661 | } else { |
| 662 | if (m != mem) { |
| 663 | return false; |
| 664 | } |
| 665 | } |
| 666 | } |
| 667 | } |
| 668 | } |
| 669 | return true; |
| 670 | } |
| 671 | |
| 672 | SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop) { |
| 673 | IfNode* exit_test = back_control->in(0)->as_If(); |
| 674 | SafePointNode* safepoint = NULL__null; |
| 675 | if (exit_test->in(0)->is_SafePoint() && exit_test->in(0)->outcnt() == 1) { |
| 676 | safepoint = exit_test->in(0)->as_SafePoint(); |
| 677 | } else { |
| 678 | Node* c = back_control; |
| 679 | while (c != x && c->Opcode() != Op_SafePoint) { |
| 680 | c = idom(c); |
| 681 | } |
| 682 | |
| 683 | if (c->Opcode() == Op_SafePoint) { |
| 684 | safepoint = c->as_SafePoint(); |
| 685 | } |
| 686 | |
| 687 | if (safepoint == NULL__null) { |
| 688 | return NULL__null; |
| 689 | } |
| 690 | |
| 691 | Node* mem = safepoint->in(TypeFunc::Memory); |
| 692 | |
| 693 | // We can only use that safepoint if there's not side effect |
| 694 | // between the backedge and the safepoint. |
| 695 | |
| 696 | // mm is used for book keeping |
| 697 | MergeMemNode* mm = NULL__null; |
| 698 | #ifdef ASSERT1 |
| 699 | if (mem->is_MergeMem()) { |
| 700 | mm = mem->clone()->as_MergeMem(); |
| 701 | _igvn._worklist.push(mm); |
| 702 | for (MergeMemStream mms(mem->as_MergeMem()); mms.next_non_empty(); ) { |
| 703 | if (mms.alias_idx() != Compile::AliasIdxBot && loop != get_loop(ctrl_or_self(mms.memory()))) { |
| 704 | mm->set_memory_at(mms.alias_idx(), mem->as_MergeMem()->base_memory()); |
| 705 | } |
| 706 | } |
| 707 | } |
| 708 | #endif |
| 709 | if (!no_side_effect_since_safepoint(C, x, mem, mm)) { |
| 710 | safepoint = NULL__null; |
| 711 | } else { |
| 712 | assert(mm == NULL|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory(), "all memory state should have been processed")do { if (!(mm == __null|| _igvn.transform(mm) == mem->as_MergeMem ()->base_memory())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 712, "assert(" "mm == __null|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory()" ") failed", "all memory state should have been processed"); :: breakpoint(); } } while (0); |
| 713 | } |
| 714 | #ifdef ASSERT1 |
| 715 | if (mm != NULL__null) { |
| 716 | _igvn.remove_dead_node(mm); |
| 717 | } |
| 718 | #endif |
| 719 | } |
| 720 | return safepoint; |
| 721 | } |
| 722 | |
| 723 | // If the loop has the shape of a counted loop but with a long |
| 724 | // induction variable, transform the loop in a loop nest: an inner |
| 725 | // loop that iterates for at most max int iterations with an integer |
| 726 | // induction variable and an outer loop that iterates over the full |
| 727 | // range of long values from the initial loop in (at most) max int |
| 728 | // steps. That is: |
| 729 | // |
| 730 | // x: for (long phi = init; phi < limit; phi += stride) { |
| 731 | // // phi := Phi(L, init, incr) |
| 732 | // // incr := AddL(phi, longcon(stride)) |
| 733 | // long incr = phi + stride; |
| 734 | // ... use phi and incr ... |
| 735 | // } |
| 736 | // |
| 737 | // OR: |
| 738 | // |
| 739 | // x: for (long phi = init; (phi += stride) < limit; ) { |
| 740 | // // phi := Phi(L, AddL(init, stride), incr) |
| 741 | // // incr := AddL(phi, longcon(stride)) |
| 742 | // long incr = phi + stride; |
| 743 | // ... use phi and (phi + stride) ... |
| 744 | // } |
| 745 | // |
| 746 | // ==transform=> |
| 747 | // |
| 748 | // const ulong inner_iters_limit = INT_MAX - stride - 1; //near 0x7FFFFFF0 |
| 749 | // assert(stride <= inner_iters_limit); // else abort transform |
| 750 | // assert((extralong)limit + stride <= LONG_MAX); // else deopt |
| 751 | // outer_head: for (long outer_phi = init;;) { |
| 752 | // // outer_phi := Phi(outer_head, init, AddL(outer_phi, I2L(inner_phi))) |
| 753 | // ulong inner_iters_max = (ulong) MAX(0, ((extralong)limit + stride - outer_phi)); |
| 754 | // long inner_iters_actual = MIN(inner_iters_limit, inner_iters_max); |
| 755 | // assert(inner_iters_actual == (int)inner_iters_actual); |
| 756 | // int inner_phi, inner_incr; |
| 757 | // x: for (inner_phi = 0;; inner_phi = inner_incr) { |
| 758 | // // inner_phi := Phi(x, intcon(0), inner_incr) |
| 759 | // // inner_incr := AddI(inner_phi, intcon(stride)) |
| 760 | // inner_incr = inner_phi + stride; |
| 761 | // if (inner_incr < inner_iters_actual) { |
| 762 | // ... use phi=>(outer_phi+inner_phi) and incr=>(outer_phi+inner_incr) ... |
| 763 | // continue; |
| 764 | // } |
| 765 | // else break; |
| 766 | // } |
| 767 | // if ((outer_phi+inner_phi) < limit) //OR (outer_phi+inner_incr) < limit |
| 768 | // continue; |
| 769 | // else break; |
| 770 | // } |
| 771 | // |
| 772 | // The same logic is used to transform an int counted loop that contains long range checks into a loop nest of 2 int |
| 773 | // loops with long range checks transformed to int range checks in the inner loop. |
| 774 | bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) { |
| 775 | Node* x = loop->_head; |
| 776 | // Only for inner loops |
| 777 | if (loop->_child != NULL__null || !x->is_BaseCountedLoop() || x->as_Loop()->is_loop_nest_outer_loop()) { |
| 778 | return false; |
| 779 | } |
| 780 | |
| 781 | if (x->is_CountedLoop() && !x->as_CountedLoop()->is_main_loop() && !x->as_CountedLoop()->is_normal_loop()) { |
| 782 | return false; |
| 783 | } |
| 784 | |
| 785 | BaseCountedLoopNode* head = x->as_BaseCountedLoop(); |
| 786 | BasicType bt = x->as_BaseCountedLoop()->bt(); |
| 787 | |
| 788 | check_counted_loop_shape(loop, x, bt); |
| 789 | |
| 790 | #ifndef PRODUCT |
| 791 | if (bt == T_LONG) { |
| 792 | Atomic::inc(&_long_loop_candidates); |
| 793 | } |
| 794 | #endif |
| 795 | |
| 796 | jlong stride_con = head->stride_con(); |
| 797 | assert(stride_con != 0, "missed some peephole opt")do { if (!(stride_con != 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 797, "assert(" "stride_con != 0" ") failed", "missed some peephole opt" ); ::breakpoint(); } } while (0); |
| 798 | // We can't iterate for more than max int at a time. |
| 799 | if (stride_con != (jint)stride_con) { |
| 800 | assert(bt == T_LONG, "only for long loops")do { if (!(bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 800, "assert(" "bt == T_LONG" ") failed", "only for long loops" ); ::breakpoint(); } } while (0); |
| 801 | return false; |
| 802 | } |
| 803 | // The number of iterations for the integer count loop: guarantee no |
| 804 | // overflow: max_jint - stride_con max. -1 so there's no need for a |
| 805 | // loop limit check if the exit test is <= or >=. |
| 806 | int iters_limit = max_jint - ABS(stride_con) - 1; |
| 807 | #ifdef ASSERT1 |
| 808 | if (bt == T_LONG && StressLongCountedLoop > 0) { |
| 809 | iters_limit = iters_limit / StressLongCountedLoop; |
| 810 | } |
| 811 | #endif |
| 812 | // At least 2 iterations so counted loop construction doesn't fail |
| 813 | if (iters_limit/ABS(stride_con) < 2) { |
| 814 | return false; |
| 815 | } |
| 816 | |
| 817 | PhiNode* phi = head->phi()->as_Phi(); |
| 818 | Node* incr = head->incr(); |
| 819 | |
| 820 | Node* back_control = head->in(LoopNode::LoopBackControl); |
| 821 | |
| 822 | // data nodes on back branch not supported |
| 823 | if (back_control->outcnt() > 1) { |
| 824 | return false; |
| 825 | } |
| 826 | |
| 827 | Node* limit = head->limit(); |
| 828 | // We'll need to use the loop limit before the inner loop is entered |
| 829 | if (!is_dominator(get_ctrl(limit), x)) { |
| 830 | return false; |
| 831 | } |
| 832 | |
| 833 | // May not have gone thru igvn yet so don't use _igvn.type(phi) (PhaseIdealLoop::is_counted_loop() sets the iv phi's type) |
| 834 | const TypeInteger* phi_t = phi->bottom_type()->is_integer(bt); |
| 835 | assert(phi_t->hi_as_long() >= phi_t->lo_as_long(), "dead phi?")do { if (!(phi_t->hi_as_long() >= phi_t->lo_as_long( ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 835, "assert(" "phi_t->hi_as_long() >= phi_t->lo_as_long()" ") failed", "dead phi?"); ::breakpoint(); } } while (0); |
| 836 | iters_limit = checked_cast<int>(MIN2((julong)iters_limit, (julong)(phi_t->hi_as_long() - phi_t->lo_as_long()))); |
| 837 | |
| 838 | IfNode* exit_test = head->loopexit(); |
| 839 | BoolTest::mask mask = exit_test->as_BaseCountedLoopEnd()->test_trip(); |
| 840 | Node* cmp = exit_test->as_BaseCountedLoopEnd()->cmp_node(); |
| 841 | |
| 842 | assert(back_control->Opcode() == Op_IfTrue, "wrong projection for back edge")do { if (!(back_control->Opcode() == Op_IfTrue)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 842, "assert(" "back_control->Opcode() == Op_IfTrue" ") failed" , "wrong projection for back edge"); ::breakpoint(); } } while (0); |
| 843 | |
| 844 | Node_List range_checks; |
| 845 | iters_limit = extract_long_range_checks(loop, stride_con, iters_limit, phi, range_checks); |
| 846 | |
| 847 | if (bt == T_INT) { |
| 848 | // The only purpose of creating a loop nest is to handle long range checks. If there are none, do not proceed further. |
| 849 | if (range_checks.size() == 0) { |
| 850 | return false; |
| 851 | } |
| 852 | } |
| 853 | |
| 854 | // We need a safepoint to insert empty predicates for the inner loop. |
| 855 | SafePointNode* safepoint; |
| 856 | if (bt == T_INT && head->as_CountedLoop()->is_strip_mined()) { |
| 857 | // Loop is strip mined: use the safepoint of the outer strip mined loop |
| 858 | strip_mined_nest_back_to_counted_loop(loop, head, back_control, exit_test, safepoint); |
| 859 | } else { |
| 860 | safepoint = find_safepoint(back_control, x, loop); |
| 861 | } |
| 862 | |
| 863 | Node* exit_branch = exit_test->proj_out(false); |
| 864 | Node* entry_control = head->in(LoopNode::EntryControl); |
| 865 | |
| 866 | // Clone the control flow of the loop to build an outer loop |
| 867 | Node* outer_back_branch = back_control->clone(); |
| 868 | Node* outer_exit_test = new IfNode(exit_test->in(0), exit_test->in(1), exit_test->_prob, exit_test->_fcnt); |
| 869 | Node* inner_exit_branch = exit_branch->clone(); |
| 870 | |
| 871 | LoopNode* outer_head = new LoopNode(entry_control, outer_back_branch); |
| 872 | IdealLoopTree* outer_ilt = insert_outer_loop(loop, outer_head, outer_back_branch); |
| 873 | |
| 874 | const bool body_populated = true; |
| 875 | register_control(outer_head, outer_ilt, entry_control, body_populated); |
| 876 | |
| 877 | _igvn.register_new_node_with_optimizer(inner_exit_branch); |
| 878 | set_loop(inner_exit_branch, outer_ilt); |
| 879 | set_idom(inner_exit_branch, exit_test, dom_depth(exit_branch)); |
| 880 | |
| 881 | outer_exit_test->set_req(0, inner_exit_branch); |
| 882 | register_control(outer_exit_test, outer_ilt, inner_exit_branch, body_populated); |
| 883 | |
| 884 | _igvn.replace_input_of(exit_branch, 0, outer_exit_test); |
| 885 | set_idom(exit_branch, outer_exit_test, dom_depth(exit_branch)); |
| 886 | |
| 887 | outer_back_branch->set_req(0, outer_exit_test); |
| 888 | register_control(outer_back_branch, outer_ilt, outer_exit_test, body_populated); |
| 889 | |
| 890 | _igvn.replace_input_of(x, LoopNode::EntryControl, outer_head); |
| 891 | set_idom(x, outer_head, dom_depth(x)); |
| 892 | |
| 893 | // add an iv phi to the outer loop and use it to compute the inner |
| 894 | // loop iteration limit |
| 895 | Node* outer_phi = phi->clone(); |
| 896 | outer_phi->set_req(0, outer_head); |
| 897 | register_new_node(outer_phi, outer_head); |
| 898 | |
| 899 | Node* inner_iters_max = NULL__null; |
| 900 | if (stride_con > 0) { |
| 901 | inner_iters_max = MaxNode::max_diff_with_zero(limit, outer_phi, TypeInteger::bottom(bt), _igvn); |
| 902 | } else { |
| 903 | inner_iters_max = MaxNode::max_diff_with_zero(outer_phi, limit, TypeInteger::bottom(bt), _igvn); |
| 904 | } |
| 905 | |
| 906 | Node* inner_iters_limit = _igvn.integercon(iters_limit, bt); |
| 907 | // inner_iters_max may not fit in a signed integer (iterating from |
| 908 | // Long.MIN_VALUE to Long.MAX_VALUE for instance). Use an unsigned |
| 909 | // min. |
| 910 | Node* inner_iters_actual = MaxNode::unsigned_min(inner_iters_max, inner_iters_limit, TypeInteger::make(0, iters_limit, Type::WidenMin, bt), _igvn); |
| 911 | |
| 912 | Node* inner_iters_actual_int; |
| 913 | if (bt == T_LONG) { |
| 914 | inner_iters_actual_int = new ConvL2INode(inner_iters_actual); |
| 915 | _igvn.register_new_node_with_optimizer(inner_iters_actual_int); |
| 916 | } else { |
| 917 | inner_iters_actual_int = inner_iters_actual; |
| 918 | } |
| 919 | |
| 920 | Node* int_zero = _igvn.intcon(0); |
| 921 | set_ctrl(int_zero, C->root()); |
| 922 | if (stride_con < 0) { |
| 923 | inner_iters_actual_int = new SubINode(int_zero, inner_iters_actual_int); |
| 924 | _igvn.register_new_node_with_optimizer(inner_iters_actual_int); |
| 925 | } |
| 926 | |
| 927 | // Clone the iv data nodes as an integer iv |
| 928 | Node* int_stride = _igvn.intcon(checked_cast<int>(stride_con)); |
| 929 | set_ctrl(int_stride, C->root()); |
| 930 | Node* inner_phi = new PhiNode(x->in(0), TypeInt::INT); |
| 931 | Node* inner_incr = new AddINode(inner_phi, int_stride); |
| 932 | Node* inner_cmp = NULL__null; |
| 933 | inner_cmp = new CmpINode(inner_incr, inner_iters_actual_int); |
| 934 | Node* inner_bol = new BoolNode(inner_cmp, exit_test->in(1)->as_Bool()->_test._test); |
| 935 | inner_phi->set_req(LoopNode::EntryControl, int_zero); |
| 936 | inner_phi->set_req(LoopNode::LoopBackControl, inner_incr); |
| 937 | register_new_node(inner_phi, x); |
| 938 | register_new_node(inner_incr, x); |
| 939 | register_new_node(inner_cmp, x); |
| 940 | register_new_node(inner_bol, x); |
| 941 | |
| 942 | _igvn.replace_input_of(exit_test, 1, inner_bol); |
| 943 | |
| 944 | // Clone inner loop phis to outer loop |
| 945 | for (uint i = 0; i < head->outcnt(); i++) { |
| 946 | Node* u = head->raw_out(i); |
| 947 | if (u->is_Phi() && u != inner_phi && u != phi) { |
| 948 | assert(u->in(0) == head, "inconsistent")do { if (!(u->in(0) == head)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 948, "assert(" "u->in(0) == head" ") failed", "inconsistent" ); ::breakpoint(); } } while (0); |
| 949 | Node* clone = u->clone(); |
| 950 | clone->set_req(0, outer_head); |
| 951 | register_new_node(clone, outer_head); |
| 952 | _igvn.replace_input_of(u, LoopNode::EntryControl, clone); |
| 953 | } |
| 954 | } |
| 955 | |
| 956 | // Replace inner loop long iv phi as inner loop int iv phi + outer |
| 957 | // loop iv phi |
| 958 | Node* iv_add = loop_nest_replace_iv(phi, inner_phi, outer_phi, head, bt); |
| 959 | |
| 960 | // Replace inner loop long iv incr with inner loop int incr + outer |
| 961 | // loop iv phi |
| 962 | loop_nest_replace_iv(incr, inner_incr, outer_phi, head, bt); |
| 963 | |
| 964 | set_subtree_ctrl(inner_iters_actual_int, body_populated); |
| 965 | |
| 966 | LoopNode* inner_head = create_inner_head(loop, head, exit_test); |
| 967 | |
| 968 | // Summary of steps from inital loop to loop nest: |
| 969 | // |
| 970 | // == old IR nodes => |
| 971 | // |
| 972 | // entry_control: {...} |
| 973 | // x: |
| 974 | // for (long phi = init;;) { |
| 975 | // // phi := Phi(x, init, incr) |
| 976 | // // incr := AddL(phi, longcon(stride)) |
| 977 | // exit_test: |
| 978 | // if (phi < limit) |
| 979 | // back_control: fallthrough; |
| 980 | // else |
| 981 | // exit_branch: break; |
| 982 | // long incr = phi + stride; |
| 983 | // ... use phi and incr ... |
| 984 | // phi = incr; |
| 985 | // } |
| 986 | // |
| 987 | // == new IR nodes (just before final peel) => |
| 988 | // |
| 989 | // entry_control: {...} |
| 990 | // long adjusted_limit = limit + stride; //because phi_incr != NULL |
| 991 | // assert(!limit_check_required || (extralong)limit + stride == adjusted_limit); // else deopt |
| 992 | // ulong inner_iters_limit = max_jint - ABS(stride) - 1; //near 0x7FFFFFF0 |
| 993 | // outer_head: |
| 994 | // for (long outer_phi = init;;) { |
| 995 | // // outer_phi := phi->clone(), in(0):=outer_head, => Phi(outer_head, init, incr) |
| 996 | // // REPLACE phi => AddL(outer_phi, I2L(inner_phi)) |
| 997 | // // REPLACE incr => AddL(outer_phi, I2L(inner_incr)) |
| 998 | // // SO THAT outer_phi := Phi(outer_head, init, AddL(outer_phi, I2L(inner_incr))) |
| 999 | // ulong inner_iters_max = (ulong) MAX(0, ((extralong)adjusted_limit - outer_phi) * SGN(stride)); |
| 1000 | // int inner_iters_actual_int = (int) MIN(inner_iters_limit, inner_iters_max) * SGN(stride); |
| 1001 | // inner_head: x: //in(1) := outer_head |
| 1002 | // int inner_phi; |
| 1003 | // for (inner_phi = 0;;) { |
| 1004 | // // inner_phi := Phi(x, intcon(0), inner_phi + stride) |
| 1005 | // int inner_incr = inner_phi + stride; |
| 1006 | // bool inner_bol = (inner_incr < inner_iters_actual_int); |
| 1007 | // exit_test: //exit_test->in(1) := inner_bol; |
| 1008 | // if (inner_bol) // WAS (phi < limit) |
| 1009 | // back_control: fallthrough; |
| 1010 | // else |
| 1011 | // inner_exit_branch: break; //exit_branch->clone() |
| 1012 | // ... use phi=>(outer_phi+inner_phi) and incr=>(outer_phi+inner_incr) ... |
| 1013 | // inner_phi = inner_phi + stride; // inner_incr |
| 1014 | // } |
| 1015 | // outer_exit_test: //exit_test->clone(), in(0):=inner_exit_branch |
| 1016 | // if ((outer_phi+inner_phi) < limit) // WAS (phi < limit) |
| 1017 | // outer_back_branch: fallthrough; //back_control->clone(), in(0):=outer_exit_test |
| 1018 | // else |
| 1019 | // exit_branch: break; //in(0) := outer_exit_test |
| 1020 | // } |
| 1021 | |
| 1022 | if (bt == T_INT) { |
| 1023 | outer_phi = new ConvI2LNode(outer_phi); |
| 1024 | register_new_node(outer_phi, outer_head); |
| 1025 | } |
| 1026 | |
| 1027 | transform_long_range_checks(checked_cast<int>(stride_con), range_checks, outer_phi, inner_iters_actual_int, |
| 1028 | inner_phi, iv_add, inner_head); |
| 1029 | // Peel one iteration of the loop and use the safepoint at the end |
| 1030 | // of the peeled iteration to insert empty predicates. If no well |
| 1031 | // positioned safepoint peel to guarantee a safepoint in the outer |
| 1032 | // loop. |
| 1033 | if (safepoint != NULL__null || !loop->_has_call) { |
| 1034 | old_new.clear(); |
| 1035 | do_peeling(loop, old_new); |
| 1036 | } else { |
| 1037 | C->set_major_progress(); |
| 1038 | } |
| 1039 | |
| 1040 | if (safepoint != NULL__null) { |
| 1041 | SafePointNode* cloned_sfpt = old_new[safepoint->_idx]->as_SafePoint(); |
| 1042 | |
| 1043 | if (UseLoopPredicate) { |
| 1044 | add_empty_predicate(Deoptimization::Reason_predicate, inner_head, outer_ilt, cloned_sfpt); |
| 1045 | } |
| 1046 | if (UseProfiledLoopPredicate) { |
| 1047 | add_empty_predicate(Deoptimization::Reason_profile_predicate, inner_head, outer_ilt, cloned_sfpt); |
| 1048 | } |
| 1049 | add_empty_predicate(Deoptimization::Reason_loop_limit_check, inner_head, outer_ilt, cloned_sfpt); |
| 1050 | } |
| 1051 | |
| 1052 | #ifndef PRODUCT |
| 1053 | if (bt == T_LONG) { |
| 1054 | Atomic::inc(&_long_loop_nests); |
| 1055 | } |
| 1056 | #endif |
| 1057 | |
| 1058 | inner_head->mark_loop_nest_inner_loop(); |
| 1059 | outer_head->mark_loop_nest_outer_loop(); |
| 1060 | |
| 1061 | return true; |
| 1062 | } |
| 1063 | |
| 1064 | // Convert the strip mined loop nest back to a single loop with the safepoint right before the loop exit test |
| 1065 | void PhaseIdealLoop::strip_mined_nest_back_to_counted_loop(IdealLoopTree* loop, const BaseCountedLoopNode* head, |
| 1066 | Node* back_control, IfNode*& exit_test, |
| 1067 | SafePointNode*& safepoint) { |
| 1068 | CountedLoopNode* cl = head->as_CountedLoop(); |
| 1069 | cl->verify_strip_mined(1); |
| 1070 | safepoint = cl->outer_safepoint(); |
| 1071 | CountedLoopEndNode* cle = cl->loopexit(); |
| 1072 | OuterStripMinedLoopNode* outer_head = cl->outer_loop(); |
| 1073 | OuterStripMinedLoopEndNode* outer_end = cl->outer_loop_end(); |
| 1074 | |
| 1075 | cl->clear_strip_mined(); |
| 1076 | |
| 1077 | _igvn.replace_input_of(cl, LoopNode::EntryControl, outer_head->in(LoopNode::EntryControl)); |
| 1078 | _igvn.replace_input_of(outer_head, LoopNode::EntryControl, C->top()); |
| 1079 | set_idom(cl, cl->in(LoopNode::EntryControl), dom_depth(cl)); |
| 1080 | |
| 1081 | Node* exit_bol = cle->in(1); |
| 1082 | Node *zero = _igvn.intcon(0); |
| 1083 | set_ctrl(zero, C->root()); |
| 1084 | _igvn.replace_input_of(cle, 1, zero); |
| 1085 | |
| 1086 | _igvn.replace_input_of(outer_end, 1, exit_bol); |
| 1087 | |
| 1088 | assert(outer_head->in(LoopNode::LoopBackControl)->in(0) == outer_end, "")do { if (!(outer_head->in(LoopNode::LoopBackControl)->in (0) == outer_end)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1088, "assert(" "outer_head->in(LoopNode::LoopBackControl)->in(0) == outer_end" ") failed", ""); ::breakpoint(); } } while (0); |
| 1089 | _igvn.replace_input_of(outer_head->in(LoopNode::LoopBackControl), 0, C->top()); |
| 1090 | _igvn.replace_input_of(back_control, 0, outer_end); |
| 1091 | set_idom(back_control, outer_end, dom_depth(outer_end) + 1); |
| 1092 | |
| 1093 | Unique_Node_List wq; |
| 1094 | wq.push(safepoint); |
| 1095 | |
| 1096 | IdealLoopTree* outer_loop_ilt = get_loop(outer_head); |
| 1097 | |
| 1098 | for (uint i = 0; i < wq.size(); i++) { |
| 1099 | Node* n = wq.at(i); |
| 1100 | for (uint j = 0; j < n->req(); ++j) { |
| 1101 | Node* in = n->in(j); |
| 1102 | if (in == NULL__null || in->is_CFG()) { |
| 1103 | continue; |
| 1104 | } |
| 1105 | if (get_loop(get_ctrl(in)) != outer_loop_ilt) { |
| 1106 | continue; |
| 1107 | } |
| 1108 | assert(!loop->_body.contains(in), "")do { if (!(!loop->_body.contains(in))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1108, "assert(" "!loop->_body.contains(in)" ") failed", "" ); ::breakpoint(); } } while (0); |
| 1109 | loop->_body.push(in); |
| 1110 | wq.push(in); |
| 1111 | } |
| 1112 | } |
| 1113 | |
| 1114 | set_loop(outer_end, loop); |
| 1115 | loop->_body.push(outer_end); |
| 1116 | set_loop(safepoint, loop); |
| 1117 | loop->_body.push(safepoint); |
| 1118 | set_loop(safepoint->in(0), loop); |
| 1119 | loop->_body.push(safepoint->in(0)); |
| 1120 | |
| 1121 | exit_test = outer_end; |
| 1122 | |
| 1123 | outer_loop_ilt->_tail = C->top(); |
| 1124 | } |
| 1125 | |
| 1126 | int PhaseIdealLoop::extract_long_range_checks(const IdealLoopTree* loop, jlong stride_con, int iters_limit, PhiNode* phi, |
| 1127 | Node_List& range_checks) { |
| 1128 | if (stride_con < 0) { // only for stride_con > 0 && scale > 0 for now |
| 1129 | return iters_limit; |
| 1130 | } |
| 1131 | const jlong min_iters = 2; |
| 1132 | jlong reduced_iters_limit = iters_limit; |
| 1133 | jlong original_iters_limit = iters_limit; |
| 1134 | for (uint i = 0; i < loop->_body.size(); i++) { |
| 1135 | Node* c = loop->_body.at(i); |
| 1136 | if (c->is_IfProj() && c->in(0)->is_RangeCheck()) { |
| 1137 | CallStaticJavaNode* call = c->as_IfProj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); |
| 1138 | if (call != NULL__null) { |
| 1139 | Node* range = NULL__null; |
| 1140 | Node* offset = NULL__null; |
| 1141 | jlong scale = 0; |
| 1142 | RangeCheckNode* rc = c->in(0)->as_RangeCheck(); |
| 1143 | if (loop->is_range_check_if(rc, this, T_LONG, phi, range, offset, scale) && |
| 1144 | loop->is_invariant(range) && loop->is_invariant(offset) && |
| 1145 | scale > 0 && // only for stride_con > 0 && scale > 0 for now |
| 1146 | original_iters_limit / ABS(scale * stride_con) >= min_iters) { |
| 1147 | reduced_iters_limit = MIN2(reduced_iters_limit, original_iters_limit/ABS(scale)); |
| 1148 | range_checks.push(c); |
| 1149 | } |
| 1150 | } |
| 1151 | } |
| 1152 | } |
| 1153 | |
| 1154 | return checked_cast<int>(reduced_iters_limit); |
| 1155 | } |
| 1156 | |
| 1157 | // One execution of the inner loop covers a sub-range of the entire iteration range of the loop: [A,Z), aka [A=init, |
| 1158 | // Z=limit). If the loop has at least one trip (which is the case here), the iteration variable i always takes A as its |
| 1159 | // first value, followed by A+S (S is the stride), next A+2S, etc. The limit is exclusive, so that the final value B of |
| 1160 | // i is never Z. It will be B=Z-1 if S=1, or B=Z+1 if S=-1. If |S|>1 the formula for the last value requires a floor |
| 1161 | // operation, specifically B=floor((Z-sgn(S)-A)/S)*S+A. Thus i ranges as i:[A,B] or i:[A,Z) or i:[A,Z-U) for some U<S. |
| 1162 | |
| 1163 | // N.B. We handle only the case of positive S currently, so comments about S<0 are not operative at present. Also, |
| 1164 | // we only support positive index scale value (K > 0) to simplify the logic for clamping 32-bit bounds (L_2, R_2). |
| 1165 | // For restrictions on S and K, see the guards in extract_long_range_checks. |
| 1166 | |
| 1167 | // Within the loop there may be many range checks. Each such range check (R.C.) is of the form 0 <= i*K+L < R, where K |
| 1168 | // is a scale factor applied to the loop iteration variable i, and L is some offset; K, L, and R are loop-invariant. |
| 1169 | // Because R is never negative, this check can always be simplified to an unsigned check i*K+L <u R. |
| 1170 | |
| 1171 | // When a long loop over a 64-bit variable i (outer_iv) is decomposed into a series of shorter sub-loops over a 32-bit |
| 1172 | // variable j (inner_iv), j ranges over a shorter interval j:[0,Z_2), where the limit is chosen to prevent various cases |
| 1173 | // of 32-bit overflow (including multiplications j*K below). In the sub-loop the logical value i is offset from j by a |
| 1174 | // 64-bit constant C, so i ranges in i:C+[0,Z_2). |
| 1175 | |
| 1176 | // The union of all the C+[0,Z_2) ranges from the sub-loops must be identical to the whole range [A,B]. Assuming S>0, |
| 1177 | // the first C must be A itself, and the next C value is the previous C+Z_2. In each sub-loop, j counts up from zero |
| 1178 | // and exits just before i=C+Z_2. |
| 1179 | |
| 1180 | // (N.B. If S<0 the formulas are different, because all the loops count downward.) |
| 1181 | |
| 1182 | // Returning to range checks, we see that each i*K+L <u R expands to (C+j)*K+L <u R, or j*K+Q <u R, where Q=(C*K+L). |
| 1183 | // (Recall that K and L and R are loop-invariant scale, offset and range values for a particular R.C.) This is still a |
| 1184 | // 64-bit comparison, so the range check elimination logic will not apply to it. (The R.C.E. transforms operate only on |
| 1185 | // 32-bit indexes and comparisons, because they use 64-bit temporary values to avoid overflow; see |
| 1186 | // PhaseIdealLoop::add_constraint.) |
| 1187 | |
| 1188 | // We must transform this comparison so that it gets the same answer, but by means of a 32-bit R.C. (using j not i) of |
| 1189 | // the form j*K+L_2 <u32 R_2. Note that L_2 and R_2 must be loop-invariant, but only with respect to the sub-loop. Thus, the |
| 1190 | // problem reduces to computing values for L_2 and R_2 (for each R.C. in the loop) in the loop header for the sub-loop. |
| 1191 | // Then the standard R.C.E. transforms can take those as inputs and further compute the necessary minimum and maximum |
| 1192 | // values for the 32-bit counter j within which the range checks can be eliminated. |
| 1193 | |
| 1194 | // So, given j*K+Q <u R, we need to find some j*K+L_2 <u32 R_2, where L_2 and R_2 fit in 32 bits, and the 32-bit operations do |
| 1195 | // not overflow. We also need to cover the cases where i*K+L (= j*K+Q) overflows to a 64-bit negative, since that is |
| 1196 | // allowed as an input to the R.C., as long as the R.C. as a whole fails. |
| 1197 | |
| 1198 | // If 32-bit multiplication j*K might overflow, we adjust the sub-loop limit Z_2 closer to zero to reduce j's range. |
| 1199 | |
| 1200 | // For each R.C. j*K+Q <u32 R, the range of mathematical values of j*K+Q in the sub-loop is [Q_min, Q_max), where |
| 1201 | // Q_min=Q and Q_max=Z_2*K+Q. Making the upper limit Q_max be exclusive helps it integrate correctly with the strict |
| 1202 | // comparisons against R and R_2. Sometimes a very high R will be replaced by an R_2 derived from the more moderate |
| 1203 | // Q_max, and replacing one exclusive limit by another exclusive limit avoids off-by-one complexities. |
| 1204 | |
| 1205 | // N.B. If (S*K)<0 then the formulas for Q_min and Q_max may differ; the values may need to be swapped and adjusted to |
| 1206 | // the correct type of bound (inclusive or exclusive). |
| 1207 | |
| 1208 | // Case A: Some Negatives (but no overflow). |
| 1209 | // Number line: |
| 1210 | // |s64_min . . . 0 . . . s64_max| |
| 1211 | // | . Q_min..Q_max . 0 . . . . | s64 negative |
| 1212 | // | . . . Q_min..0..Q_max . . . | small mixed |
| 1213 | // |
| 1214 | // if Q_min <s64 0, then use this test: |
| 1215 | // j*K + s32_trunc(Q_min) <u32 clamp(R, 0, Q_max) |
| 1216 | |
| 1217 | // If the 32-bit truncation loses information, no harm is done, since certainly the clamp also returns R_2=zero. |
| 1218 | |
| 1219 | // Case B: No Negatives. |
| 1220 | // Number line: |
| 1221 | // |s64_min . . . 0 . . . s64_max| |
| 1222 | // | . . . . 0 Q_min..Q_max . . | small positive |
| 1223 | // | . . . . 0 . Q_min..Q_max . | s64 positive |
| 1224 | // |
| 1225 | // if both Q_min, Q_max >=s64 0, then use this test: |
| 1226 | // j*K + 0 <u32 clamp(R, Q_min, Q_max) - Q_min |
| 1227 | // or equivalently: |
| 1228 | // j*K + 0 <u32 clamp(R - Q_min, 0, Q_max - Q_min) |
| 1229 | |
| 1230 | // Case C: Overflow in the 64-bit domain |
| 1231 | // Number line: |
| 1232 | // |..Q_max-2^64 . . 0 . . . Q_min..| s64 overflow |
| 1233 | // |
| 1234 | // if Q_min >=s64 0 but Q_max <s64 0, then use this test: |
| 1235 | // j*K + 0 <u32 clamp(R, Q_min, R) - Q_min |
| 1236 | // or equivalently: |
| 1237 | // j*K + 0 <u32 clamp(R - Q_min, 0, R - Q_min) |
| 1238 | // or also equivalently: |
| 1239 | // j*K + 0 <u32 max(0, R - Q_min) |
| 1240 | // |
| 1241 | // Here the clamp function is a simple 64-bit min/max: |
| 1242 | // clamp(X, L, H) := max(L, min(X, H)) |
| 1243 | // When degenerately L > H, it returns L not H. |
| 1244 | // |
| 1245 | // Tests above can be merged into a single one: |
| 1246 | // L_clamp = Q_min < 0 ? 0 : Q_min |
| 1247 | // H_clamp = Q_max < Q_min ? R : Q_max |
| 1248 | // j*K + Q_min - L_clamp <u32 clamp(R, L_clamp, H_clamp) - L_clamp |
| 1249 | // or equivalently: |
| 1250 | // j*K + Q_min - L_clamp <u32 clamp(R - L_clamp, 0, H_clamp - L_clamp) |
| 1251 | // |
| 1252 | // Readers may find the equivalent forms easier to reason about, but the forms given first generate better code. |
| 1253 | |
| 1254 | void PhaseIdealLoop::transform_long_range_checks(int stride_con, const Node_List &range_checks, Node* outer_phi, |
| 1255 | Node* inner_iters_actual_int, Node* inner_phi, |
| 1256 | Node* iv_add, LoopNode* inner_head) { |
| 1257 | Node* long_zero = _igvn.longcon(0); |
| 1258 | set_ctrl(long_zero, C->root()); |
| 1259 | |
| 1260 | for (uint i = 0; i < range_checks.size(); i++) { |
| 1261 | ProjNode* proj = range_checks.at(i)->as_Proj(); |
| 1262 | ProjNode* unc_proj = proj->other_if_proj(); |
| 1263 | RangeCheckNode* rc = proj->in(0)->as_RangeCheck(); |
| 1264 | jlong scale = 0; |
| 1265 | Node* offset = NULL__null; |
| 1266 | Node* rc_bol = rc->in(1); |
| 1267 | Node* rc_cmp = rc_bol->in(1); |
| 1268 | if (rc_cmp->Opcode() == Op_CmpU) { |
| 1269 | // could be shared and have already been taken care of |
| 1270 | continue; |
| 1271 | } |
| 1272 | bool converted = false; |
| 1273 | bool ok = is_scaled_iv_plus_offset(rc_cmp->in(1), iv_add, &scale, &offset, T_LONG, &converted); |
| 1274 | assert(ok, "inconsistent: was tested before")do { if (!(ok)) { (*g_assert_poison) = 'X';; report_vm_error( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1274, "assert(" "ok" ") failed", "inconsistent: was tested before" ); ::breakpoint(); } } while (0); |
| 1275 | Node* range = rc_cmp->in(2); |
| 1276 | Node* c = rc->in(0); |
| 1277 | Node* entry_control = inner_head->in(LoopNode::EntryControl); |
| 1278 | |
| 1279 | Node* R = range; |
| 1280 | Node* K = _igvn.longcon(scale); |
| 1281 | set_ctrl(K, this->C->root()); |
| 1282 | |
| 1283 | Node* L = offset; |
| 1284 | |
| 1285 | if (converted) { |
| 1286 | // This converts: |
| 1287 | // i*K + L <u64 R |
| 1288 | // with K an int into: |
| 1289 | // i*(long)K + L <u64 unsigned_min((long)max_jint + L + 1, R) |
| 1290 | // to protect against an overflow of i*K |
| 1291 | // |
| 1292 | // Because if i*K overflows, there are K,L where: |
| 1293 | // i*K + L <u64 R is false |
| 1294 | // when |
| 1295 | // i*(long)K is > (long)max_jint and < R |
| 1296 | // and so i*(long)K + L <u64 R is true |
| 1297 | // As a consequence simply converting: |
| 1298 | // i*K + L <u64 R to i*(long)K + L <u64 R could cause incorrect execution |
| 1299 | // |
| 1300 | // It's always true that: |
| 1301 | // i*K <u64 (long)max_jint + 1 |
| 1302 | // which implies i*K + L <u64 (long)max_jint + 1 + L |
| 1303 | // As a consequence: |
| 1304 | // i*(long)K + L <u64 unsigned_min((long)max_jint + L + 1, R) |
| 1305 | // is always false in case of overflow of i*K |
| 1306 | // |
| 1307 | // Note, there are K,L where i*K overflows and |
| 1308 | // i*K + L <u64 R is true, but |
| 1309 | // i*(long)K + L <u64 unsigned_min((long)max_jint + L + 1, R) is false |
| 1310 | // So this transformation could cause spurious deoptimizations and failed range check elimination |
| 1311 | // (but not incorrect execution) for unlikely corner cases with overflow |
| 1312 | Node* max_jint_plus_one_long = _igvn.longcon((jlong)max_jint + 1); |
| 1313 | set_ctrl(max_jint_plus_one_long, C->root()); |
| 1314 | Node* max_range = new AddLNode(max_jint_plus_one_long, L); |
| 1315 | register_new_node(max_range, entry_control); |
| 1316 | R = MaxNode::unsigned_min(R, max_range, TypeLong::POS, _igvn); |
| 1317 | set_subtree_ctrl(R, true); |
| 1318 | } |
| 1319 | |
| 1320 | Node* C = outer_phi; |
| 1321 | Node* Z_2 = new ConvI2LNode(inner_iters_actual_int, TypeLong::LONG); |
| 1322 | register_new_node(Z_2, entry_control); |
| 1323 | |
| 1324 | // Start with 64-bit values: |
| 1325 | // i*K + L <u64 R |
| 1326 | // (C+j)*K + L <u64 R |
| 1327 | // j*K + L_2 <u64 R where L_2 = C*K+L |
| 1328 | Node* L_2 = new MulLNode(C, K); |
| 1329 | register_new_node(L_2, entry_control); |
| 1330 | L_2 = new AddLNode(L_2, L); |
| 1331 | register_new_node(L_2, entry_control); |
| 1332 | |
| 1333 | // Compute endpoints of the range of values j*K. |
| 1334 | // Q_min = (j=0)*K + L_2; Q_max = (j=Z_2)*K + L_2 |
| 1335 | Node* Q_min = L_2; |
| 1336 | Node* Q_max = new MulLNode(Z_2, K); |
| 1337 | register_new_node(Q_max, entry_control); |
| 1338 | Q_max = new AddLNode(Q_max, L_2); |
| 1339 | register_new_node(Q_max, entry_control); |
| 1340 | |
| 1341 | // L_clamp = Q_min < 0 ? 0 : Q_min |
| 1342 | Node* Q_min_cmp = new CmpLNode(Q_min, long_zero); |
| 1343 | register_new_node(Q_min_cmp, entry_control); |
| 1344 | Node* Q_min_bool = new BoolNode(Q_min_cmp, BoolTest::lt); |
| 1345 | register_new_node(Q_min_bool, entry_control); |
| 1346 | Node* L_clamp = new CMoveLNode(Q_min_bool, Q_min, long_zero, TypeLong::LONG); |
| 1347 | register_new_node(L_clamp, entry_control); |
| 1348 | |
| 1349 | // H_clamp = Q_max < Q_min ? R : Q_max |
| 1350 | Node* Q_max_cmp = new CmpLNode(Q_max, Q_min); |
| 1351 | register_new_node(Q_max_cmp, entry_control); |
| 1352 | Node* Q_max_bool = new BoolNode(Q_max_cmp, BoolTest::lt); |
| 1353 | register_new_node(Q_max_bool, entry_control); |
| 1354 | Node* H_clamp = new CMoveLNode(Q_max_bool, Q_max, R, TypeLong::LONG); |
| 1355 | register_new_node(H_clamp, entry_control); |
| 1356 | |
| 1357 | // R_2 = clamp(R, L_clamp, H_clamp) - L_clamp |
| 1358 | // that is: R_2 = clamp(R, L_clamp, H_clamp) if Q_min < 0 |
| 1359 | // or: R_2 = clamp(R, L_clamp, H_clamp) - Q_min if Q_min > 0 |
| 1360 | Node* R_2 = clamp(R, L_clamp, H_clamp); |
| 1361 | R_2 = new SubLNode(R_2, L_clamp); |
| 1362 | register_new_node(R_2, entry_control); |
| 1363 | R_2 = new ConvL2INode(R_2, TypeInt::POS); |
| 1364 | register_new_node(R_2, entry_control); |
| 1365 | |
| 1366 | // Q = Q_min - L_clamp |
| 1367 | // that is: Q = Q_min - 0 if Q_min < 0 |
| 1368 | // or: Q = Q_min - Q_min = 0 if Q_min > 0 |
| 1369 | Node* Q = new SubLNode(Q_min, L_clamp); |
| 1370 | register_new_node(Q, entry_control); |
| 1371 | Q = new ConvL2INode(Q, TypeInt::INT); |
| 1372 | register_new_node(Q, entry_control); |
| 1373 | |
| 1374 | // Transform the range check |
| 1375 | K = _igvn.intcon(checked_cast<int>(scale)); |
| 1376 | set_ctrl(K, this->C->root()); |
| 1377 | Node* scaled_iv = new MulINode(inner_phi, K); |
| 1378 | register_new_node(scaled_iv, c); |
| 1379 | Node* scaled_iv_plus_offset = scaled_iv_plus_offset = new AddINode(scaled_iv, Q); |
| 1380 | register_new_node(scaled_iv_plus_offset, c); |
| 1381 | |
| 1382 | Node* new_rc_cmp = new CmpUNode(scaled_iv_plus_offset, R_2); |
| 1383 | register_new_node(new_rc_cmp, c); |
| 1384 | |
| 1385 | _igvn.replace_input_of(rc_bol, 1, new_rc_cmp); |
| 1386 | } |
| 1387 | } |
| 1388 | |
| 1389 | Node* PhaseIdealLoop::clamp(Node* R, Node* L, Node* H) { |
| 1390 | Node* min = MaxNode::signed_min(R, H, TypeLong::LONG, _igvn); |
| 1391 | set_subtree_ctrl(min, true); |
| 1392 | Node* max = MaxNode::signed_max(L, min, TypeLong::LONG, _igvn); |
| 1393 | set_subtree_ctrl(max, true); |
| 1394 | return max; |
| 1395 | } |
| 1396 | |
| 1397 | LoopNode* PhaseIdealLoop::create_inner_head(IdealLoopTree* loop, BaseCountedLoopNode* head, |
| 1398 | IfNode* exit_test) { |
| 1399 | LoopNode* new_inner_head = new LoopNode(head->in(1), head->in(2)); |
| 1400 | IfNode* new_inner_exit = new IfNode(exit_test->in(0), exit_test->in(1), exit_test->_prob, exit_test->_fcnt); |
| 1401 | _igvn.register_new_node_with_optimizer(new_inner_head); |
| 1402 | _igvn.register_new_node_with_optimizer(new_inner_exit); |
| 1403 | loop->_body.push(new_inner_head); |
| 1404 | loop->_body.push(new_inner_exit); |
| 1405 | loop->_body.yank(head); |
| 1406 | loop->_body.yank(exit_test); |
| 1407 | set_loop(new_inner_head, loop); |
| 1408 | set_loop(new_inner_exit, loop); |
| 1409 | set_idom(new_inner_head, idom(head), dom_depth(head)); |
| 1410 | set_idom(new_inner_exit, idom(exit_test), dom_depth(exit_test)); |
| 1411 | lazy_replace(head, new_inner_head); |
| 1412 | lazy_replace(exit_test, new_inner_exit); |
| 1413 | loop->_head = new_inner_head; |
| 1414 | return new_inner_head; |
| 1415 | } |
| 1416 | |
| 1417 | #ifdef ASSERT1 |
| 1418 | void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, BasicType bt) { |
| 1419 | Node* back_control = loop_exit_control(x, loop); |
| 1420 | assert(back_control != NULL, "no back control")do { if (!(back_control != __null)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1420, "assert(" "back_control != __null" ") failed", "no back control" ); ::breakpoint(); } } while (0); |
| 1421 | |
| 1422 | BoolTest::mask mask = BoolTest::illegal; |
| 1423 | float cl_prob = 0; |
| 1424 | Node* incr = NULL__null; |
| 1425 | Node* limit = NULL__null; |
| 1426 | |
| 1427 | Node* cmp = loop_exit_test(back_control, loop, incr, limit, mask, cl_prob); |
| 1428 | assert(cmp != NULL && cmp->Opcode() == Op_Cmp(bt), "no exit test")do { if (!(cmp != __null && cmp->Opcode() == Op_Cmp (bt))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1428, "assert(" "cmp != __null && cmp->Opcode() == Op_Cmp(bt)" ") failed", "no exit test"); ::breakpoint(); } } while (0); |
| 1429 | |
| 1430 | Node* phi_incr = NULL__null; |
| 1431 | incr = loop_iv_incr(incr, x, loop, phi_incr); |
| 1432 | assert(incr != NULL && incr->Opcode() == Op_Add(bt), "no incr")do { if (!(incr != __null && incr->Opcode() == Op_Add (bt))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1432, "assert(" "incr != __null && incr->Opcode() == Op_Add(bt)" ") failed", "no incr"); ::breakpoint(); } } while (0); |
| 1433 | |
| 1434 | Node* xphi = NULL__null; |
| 1435 | Node* stride = loop_iv_stride(incr, loop, xphi); |
| 1436 | |
| 1437 | assert(stride != NULL, "no stride")do { if (!(stride != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1437, "assert(" "stride != __null" ") failed", "no stride") ; ::breakpoint(); } } while (0); |
| 1438 | |
| 1439 | PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); |
| 1440 | |
| 1441 | assert(phi != NULL && phi->in(LoopNode::LoopBackControl) == incr, "No phi")do { if (!(phi != __null && phi->in(LoopNode::LoopBackControl ) == incr)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1441, "assert(" "phi != __null && phi->in(LoopNode::LoopBackControl) == incr" ") failed", "No phi"); ::breakpoint(); } } while (0); |
| 1442 | |
| 1443 | jlong stride_con = stride->get_integer_as_long(bt); |
| 1444 | |
| 1445 | assert(condition_stride_ok(mask, stride_con), "illegal condition")do { if (!(condition_stride_ok(mask, stride_con))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1445, "assert(" "condition_stride_ok(mask, stride_con)" ") failed" , "illegal condition"); ::breakpoint(); } } while (0); |
| 1446 | |
| 1447 | assert(mask != BoolTest::ne, "unexpected condition")do { if (!(mask != BoolTest::ne)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1447, "assert(" "mask != BoolTest::ne" ") failed", "unexpected condition" ); ::breakpoint(); } } while (0); |
| 1448 | assert(phi_incr == NULL, "bad loop shape")do { if (!(phi_incr == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1448, "assert(" "phi_incr == __null" ") failed", "bad loop shape" ); ::breakpoint(); } } while (0); |
| 1449 | assert(cmp->in(1) == incr, "bad exit test shape")do { if (!(cmp->in(1) == incr)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1449, "assert(" "cmp->in(1) == incr" ") failed", "bad exit test shape" ); ::breakpoint(); } } while (0); |
| 1450 | |
| 1451 | // Safepoint on backedge not supported |
| 1452 | assert(x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint, "no safepoint on backedge")do { if (!(x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint)) { (*g_assert_poison) = 'X';; report_vm_error( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1452, "assert(" "x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint" ") failed", "no safepoint on backedge"); ::breakpoint(); } } while (0); |
| 1453 | } |
| 1454 | #endif |
| 1455 | |
| 1456 | #ifdef ASSERT1 |
| 1457 | // convert an int counted loop to a long counted to stress handling of |
| 1458 | // long counted loops |
| 1459 | bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop) { |
| 1460 | Unique_Node_List iv_nodes; |
| 1461 | Node_List old_new; |
| 1462 | iv_nodes.push(cmp); |
| 1463 | bool failed = false; |
| 1464 | |
| 1465 | for (uint i = 0; i < iv_nodes.size() && !failed; i++) { |
| 1466 | Node* n = iv_nodes.at(i); |
| 1467 | switch(n->Opcode()) { |
| 1468 | case Op_Phi: { |
| 1469 | Node* clone = new PhiNode(n->in(0), TypeLong::LONG); |
| 1470 | old_new.map(n->_idx, clone); |
| 1471 | break; |
| 1472 | } |
| 1473 | case Op_CmpI: { |
| 1474 | Node* clone = new CmpLNode(NULL__null, NULL__null); |
| 1475 | old_new.map(n->_idx, clone); |
| 1476 | break; |
| 1477 | } |
| 1478 | case Op_AddI: { |
| 1479 | Node* clone = new AddLNode(NULL__null, NULL__null); |
| 1480 | old_new.map(n->_idx, clone); |
| 1481 | break; |
| 1482 | } |
| 1483 | case Op_CastII: { |
| 1484 | failed = true; |
| 1485 | break; |
| 1486 | } |
| 1487 | default: |
| 1488 | DEBUG_ONLY(n->dump())n->dump(); |
| 1489 | fatal("unexpected")do { (*g_assert_poison) = 'X';; report_fatal(INTERNAL_ERROR, "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1489, "unexpected"); ::breakpoint(); } while (0); |
| 1490 | } |
| 1491 | |
| 1492 | for (uint i = 1; i < n->req(); i++) { |
| 1493 | Node* in = n->in(i); |
| 1494 | if (in == NULL__null) { |
| 1495 | continue; |
| 1496 | } |
| 1497 | if (loop->is_member(get_loop(get_ctrl(in)))) { |
| 1498 | iv_nodes.push(in); |
| 1499 | } |
| 1500 | } |
| 1501 | } |
| 1502 | |
| 1503 | if (failed) { |
| 1504 | for (uint i = 0; i < iv_nodes.size(); i++) { |
| 1505 | Node* n = iv_nodes.at(i); |
| 1506 | Node* clone = old_new[n->_idx]; |
| 1507 | if (clone != NULL__null) { |
| 1508 | _igvn.remove_dead_node(clone); |
| 1509 | } |
| 1510 | } |
| 1511 | return false; |
| 1512 | } |
| 1513 | |
| 1514 | for (uint i = 0; i < iv_nodes.size(); i++) { |
| 1515 | Node* n = iv_nodes.at(i); |
| 1516 | Node* clone = old_new[n->_idx]; |
| 1517 | for (uint i = 1; i < n->req(); i++) { |
| 1518 | Node* in = n->in(i); |
| 1519 | if (in == NULL__null) { |
| 1520 | continue; |
| 1521 | } |
| 1522 | Node* in_clone = old_new[in->_idx]; |
| 1523 | if (in_clone == NULL__null) { |
| 1524 | assert(_igvn.type(in)->isa_int(), "")do { if (!(_igvn.type(in)->isa_int())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1524, "assert(" "_igvn.type(in)->isa_int()" ") failed", "" ); ::breakpoint(); } } while (0); |
| 1525 | in_clone = new ConvI2LNode(in); |
| 1526 | _igvn.register_new_node_with_optimizer(in_clone); |
| 1527 | set_subtree_ctrl(in_clone, false); |
| 1528 | } |
| 1529 | if (in_clone->in(0) == NULL__null) { |
| 1530 | in_clone->set_req(0, C->top()); |
| 1531 | clone->set_req(i, in_clone); |
| 1532 | in_clone->set_req(0, NULL__null); |
| 1533 | } else { |
| 1534 | clone->set_req(i, in_clone); |
| 1535 | } |
| 1536 | } |
| 1537 | _igvn.register_new_node_with_optimizer(clone); |
| 1538 | } |
| 1539 | set_ctrl(old_new[phi->_idx], phi->in(0)); |
| 1540 | |
| 1541 | for (uint i = 0; i < iv_nodes.size(); i++) { |
| 1542 | Node* n = iv_nodes.at(i); |
| 1543 | Node* clone = old_new[n->_idx]; |
| 1544 | set_subtree_ctrl(clone, false); |
| 1545 | Node* m = n->Opcode() == Op_CmpI ? clone : NULL__null; |
| 1546 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
| 1547 | Node* u = n->fast_out(i); |
| 1548 | if (iv_nodes.member(u)) { |
| 1549 | continue; |
| 1550 | } |
| 1551 | if (m == NULL__null) { |
| 1552 | m = new ConvL2INode(clone); |
| 1553 | _igvn.register_new_node_with_optimizer(m); |
| 1554 | set_subtree_ctrl(m, false); |
| 1555 | } |
| 1556 | _igvn.rehash_node_delayed(u); |
| 1557 | int nb = u->replace_edge(n, m, &_igvn); |
| 1558 | --i, imax -= nb; |
| 1559 | } |
| 1560 | } |
| 1561 | return true; |
| 1562 | } |
| 1563 | #endif |
| 1564 | |
| 1565 | //------------------------------is_counted_loop-------------------------------- |
| 1566 | bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt) { |
| 1567 | PhaseGVN *gvn = &_igvn; |
| 1568 | |
| 1569 | Node* back_control = loop_exit_control(x, loop); |
| 1570 | if (back_control == NULL__null) { |
| 1571 | return false; |
| 1572 | } |
| 1573 | |
| 1574 | BoolTest::mask bt = BoolTest::illegal; |
| 1575 | float cl_prob = 0; |
| 1576 | Node* incr = NULL__null; |
| 1577 | Node* limit = NULL__null; |
| 1578 | Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); |
| 1579 | if (cmp == NULL__null || cmp->Opcode() != Op_Cmp(iv_bt)) { |
| 1580 | return false; // Avoid pointer & float & 64-bit compares |
| 1581 | } |
| 1582 | |
| 1583 | // Trip-counter increment must be commutative & associative. |
| 1584 | if (incr->Opcode() == Op_Cast(iv_bt)) { |
| 1585 | incr = incr->in(1); |
| 1586 | } |
| 1587 | |
| 1588 | Node* phi_incr = NULL__null; |
| 1589 | incr = loop_iv_incr(incr, x, loop, phi_incr); |
| 1590 | if (incr == NULL__null) { |
| 1591 | return false; |
| 1592 | } |
| 1593 | |
| 1594 | Node* trunc1 = NULL__null; |
| 1595 | Node* trunc2 = NULL__null; |
| 1596 | const TypeInteger* iv_trunc_t = NULL__null; |
| 1597 | Node* orig_incr = incr; |
| 1598 | if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t, iv_bt))) { |
| 1599 | return false; // Funny increment opcode |
| 1600 | } |
| 1601 | assert(incr->Opcode() == Op_Add(iv_bt), "wrong increment code")do { if (!(incr->Opcode() == Op_Add(iv_bt))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1601, "assert(" "incr->Opcode() == Op_Add(iv_bt)" ") failed" , "wrong increment code"); ::breakpoint(); } } while (0); |
| 1602 | |
| 1603 | Node* xphi = NULL__null; |
| 1604 | Node* stride = loop_iv_stride(incr, loop, xphi); |
| 1605 | |
| 1606 | if (stride == NULL__null) { |
| 1607 | return false; |
| 1608 | } |
| 1609 | |
| 1610 | if (xphi->Opcode() == Op_Cast(iv_bt)) { |
| 1611 | xphi = xphi->in(1); |
| 1612 | } |
| 1613 | |
| 1614 | // Stride must be constant |
| 1615 | jlong stride_con = stride->get_integer_as_long(iv_bt); |
| 1616 | assert(stride_con != 0, "missed some peephole opt")do { if (!(stride_con != 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1616, "assert(" "stride_con != 0" ") failed", "missed some peephole opt" ); ::breakpoint(); } } while (0); |
| 1617 | |
| 1618 | PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); |
| 1619 | |
| 1620 | if (phi == NULL__null || |
| 1621 | (trunc1 == NULL__null && phi->in(LoopNode::LoopBackControl) != incr) || |
| 1622 | (trunc1 != NULL__null && phi->in(LoopNode::LoopBackControl) != trunc1)) { |
| 1623 | return false; |
| 1624 | } |
| 1625 | |
| 1626 | if (x->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint && |
| 1627 | ((iv_bt == T_INT && LoopStripMiningIter != 0) || |
| 1628 | iv_bt == T_LONG)) { |
| 1629 | // Leaving the safepoint on the backedge and creating a |
| 1630 | // CountedLoop will confuse optimizations. We can't move the |
| 1631 | // safepoint around because its jvm state wouldn't match a new |
| 1632 | // location. Give up on that loop. |
| 1633 | return false; |
| 1634 | } |
| 1635 | |
| 1636 | Node* iftrue = back_control; |
| 1637 | uint iftrue_op = iftrue->Opcode(); |
| 1638 | Node* iff = iftrue->in(0); |
| 1639 | BoolNode* test = iff->in(1)->as_Bool(); |
| 1640 | |
| 1641 | const TypeInteger* limit_t = gvn->type(limit)->is_integer(iv_bt); |
| 1642 | if (trunc1 != NULL__null) { |
| 1643 | // When there is a truncation, we must be sure that after the truncation |
| 1644 | // the trip counter will end up higher than the limit, otherwise we are looking |
| 1645 | // at an endless loop. Can happen with range checks. |
| 1646 | |
| 1647 | // Example: |
| 1648 | // int i = 0; |
| 1649 | // while (true) |
| 1650 | // sum + = array[i]; |
| 1651 | // i++; |
| 1652 | // i = i && 0x7fff; |
| 1653 | // } |
| 1654 | // |
| 1655 | // If the array is shorter than 0x8000 this exits through a AIOOB |
| 1656 | // - Counted loop transformation is ok |
| 1657 | // If the array is longer then this is an endless loop |
| 1658 | // - No transformation can be done. |
| 1659 | |
| 1660 | const TypeInteger* incr_t = gvn->type(orig_incr)->is_integer(iv_bt); |
| 1661 | if (limit_t->hi_as_long() > incr_t->hi_as_long()) { |
| 1662 | // if the limit can have a higher value than the increment (before the phi) |
| 1663 | return false; |
| 1664 | } |
| 1665 | } |
| 1666 | |
| 1667 | Node *init_trip = phi->in(LoopNode::EntryControl); |
| 1668 | |
| 1669 | // If iv trunc type is smaller than int, check for possible wrap. |
| 1670 | if (!TypeInteger::bottom(iv_bt)->higher_equal(iv_trunc_t)) { |
| 1671 | assert(trunc1 != NULL, "must have found some truncation")do { if (!(trunc1 != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1671, "assert(" "trunc1 != __null" ") failed", "must have found some truncation" ); ::breakpoint(); } } while (0); |
| 1672 | |
| 1673 | // Get a better type for the phi (filtered thru if's) |
| 1674 | const TypeInteger* phi_ft = filtered_type(phi); |
| 1675 | |
| 1676 | // Can iv take on a value that will wrap? |
| 1677 | // |
| 1678 | // Ensure iv's limit is not within "stride" of the wrap value. |
| 1679 | // |
| 1680 | // Example for "short" type |
| 1681 | // Truncation ensures value is in the range -32768..32767 (iv_trunc_t) |
| 1682 | // If the stride is +10, then the last value of the induction |
| 1683 | // variable before the increment (phi_ft->_hi) must be |
| 1684 | // <= 32767 - 10 and (phi_ft->_lo) must be >= -32768 to |
| 1685 | // ensure no truncation occurs after the increment. |
| 1686 | |
| 1687 | if (stride_con > 0) { |
| 1688 | if (iv_trunc_t->hi_as_long() - phi_ft->hi_as_long() < stride_con || |
| 1689 | iv_trunc_t->lo_as_long() > phi_ft->lo_as_long()) { |
| 1690 | return false; // truncation may occur |
| 1691 | } |
| 1692 | } else if (stride_con < 0) { |
| 1693 | if (iv_trunc_t->lo_as_long() - phi_ft->lo_as_long() > stride_con || |
| 1694 | iv_trunc_t->hi_as_long() < phi_ft->hi_as_long()) { |
| 1695 | return false; // truncation may occur |
| 1696 | } |
| 1697 | } |
| 1698 | // No possibility of wrap so truncation can be discarded |
| 1699 | // Promote iv type to Int |
| 1700 | } else { |
| 1701 | assert(trunc1 == NULL && trunc2 == NULL, "no truncation for int")do { if (!(trunc1 == __null && trunc2 == __null)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1701, "assert(" "trunc1 == __null && trunc2 == __null" ") failed", "no truncation for int"); ::breakpoint(); } } while (0); |
| 1702 | } |
| 1703 | |
| 1704 | if (!condition_stride_ok(bt, stride_con)) { |
| 1705 | return false; |
| 1706 | } |
| 1707 | |
| 1708 | const TypeInteger* init_t = gvn->type(init_trip)->is_integer(iv_bt); |
| 1709 | |
| 1710 | if (stride_con > 0) { |
| 1711 | if (init_t->lo_as_long() > max_signed_integer(iv_bt) - stride_con) { |
| 1712 | return false; // cyclic loop |
| 1713 | } |
| 1714 | } else { |
| 1715 | if (init_t->hi_as_long() < min_signed_integer(iv_bt) - stride_con) { |
| 1716 | return false; // cyclic loop |
| 1717 | } |
| 1718 | } |
| 1719 | |
| 1720 | if (phi_incr != NULL__null && bt != BoolTest::ne) { |
| 1721 | // check if there is a possiblity of IV overflowing after the first increment |
| 1722 | if (stride_con > 0) { |
| 1723 | if (init_t->hi_as_long() > max_signed_integer(iv_bt) - stride_con) { |
| 1724 | return false; |
| 1725 | } |
| 1726 | } else { |
| 1727 | if (init_t->lo_as_long() < min_signed_integer(iv_bt) - stride_con) { |
| 1728 | return false; |
| 1729 | } |
| 1730 | } |
| 1731 | } |
| 1732 | |
| 1733 | // ================================================= |
| 1734 | // ---- SUCCESS! Found A Trip-Counted Loop! ----- |
| 1735 | // |
| 1736 | assert(x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop, "regular loops only")do { if (!(x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1736, "assert(" "x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop" ") failed", "regular loops only"); ::breakpoint(); } } while (0); |
| 1737 | C->print_method(PHASE_BEFORE_CLOOPS, 3); |
| 1738 | |
| 1739 | // =================================================== |
| 1740 | // Generate loop limit check to avoid integer overflow |
| 1741 | // in cases like next (cyclic loops): |
| 1742 | // |
| 1743 | // for (i=0; i <= max_jint; i++) {} |
| 1744 | // for (i=0; i < max_jint; i+=2) {} |
| 1745 | // |
| 1746 | // |
| 1747 | // Limit check predicate depends on the loop test: |
| 1748 | // |
| 1749 | // for(;i != limit; i++) --> limit <= (max_jint) |
| 1750 | // for(;i < limit; i+=stride) --> limit <= (max_jint - stride + 1) |
| 1751 | // for(;i <= limit; i+=stride) --> limit <= (max_jint - stride ) |
| 1752 | // |
| 1753 | |
| 1754 | // Check if limit is excluded to do more precise int overflow check. |
| 1755 | bool incl_limit = (bt == BoolTest::le || bt == BoolTest::ge); |
| 1756 | jlong stride_m = stride_con - (incl_limit ? 0 : (stride_con > 0 ? 1 : -1)); |
| 1757 | |
| 1758 | // If compare points directly to the phi we need to adjust |
| 1759 | // the compare so that it points to the incr. Limit have |
| 1760 | // to be adjusted to keep trip count the same and the |
| 1761 | // adjusted limit should be checked for int overflow. |
| 1762 | Node* adjusted_limit = limit; |
| 1763 | if (phi_incr != NULL__null) { |
| 1764 | stride_m += stride_con; |
| 1765 | } |
| 1766 | |
| 1767 | Node *init_control = x->in(LoopNode::EntryControl); |
| 1768 | |
| 1769 | int sov = check_stride_overflow(stride_m, limit_t, iv_bt); |
| 1770 | // If sov==0, limit's type always satisfies the condition, for |
| 1771 | // example, when it is an array length. |
| 1772 | if (sov != 0) { |
| 1773 | if (sov < 0) { |
| 1774 | return false; // Bailout: integer overflow is certain. |
| 1775 | } |
| 1776 | assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed")do { if (!(!x->as_Loop()->is_loop_nest_inner_loop())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1776, "assert(" "!x->as_Loop()->is_loop_nest_inner_loop()" ") failed", "loop was transformed"); ::breakpoint(); } } while (0); |
| 1777 | // Generate loop's limit check. |
| 1778 | // Loop limit check predicate should be near the loop. |
| 1779 | ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check); |
| 1780 | if (!limit_check_proj) { |
| 1781 | // The limit check predicate is not generated if this method trapped here before. |
| 1782 | #ifdef ASSERT1 |
| 1783 | if (TraceLoopLimitCheck) { |
| 1784 | tty->print("missing loop limit check:"); |
| 1785 | loop->dump_head(); |
| 1786 | x->dump(1); |
| 1787 | } |
| 1788 | #endif |
| 1789 | return false; |
| 1790 | } |
| 1791 | |
| 1792 | IfNode* check_iff = limit_check_proj->in(0)->as_If(); |
| 1793 | |
| 1794 | if (!is_dominator(get_ctrl(limit), check_iff->in(0))) { |
| 1795 | return false; |
| 1796 | } |
| 1797 | |
| 1798 | Node* cmp_limit; |
| 1799 | Node* bol; |
| 1800 | |
| 1801 | if (stride_con > 0) { |
| 1802 | cmp_limit = CmpNode::make(limit, _igvn.integercon(max_signed_integer(iv_bt) - stride_m, iv_bt), iv_bt); |
| 1803 | bol = new BoolNode(cmp_limit, BoolTest::le); |
| 1804 | } else { |
| 1805 | cmp_limit = CmpNode::make(limit, _igvn.integercon(min_signed_integer(iv_bt) - stride_m, iv_bt), iv_bt); |
| 1806 | bol = new BoolNode(cmp_limit, BoolTest::ge); |
| 1807 | } |
| 1808 | |
| 1809 | insert_loop_limit_check(limit_check_proj, cmp_limit, bol); |
| 1810 | } |
| 1811 | |
| 1812 | // Now we need to canonicalize loop condition. |
| 1813 | if (bt == BoolTest::ne) { |
| 1814 | assert(stride_con == 1 || stride_con == -1, "simple increment only")do { if (!(stride_con == 1 || stride_con == -1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1814, "assert(" "stride_con == 1 || stride_con == -1" ") failed" , "simple increment only"); ::breakpoint(); } } while (0); |
| 1815 | if (stride_con > 0 && init_t->hi_as_long() < limit_t->lo_as_long()) { |
| 1816 | // 'ne' can be replaced with 'lt' only when init < limit. |
| 1817 | bt = BoolTest::lt; |
| 1818 | } else if (stride_con < 0 && init_t->lo_as_long() > limit_t->hi_as_long()) { |
| 1819 | // 'ne' can be replaced with 'gt' only when init > limit. |
| 1820 | bt = BoolTest::gt; |
| 1821 | } else { |
| 1822 | ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check); |
| 1823 | if (!limit_check_proj) { |
| 1824 | // The limit check predicate is not generated if this method trapped here before. |
| 1825 | #ifdef ASSERT1 |
| 1826 | if (TraceLoopLimitCheck) { |
| 1827 | tty->print("missing loop limit check:"); |
| 1828 | loop->dump_head(); |
| 1829 | x->dump(1); |
| 1830 | } |
| 1831 | #endif |
| 1832 | return false; |
| 1833 | } |
| 1834 | IfNode* check_iff = limit_check_proj->in(0)->as_If(); |
| 1835 | |
| 1836 | if (!is_dominator(get_ctrl(limit), check_iff->in(0)) || |
| 1837 | !is_dominator(get_ctrl(init_trip), check_iff->in(0))) { |
| 1838 | return false; |
| 1839 | } |
| 1840 | |
| 1841 | Node* cmp_limit; |
| 1842 | Node* bol; |
| 1843 | |
| 1844 | if (stride_con > 0) { |
| 1845 | cmp_limit = CmpNode::make(init_trip, limit, iv_bt); |
| 1846 | bol = new BoolNode(cmp_limit, BoolTest::lt); |
| 1847 | } else { |
| 1848 | cmp_limit = CmpNode::make(init_trip, limit, iv_bt); |
| 1849 | bol = new BoolNode(cmp_limit, BoolTest::gt); |
| 1850 | } |
| 1851 | |
| 1852 | insert_loop_limit_check(limit_check_proj, cmp_limit, bol); |
| 1853 | |
| 1854 | if (stride_con > 0) { |
| 1855 | // 'ne' can be replaced with 'lt' only when init < limit. |
| 1856 | bt = BoolTest::lt; |
| 1857 | } else if (stride_con < 0) { |
| 1858 | // 'ne' can be replaced with 'gt' only when init > limit. |
| 1859 | bt = BoolTest::gt; |
| 1860 | } |
| 1861 | } |
| 1862 | } |
| 1863 | |
| 1864 | #ifdef ASSERT1 |
| 1865 | if (iv_bt == T_INT && |
| 1866 | !x->as_Loop()->is_loop_nest_inner_loop() && |
| 1867 | StressLongCountedLoop > 0 && |
| 1868 | trunc1 == NULL__null && |
| 1869 | convert_to_long_loop(cmp, phi, loop)) { |
| 1870 | return false; |
| 1871 | } |
| 1872 | #endif |
| 1873 | |
| 1874 | if (phi_incr != NULL__null) { |
| 1875 | // If compare points directly to the phi we need to adjust |
| 1876 | // the compare so that it points to the incr. Limit have |
| 1877 | // to be adjusted to keep trip count the same and we |
| 1878 | // should avoid int overflow. |
| 1879 | // |
| 1880 | // i = init; do {} while(i++ < limit); |
| 1881 | // is converted to |
| 1882 | // i = init; do {} while(++i < limit+1); |
| 1883 | // |
| 1884 | adjusted_limit = gvn->transform(AddNode::make(limit, stride, iv_bt)); |
| 1885 | } |
| 1886 | |
| 1887 | if (incl_limit) { |
| 1888 | // The limit check guaranties that 'limit <= (max_jint - stride)' so |
| 1889 | // we can convert 'i <= limit' to 'i < limit+1' since stride != 0. |
| 1890 | // |
| 1891 | Node* one = (stride_con > 0) ? gvn->integercon( 1, iv_bt) : gvn->integercon(-1, iv_bt); |
| 1892 | adjusted_limit = gvn->transform(AddNode::make(adjusted_limit, one, iv_bt)); |
| 1893 | if (bt == BoolTest::le) |
| 1894 | bt = BoolTest::lt; |
| 1895 | else if (bt == BoolTest::ge) |
| 1896 | bt = BoolTest::gt; |
| 1897 | else |
| 1898 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1898); ::breakpoint(); } while (0); |
| 1899 | } |
| 1900 | set_subtree_ctrl(adjusted_limit, false); |
| 1901 | |
| 1902 | if (iv_bt == T_INT && LoopStripMiningIter == 0) { |
| 1903 | // Check for SafePoint on backedge and remove |
| 1904 | Node *sfpt = x->in(LoopNode::LoopBackControl); |
| 1905 | if (sfpt->Opcode() == Op_SafePoint && is_deleteable_safept(sfpt)) { |
| 1906 | lazy_replace( sfpt, iftrue ); |
| 1907 | if (loop->_safepts != NULL__null) { |
| 1908 | loop->_safepts->yank(sfpt); |
| 1909 | } |
| 1910 | loop->_tail = iftrue; |
| 1911 | } |
| 1912 | } |
| 1913 | |
| 1914 | // Build a canonical trip test. |
| 1915 | // Clone code, as old values may be in use. |
| 1916 | incr = incr->clone(); |
| 1917 | incr->set_req(1,phi); |
| 1918 | incr->set_req(2,stride); |
| 1919 | incr = _igvn.register_new_node_with_optimizer(incr); |
| 1920 | set_early_ctrl(incr, false); |
| 1921 | _igvn.rehash_node_delayed(phi); |
| 1922 | phi->set_req_X( LoopNode::LoopBackControl, incr, &_igvn ); |
| 1923 | |
| 1924 | // If phi type is more restrictive than Int, raise to |
| 1925 | // Int to prevent (almost) infinite recursion in igvn |
| 1926 | // which can only handle integer types for constants or minint..maxint. |
| 1927 | if (!TypeInteger::bottom(iv_bt)->higher_equal(phi->bottom_type())) { |
| 1928 | Node* nphi = PhiNode::make(phi->in(0), phi->in(LoopNode::EntryControl), TypeInteger::bottom(iv_bt)); |
| 1929 | nphi->set_req(LoopNode::LoopBackControl, phi->in(LoopNode::LoopBackControl)); |
| 1930 | nphi = _igvn.register_new_node_with_optimizer(nphi); |
| 1931 | set_ctrl(nphi, get_ctrl(phi)); |
| 1932 | _igvn.replace_node(phi, nphi); |
| 1933 | phi = nphi->as_Phi(); |
| 1934 | } |
| 1935 | cmp = cmp->clone(); |
| 1936 | cmp->set_req(1,incr); |
| 1937 | cmp->set_req(2, adjusted_limit); |
| 1938 | cmp = _igvn.register_new_node_with_optimizer(cmp); |
| 1939 | set_ctrl(cmp, iff->in(0)); |
| 1940 | |
| 1941 | test = test->clone()->as_Bool(); |
| 1942 | (*(BoolTest*)&test->_test)._test = bt; |
| 1943 | test->set_req(1,cmp); |
| 1944 | _igvn.register_new_node_with_optimizer(test); |
| 1945 | set_ctrl(test, iff->in(0)); |
| 1946 | |
| 1947 | // Replace the old IfNode with a new LoopEndNode |
| 1948 | Node *lex = _igvn.register_new_node_with_optimizer(BaseCountedLoopEndNode::make(iff->in(0), test, cl_prob, iff->as_If()->_fcnt, iv_bt)); |
| 1949 | IfNode *le = lex->as_If(); |
| 1950 | uint dd = dom_depth(iff); |
| 1951 | set_idom(le, le->in(0), dd); // Update dominance for loop exit |
| 1952 | set_loop(le, loop); |
| 1953 | |
| 1954 | // Get the loop-exit control |
| 1955 | Node *iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue)); |
| 1956 | |
| 1957 | // Need to swap loop-exit and loop-back control? |
| 1958 | if (iftrue_op == Op_IfFalse) { |
| 1959 | Node *ift2=_igvn.register_new_node_with_optimizer(new IfTrueNode (le)); |
| 1960 | Node *iff2=_igvn.register_new_node_with_optimizer(new IfFalseNode(le)); |
| 1961 | |
| 1962 | loop->_tail = back_control = ift2; |
| 1963 | set_loop(ift2, loop); |
| 1964 | set_loop(iff2, get_loop(iffalse)); |
| 1965 | |
| 1966 | // Lazy update of 'get_ctrl' mechanism. |
| 1967 | lazy_replace(iffalse, iff2); |
| 1968 | lazy_replace(iftrue, ift2); |
| 1969 | |
| 1970 | // Swap names |
| 1971 | iffalse = iff2; |
| 1972 | iftrue = ift2; |
| 1973 | } else { |
| 1974 | _igvn.rehash_node_delayed(iffalse); |
| 1975 | _igvn.rehash_node_delayed(iftrue); |
| 1976 | iffalse->set_req_X( 0, le, &_igvn ); |
| 1977 | iftrue ->set_req_X( 0, le, &_igvn ); |
| 1978 | } |
| 1979 | |
| 1980 | set_idom(iftrue, le, dd+1); |
| 1981 | set_idom(iffalse, le, dd+1); |
| 1982 | assert(iff->outcnt() == 0, "should be dead now")do { if (!(iff->outcnt() == 0)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1982, "assert(" "iff->outcnt() == 0" ") failed", "should be dead now" ); ::breakpoint(); } } while (0); |
| 1983 | lazy_replace( iff, le ); // fix 'get_ctrl' |
| 1984 | |
| 1985 | Node *sfpt2 = le->in(0); |
| 1986 | |
| 1987 | Node* entry_control = init_control; |
| 1988 | bool strip_mine_loop = iv_bt == T_INT && |
| 1989 | LoopStripMiningIter > 1 && |
| 1990 | loop->_child == NULL__null && |
| 1991 | sfpt2->Opcode() == Op_SafePoint && |
| 1992 | !loop->_has_call; |
| 1993 | IdealLoopTree* outer_ilt = NULL__null; |
| 1994 | if (strip_mine_loop) { |
| 1995 | outer_ilt = create_outer_strip_mined_loop(test, cmp, init_control, loop, |
| 1996 | cl_prob, le->_fcnt, entry_control, |
| 1997 | iffalse); |
| 1998 | } |
| 1999 | |
| 2000 | // Now setup a new CountedLoopNode to replace the existing LoopNode |
| 2001 | BaseCountedLoopNode *l = BaseCountedLoopNode::make(entry_control, back_control, iv_bt); |
| 2002 | l->set_unswitch_count(x->as_Loop()->unswitch_count()); // Preserve |
| 2003 | // The following assert is approximately true, and defines the intention |
| 2004 | // of can_be_counted_loop. It fails, however, because phase->type |
| 2005 | // is not yet initialized for this loop and its parts. |
| 2006 | //assert(l->can_be_counted_loop(this), "sanity"); |
| 2007 | _igvn.register_new_node_with_optimizer(l); |
| 2008 | set_loop(l, loop); |
| 2009 | loop->_head = l; |
| 2010 | // Fix all data nodes placed at the old loop head. |
| 2011 | // Uses the lazy-update mechanism of 'get_ctrl'. |
| 2012 | lazy_replace( x, l ); |
| 2013 | set_idom(l, entry_control, dom_depth(entry_control) + 1); |
| 2014 | |
| 2015 | if (iv_bt == T_INT && (LoopStripMiningIter == 0 || strip_mine_loop)) { |
| 2016 | // Check for immediately preceding SafePoint and remove |
| 2017 | if (sfpt2->Opcode() == Op_SafePoint && (LoopStripMiningIter != 0 || is_deleteable_safept(sfpt2))) { |
| 2018 | if (strip_mine_loop) { |
| 2019 | Node* outer_le = outer_ilt->_tail->in(0); |
| 2020 | Node* sfpt = sfpt2->clone(); |
| 2021 | sfpt->set_req(0, iffalse); |
| 2022 | outer_le->set_req(0, sfpt); |
| 2023 | |
| 2024 | Node* polladdr = sfpt->in(TypeFunc::Parms); |
| 2025 | if (polladdr != nullptr && polladdr->is_Load()) { |
| 2026 | // Polling load should be pinned outside inner loop. |
| 2027 | Node* new_polladdr = polladdr->clone(); |
| 2028 | new_polladdr->set_req(0, iffalse); |
| 2029 | _igvn.register_new_node_with_optimizer(new_polladdr, polladdr); |
| 2030 | set_ctrl(new_polladdr, iffalse); |
| 2031 | sfpt->set_req(TypeFunc::Parms, new_polladdr); |
| 2032 | } |
| 2033 | // When this code runs, loop bodies have not yet been populated. |
| 2034 | const bool body_populated = false; |
| 2035 | register_control(sfpt, outer_ilt, iffalse, body_populated); |
| 2036 | set_idom(outer_le, sfpt, dom_depth(sfpt)); |
| 2037 | } |
| 2038 | lazy_replace( sfpt2, sfpt2->in(TypeFunc::Control)); |
| 2039 | if (loop->_safepts != NULL__null) { |
| 2040 | loop->_safepts->yank(sfpt2); |
| 2041 | } |
| 2042 | } |
| 2043 | } |
| 2044 | |
| 2045 | #ifdef ASSERT1 |
| 2046 | assert(l->is_valid_counted_loop(iv_bt), "counted loop shape is messed up")do { if (!(l->is_valid_counted_loop(iv_bt))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2046, "assert(" "l->is_valid_counted_loop(iv_bt)" ") failed" , "counted loop shape is messed up"); ::breakpoint(); } } while (0); |
| 2047 | assert(l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex, "" )do { if (!(l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2047, "assert(" "l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex" ") failed", ""); ::breakpoint(); } } while (0); |
| 2048 | #endif |
| 2049 | #ifndef PRODUCT |
| 2050 | if (TraceLoopOpts) { |
| 2051 | tty->print("Counted "); |
| 2052 | loop->dump_head(); |
| 2053 | } |
| 2054 | #endif |
| 2055 | |
| 2056 | C->print_method(PHASE_AFTER_CLOOPS, 3); |
| 2057 | |
| 2058 | // Capture bounds of the loop in the induction variable Phi before |
| 2059 | // subsequent transformation (iteration splitting) obscures the |
| 2060 | // bounds |
| 2061 | l->phi()->as_Phi()->set_type(l->phi()->Value(&_igvn)); |
| 2062 | |
| 2063 | if (strip_mine_loop) { |
| 2064 | l->mark_strip_mined(); |
| 2065 | l->verify_strip_mined(1); |
| 2066 | outer_ilt->_head->as_Loop()->verify_strip_mined(1); |
| 2067 | loop = outer_ilt; |
| 2068 | } |
| 2069 | |
| 2070 | #ifndef PRODUCT |
| 2071 | if (x->as_Loop()->is_loop_nest_inner_loop() && iv_bt == T_LONG) { |
| 2072 | Atomic::inc(&_long_loop_counted_loops); |
| 2073 | } |
| 2074 | #endif |
| 2075 | if (iv_bt == T_LONG && x->as_Loop()->is_loop_nest_outer_loop()) { |
| 2076 | l->mark_loop_nest_outer_loop(); |
| 2077 | } |
| 2078 | |
| 2079 | return true; |
| 2080 | } |
| 2081 | |
| 2082 | //----------------------exact_limit------------------------------------------- |
| 2083 | Node* PhaseIdealLoop::exact_limit( IdealLoopTree *loop ) { |
| 2084 | assert(loop->_head->is_CountedLoop(), "")do { if (!(loop->_head->is_CountedLoop())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2084, "assert(" "loop->_head->is_CountedLoop()" ") failed" , ""); ::breakpoint(); } } while (0); |
| 2085 | CountedLoopNode *cl = loop->_head->as_CountedLoop(); |
| 2086 | assert(cl->is_valid_counted_loop(T_INT), "")do { if (!(cl->is_valid_counted_loop(T_INT))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2086, "assert(" "cl->is_valid_counted_loop(T_INT)" ") failed" , ""); ::breakpoint(); } } while (0); |
| 2087 | |
| 2088 | if (ABS(cl->stride_con()) == 1 || |
| 2089 | cl->limit()->Opcode() == Op_LoopLimit) { |
| 2090 | // Old code has exact limit (it could be incorrect in case of int overflow). |
| 2091 | // Loop limit is exact with stride == 1. And loop may already have exact limit. |
| 2092 | return cl->limit(); |
| 2093 | } |
| 2094 | Node *limit = NULL__null; |
| 2095 | #ifdef ASSERT1 |
| 2096 | BoolTest::mask bt = cl->loopexit()->test_trip(); |
| 2097 | assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected")do { if (!(bt == BoolTest::lt || bt == BoolTest::gt)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2097, "assert(" "bt == BoolTest::lt || bt == BoolTest::gt" ") failed" , "canonical test is expected"); ::breakpoint(); } } while (0 ); |
| 2098 | #endif |
| 2099 | if (cl->has_exact_trip_count()) { |
| 2100 | // Simple case: loop has constant boundaries. |
| 2101 | // Use jlongs to avoid integer overflow. |
| 2102 | int stride_con = cl->stride_con(); |
| 2103 | jlong init_con = cl->init_trip()->get_int(); |
| 2104 | jlong limit_con = cl->limit()->get_int(); |
| 2105 | julong trip_cnt = cl->trip_count(); |
| 2106 | jlong final_con = init_con + trip_cnt*stride_con; |
| 2107 | int final_int = (int)final_con; |
| 2108 | // The final value should be in integer range since the loop |
| 2109 | // is counted and the limit was checked for overflow. |
| 2110 | assert(final_con == (jlong)final_int, "final value should be integer")do { if (!(final_con == (jlong)final_int)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2110, "assert(" "final_con == (jlong)final_int" ") failed", "final value should be integer"); ::breakpoint(); } } while ( 0); |
| 2111 | limit = _igvn.intcon(final_int); |
| 2112 | } else { |
| 2113 | // Create new LoopLimit node to get exact limit (final iv value). |
| 2114 | limit = new LoopLimitNode(C, cl->init_trip(), cl->limit(), cl->stride()); |
| 2115 | register_new_node(limit, cl->in(LoopNode::EntryControl)); |
| 2116 | } |
| 2117 | assert(limit != NULL, "sanity")do { if (!(limit != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2117, "assert(" "limit != __null" ") failed", "sanity"); :: breakpoint(); } } while (0); |
| 2118 | return limit; |
| 2119 | } |
| 2120 | |
| 2121 | //------------------------------Ideal------------------------------------------ |
| 2122 | // Return a node which is more "ideal" than the current node. |
| 2123 | // Attempt to convert into a counted-loop. |
| 2124 | Node *LoopNode::Ideal(PhaseGVN *phase, bool can_reshape) { |
| 2125 | if (!can_be_counted_loop(phase) && !is_OuterStripMinedLoop()) { |
| 2126 | phase->C->set_major_progress(); |
| 2127 | } |
| 2128 | return RegionNode::Ideal(phase, can_reshape); |
| 2129 | } |
| 2130 | |
| 2131 | #ifdef ASSERT1 |
| 2132 | void LoopNode::verify_strip_mined(int expect_skeleton) const { |
| 2133 | const OuterStripMinedLoopNode* outer = NULL__null; |
| 2134 | const CountedLoopNode* inner = NULL__null; |
| 2135 | if (is_strip_mined()) { |
| 2136 | if (!is_valid_counted_loop(T_INT)) { |
| 2137 | return; // Skip malformed counted loop |
| 2138 | } |
| 2139 | assert(is_CountedLoop(), "no Loop should be marked strip mined")do { if (!(is_CountedLoop())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2139, "assert(" "is_CountedLoop()" ") failed", "no Loop should be marked strip mined" ); ::breakpoint(); } } while (0); |
| 2140 | inner = as_CountedLoop(); |
| 2141 | outer = inner->in(LoopNode::EntryControl)->as_OuterStripMinedLoop(); |
| 2142 | } else if (is_OuterStripMinedLoop()) { |
| 2143 | outer = this->as_OuterStripMinedLoop(); |
| 2144 | inner = outer->unique_ctrl_out()->as_CountedLoop(); |
| 2145 | assert(inner->is_valid_counted_loop(T_INT) && inner->is_strip_mined(), "OuterStripMinedLoop should have been removed")do { if (!(inner->is_valid_counted_loop(T_INT) && inner ->is_strip_mined())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2145, "assert(" "inner->is_valid_counted_loop(T_INT) && inner->is_strip_mined()" ") failed", "OuterStripMinedLoop should have been removed"); ::breakpoint(); } } while (0); |
| 2146 | assert(!is_strip_mined(), "outer loop shouldn't be marked strip mined")do { if (!(!is_strip_mined())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2146, "assert(" "!is_strip_mined()" ") failed", "outer loop shouldn't be marked strip mined" ); ::breakpoint(); } } while (0); |
| 2147 | } |
| 2148 | if (inner != NULL__null || outer != NULL__null) { |
| 2149 | assert(inner != NULL && outer != NULL, "missing loop in strip mined nest")do { if (!(inner != __null && outer != __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2149, "assert(" "inner != __null && outer != __null" ") failed", "missing loop in strip mined nest"); ::breakpoint (); } } while (0); |
| 2150 | Node* outer_tail = outer->in(LoopNode::LoopBackControl); |
| 2151 | Node* outer_le = outer_tail->in(0); |
| 2152 | assert(outer_le->Opcode() == Op_OuterStripMinedLoopEnd, "tail of outer loop should be an If")do { if (!(outer_le->Opcode() == Op_OuterStripMinedLoopEnd )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2152, "assert(" "outer_le->Opcode() == Op_OuterStripMinedLoopEnd" ") failed", "tail of outer loop should be an If"); ::breakpoint (); } } while (0); |
| 2153 | Node* sfpt = outer_le->in(0); |
| 2154 | assert(sfpt->Opcode() == Op_SafePoint, "where's the safepoint?")do { if (!(sfpt->Opcode() == Op_SafePoint)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2154, "assert(" "sfpt->Opcode() == Op_SafePoint" ") failed" , "where's the safepoint?"); ::breakpoint(); } } while (0); |
| 2155 | Node* inner_out = sfpt->in(0); |
| 2156 | CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd(); |
| 2157 | assert(cle == inner->loopexit_or_null(), "mismatch")do { if (!(cle == inner->loopexit_or_null())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2157, "assert(" "cle == inner->loopexit_or_null()" ") failed" , "mismatch"); ::breakpoint(); } } while (0); |
| 2158 | bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0; |
| 2159 | if (has_skeleton) { |
| 2160 | assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node")do { if (!(expect_skeleton == 1 || expect_skeleton == -1)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2160, "assert(" "expect_skeleton == 1 || expect_skeleton == -1" ") failed", "unexpected skeleton node"); ::breakpoint(); } } while (0); |
| 2161 | assert(outer->outcnt() == 2, "only control nodes")do { if (!(outer->outcnt() == 2)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2161, "assert(" "outer->outcnt() == 2" ") failed", "only control nodes" ); ::breakpoint(); } } while (0); |
| 2162 | } else { |
| 2163 | assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?")do { if (!(expect_skeleton == 0 || expect_skeleton == -1)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2163, "assert(" "expect_skeleton == 0 || expect_skeleton == -1" ") failed", "no skeleton node?"); ::breakpoint(); } } while ( 0); |
| 2164 | uint phis = 0; |
| 2165 | uint be_loads = 0; |
| 2166 | Node* be = inner->in(LoopNode::LoopBackControl); |
| 2167 | for (DUIterator_Fast imax, i = inner->fast_outs(imax); i < imax; i++) { |
| 2168 | Node* u = inner->fast_out(i); |
| 2169 | if (u->is_Phi()) { |
| 2170 | phis++; |
| 2171 | for (DUIterator_Fast jmax, j = be->fast_outs(jmax); j < jmax; j++) { |
| 2172 | Node* n = be->fast_out(j); |
| 2173 | if (n->is_Load()) { |
| 2174 | assert(n->in(0) == be || n->find_prec_edge(be) > 0, "should be on the backedge")do { if (!(n->in(0) == be || n->find_prec_edge(be) > 0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2174, "assert(" "n->in(0) == be || n->find_prec_edge(be) > 0" ") failed", "should be on the backedge"); ::breakpoint(); } } while (0); |
| 2175 | do { |
| 2176 | n = n->raw_out(0); |
| 2177 | } while (!n->is_Phi()); |
| 2178 | if (n == u) { |
| 2179 | be_loads++; |
| 2180 | break; |
| 2181 | } |
| 2182 | } |
| 2183 | } |
| 2184 | } |
| 2185 | } |
| 2186 | assert(be_loads <= phis, "wrong number phis that depends on a pinned load")do { if (!(be_loads <= phis)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2186, "assert(" "be_loads <= phis" ") failed", "wrong number phis that depends on a pinned load" ); ::breakpoint(); } } while (0); |
| 2187 | for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) { |
| 2188 | Node* u = outer->fast_out(i); |
| 2189 | assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop")do { if (!(u == outer || u == inner || u->is_Phi())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2189, "assert(" "u == outer || u == inner || u->is_Phi()" ") failed", "nothing between inner and outer loop"); ::breakpoint (); } } while (0); |
| 2190 | } |
| 2191 | uint stores = 0; |
| 2192 | for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) { |
| 2193 | Node* u = inner_out->fast_out(i); |
| 2194 | if (u->is_Store()) { |
| 2195 | stores++; |
| 2196 | } |
| 2197 | } |
| 2198 | // Late optimization of loads on backedge can cause Phi of outer loop to be eliminated but Phi of inner loop is |
| 2199 | // not guaranteed to be optimized out. |
| 2200 | assert(outer->outcnt() >= phis + 2 - be_loads && outer->outcnt() <= phis + 2 + stores + 1, "only phis")do { if (!(outer->outcnt() >= phis + 2 - be_loads && outer->outcnt() <= phis + 2 + stores + 1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2200, "assert(" "outer->outcnt() >= phis + 2 - be_loads && outer->outcnt() <= phis + 2 + stores + 1" ") failed", "only phis"); ::breakpoint(); } } while (0); |
| 2201 | } |
| 2202 | assert(sfpt->outcnt() == 1, "no data node")do { if (!(sfpt->outcnt() == 1)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2202, "assert(" "sfpt->outcnt() == 1" ") failed", "no data node" ); ::breakpoint(); } } while (0); |
| 2203 | assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node")do { if (!(outer_tail->outcnt() == 1 || !has_skeleton)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2203, "assert(" "outer_tail->outcnt() == 1 || !has_skeleton" ") failed", "no data node"); ::breakpoint(); } } while (0); |
| 2204 | } |
| 2205 | } |
| 2206 | #endif |
| 2207 | |
| 2208 | //============================================================================= |
| 2209 | //------------------------------Ideal------------------------------------------ |
| 2210 | // Return a node which is more "ideal" than the current node. |
| 2211 | // Attempt to convert into a counted-loop. |
| 2212 | Node *CountedLoopNode::Ideal(PhaseGVN *phase, bool can_reshape) { |
| 2213 | return RegionNode::Ideal(phase, can_reshape); |
| 2214 | } |
| 2215 | |
| 2216 | //------------------------------dump_spec-------------------------------------- |
| 2217 | // Dump special per-node info |
| 2218 | #ifndef PRODUCT |
| 2219 | void CountedLoopNode::dump_spec(outputStream *st) const { |
| 2220 | LoopNode::dump_spec(st); |
| 2221 | if (stride_is_con()) { |
| 2222 | st->print("stride: %d ",stride_con()); |
| 2223 | } |
| 2224 | if (is_pre_loop ()) st->print("pre of N%d" , _main_idx); |
| 2225 | if (is_main_loop()) st->print("main of N%d", _idx); |
| 2226 | if (is_post_loop()) st->print("post of N%d", _main_idx); |
| 2227 | if (is_strip_mined()) st->print(" strip mined"); |
| 2228 | } |
| 2229 | #endif |
| 2230 | |
| 2231 | //============================================================================= |
| 2232 | jlong BaseCountedLoopEndNode::stride_con() const { |
| 2233 | return stride()->bottom_type()->is_integer(bt())->get_con_as_long(bt()); |
| 2234 | } |
| 2235 | |
| 2236 | |
| 2237 | BaseCountedLoopEndNode* BaseCountedLoopEndNode::make(Node* control, Node* test, float prob, float cnt, BasicType bt) { |
| 2238 | if (bt == T_INT) { |
| 2239 | return new CountedLoopEndNode(control, test, prob, cnt); |
| 2240 | } |
| 2241 | assert(bt == T_LONG, "unsupported")do { if (!(bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2241, "assert(" "bt == T_LONG" ") failed", "unsupported"); :: breakpoint(); } } while (0); |
| 2242 | return new LongCountedLoopEndNode(control, test, prob, cnt); |
| 2243 | } |
| 2244 | |
| 2245 | //============================================================================= |
| 2246 | //------------------------------Value----------------------------------------- |
| 2247 | const Type* LoopLimitNode::Value(PhaseGVN* phase) const { |
| 2248 | const Type* init_t = phase->type(in(Init)); |
| 2249 | const Type* limit_t = phase->type(in(Limit)); |
| 2250 | const Type* stride_t = phase->type(in(Stride)); |
| 2251 | // Either input is TOP ==> the result is TOP |
| 2252 | if (init_t == Type::TOP) return Type::TOP; |
| 2253 | if (limit_t == Type::TOP) return Type::TOP; |
| 2254 | if (stride_t == Type::TOP) return Type::TOP; |
| 2255 | |
| 2256 | int stride_con = stride_t->is_int()->get_con(); |
| 2257 | if (stride_con == 1) |
| 2258 | return bottom_type(); // Identity |
| 2259 | |
| 2260 | if (init_t->is_int()->is_con() && limit_t->is_int()->is_con()) { |
| 2261 | // Use jlongs to avoid integer overflow. |
| 2262 | jlong init_con = init_t->is_int()->get_con(); |
| 2263 | jlong limit_con = limit_t->is_int()->get_con(); |
| 2264 | int stride_m = stride_con - (stride_con > 0 ? 1 : -1); |
| 2265 | jlong trip_count = (limit_con - init_con + stride_m)/stride_con; |
| 2266 | jlong final_con = init_con + stride_con*trip_count; |
| 2267 | int final_int = (int)final_con; |
| 2268 | // The final value should be in integer range since the loop |
| 2269 | // is counted and the limit was checked for overflow. |
| 2270 | assert(final_con == (jlong)final_int, "final value should be integer")do { if (!(final_con == (jlong)final_int)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2270, "assert(" "final_con == (jlong)final_int" ") failed", "final value should be integer"); ::breakpoint(); } } while ( 0); |
| 2271 | return TypeInt::make(final_int); |
| 2272 | } |
| 2273 | |
| 2274 | return bottom_type(); // TypeInt::INT |
| 2275 | } |
| 2276 | |
| 2277 | //------------------------------Ideal------------------------------------------ |
| 2278 | // Return a node which is more "ideal" than the current node. |
| 2279 | Node *LoopLimitNode::Ideal(PhaseGVN *phase, bool can_reshape) { |
| 2280 | if (phase->type(in(Init)) == Type::TOP || |
| 2281 | phase->type(in(Limit)) == Type::TOP || |
| 2282 | phase->type(in(Stride)) == Type::TOP) |
| 2283 | return NULL__null; // Dead |
| 2284 | |
| 2285 | int stride_con = phase->type(in(Stride))->is_int()->get_con(); |
| 2286 | if (stride_con == 1) |
| 2287 | return NULL__null; // Identity |
| 2288 | |
| 2289 | if (in(Init)->is_Con() && in(Limit)->is_Con()) |
| 2290 | return NULL__null; // Value |
| 2291 | |
| 2292 | // Delay following optimizations until all loop optimizations |
| 2293 | // done to keep Ideal graph simple. |
| 2294 | if (!can_reshape || !phase->C->post_loop_opts_phase()) { |
| 2295 | return NULL__null; |
| 2296 | } |
| 2297 | |
| 2298 | const TypeInt* init_t = phase->type(in(Init) )->is_int(); |
| 2299 | const TypeInt* limit_t = phase->type(in(Limit))->is_int(); |
| 2300 | int stride_p; |
| 2301 | jlong lim, ini; |
| 2302 | julong max; |
| 2303 | if (stride_con > 0) { |
| 2304 | stride_p = stride_con; |
| 2305 | lim = limit_t->_hi; |
| 2306 | ini = init_t->_lo; |
| 2307 | max = (julong)max_jint; |
| 2308 | } else { |
| 2309 | stride_p = -stride_con; |
| 2310 | lim = init_t->_hi; |
| 2311 | ini = limit_t->_lo; |
| 2312 | max = (julong)min_jint; |
| 2313 | } |
| 2314 | julong range = lim - ini + stride_p; |
| 2315 | if (range <= max) { |
| 2316 | // Convert to integer expression if it is not overflow. |
| 2317 | Node* stride_m = phase->intcon(stride_con - (stride_con > 0 ? 1 : -1)); |
| 2318 | Node *range = phase->transform(new SubINode(in(Limit), in(Init))); |
| 2319 | Node *bias = phase->transform(new AddINode(range, stride_m)); |
| 2320 | Node *trip = phase->transform(new DivINode(0, bias, in(Stride))); |
| 2321 | Node *span = phase->transform(new MulINode(trip, in(Stride))); |
| 2322 | return new AddINode(span, in(Init)); // exact limit |
| 2323 | } |
| 2324 | |
| 2325 | if (is_power_of_2(stride_p) || // divisor is 2^n |
| 2326 | !Matcher::has_match_rule(Op_LoopLimit)) { // or no specialized Mach node? |
| 2327 | // Convert to long expression to avoid integer overflow |
| 2328 | // and let igvn optimizer convert this division. |
| 2329 | // |
| 2330 | Node* init = phase->transform( new ConvI2LNode(in(Init))); |
| 2331 | Node* limit = phase->transform( new ConvI2LNode(in(Limit))); |
| 2332 | Node* stride = phase->longcon(stride_con); |
| 2333 | Node* stride_m = phase->longcon(stride_con - (stride_con > 0 ? 1 : -1)); |
| 2334 | |
| 2335 | Node *range = phase->transform(new SubLNode(limit, init)); |
| 2336 | Node *bias = phase->transform(new AddLNode(range, stride_m)); |
| 2337 | Node *span; |
| 2338 | if (stride_con > 0 && is_power_of_2(stride_p)) { |
| 2339 | // bias >= 0 if stride >0, so if stride is 2^n we can use &(-stride) |
| 2340 | // and avoid generating rounding for division. Zero trip guard should |
| 2341 | // guarantee that init < limit but sometimes the guard is missing and |
| 2342 | // we can get situation when init > limit. Note, for the empty loop |
| 2343 | // optimization zero trip guard is generated explicitly which leaves |
| 2344 | // only RCE predicate where exact limit is used and the predicate |
| 2345 | // will simply fail forcing recompilation. |
| 2346 | Node* neg_stride = phase->longcon(-stride_con); |
| 2347 | span = phase->transform(new AndLNode(bias, neg_stride)); |
| 2348 | } else { |
| 2349 | Node *trip = phase->transform(new DivLNode(0, bias, stride)); |
| 2350 | span = phase->transform(new MulLNode(trip, stride)); |
| 2351 | } |
| 2352 | // Convert back to int |
| 2353 | Node *span_int = phase->transform(new ConvL2INode(span)); |
| 2354 | return new AddINode(span_int, in(Init)); // exact limit |
| 2355 | } |
| 2356 | |
| 2357 | return NULL__null; // No progress |
| 2358 | } |
| 2359 | |
| 2360 | //------------------------------Identity--------------------------------------- |
| 2361 | // If stride == 1 return limit node. |
| 2362 | Node* LoopLimitNode::Identity(PhaseGVN* phase) { |
| 2363 | int stride_con = phase->type(in(Stride))->is_int()->get_con(); |
| 2364 | if (stride_con == 1 || stride_con == -1) |
| 2365 | return in(Limit); |
| 2366 | return this; |
| 2367 | } |
| 2368 | |
| 2369 | //============================================================================= |
| 2370 | //----------------------match_incr_with_optional_truncation-------------------- |
| 2371 | // Match increment with optional truncation: |
| 2372 | // CHAR: (i+1)&0x7fff, BYTE: ((i+1)<<8)>>8, or SHORT: ((i+1)<<16)>>16 |
| 2373 | // Return NULL for failure. Success returns the increment node. |
| 2374 | Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, |
| 2375 | const TypeInteger** trunc_type, |
| 2376 | BasicType bt) { |
| 2377 | // Quick cutouts: |
| 2378 | if (expr == NULL__null || expr->req() != 3) return NULL__null; |
| 2379 | |
| 2380 | Node *t1 = NULL__null; |
| 2381 | Node *t2 = NULL__null; |
| 2382 | Node* n1 = expr; |
| 2383 | int n1op = n1->Opcode(); |
| 2384 | const TypeInteger* trunc_t = TypeInteger::bottom(bt); |
| 2385 | |
| 2386 | if (bt == T_INT) { |
| 2387 | // Try to strip (n1 & M) or (n1 << N >> N) from n1. |
| 2388 | if (n1op == Op_AndI && |
| 2389 | n1->in(2)->is_Con() && |
| 2390 | n1->in(2)->bottom_type()->is_int()->get_con() == 0x7fff) { |
| 2391 | // %%% This check should match any mask of 2**K-1. |
| 2392 | t1 = n1; |
| 2393 | n1 = t1->in(1); |
| 2394 | n1op = n1->Opcode(); |
| 2395 | trunc_t = TypeInt::CHAR; |
| 2396 | } else if (n1op == Op_RShiftI && |
| 2397 | n1->in(1) != NULL__null && |
| 2398 | n1->in(1)->Opcode() == Op_LShiftI && |
| 2399 | n1->in(2) == n1->in(1)->in(2) && |
| 2400 | n1->in(2)->is_Con()) { |
| 2401 | jint shift = n1->in(2)->bottom_type()->is_int()->get_con(); |
| 2402 | // %%% This check should match any shift in [1..31]. |
| 2403 | if (shift == 16 || shift == 8) { |
| 2404 | t1 = n1; |
| 2405 | t2 = t1->in(1); |
| 2406 | n1 = t2->in(1); |
| 2407 | n1op = n1->Opcode(); |
| 2408 | if (shift == 16) { |
| 2409 | trunc_t = TypeInt::SHORT; |
| 2410 | } else if (shift == 8) { |
| 2411 | trunc_t = TypeInt::BYTE; |
| 2412 | } |
| 2413 | } |
| 2414 | } |
| 2415 | } |
| 2416 | |
| 2417 | // If (maybe after stripping) it is an AddI, we won: |
| 2418 | if (n1op == Op_Add(bt)) { |
| 2419 | *trunc1 = t1; |
| 2420 | *trunc2 = t2; |
| 2421 | *trunc_type = trunc_t; |
| 2422 | return n1; |
| 2423 | } |
| 2424 | |
| 2425 | // failed |
| 2426 | return NULL__null; |
| 2427 | } |
| 2428 | |
| 2429 | LoopNode* CountedLoopNode::skip_strip_mined(int expect_skeleton) { |
| 2430 | if (is_strip_mined() && in(EntryControl) != NULL__null && in(EntryControl)->is_OuterStripMinedLoop()) { |
| 2431 | verify_strip_mined(expect_skeleton); |
| 2432 | return in(EntryControl)->as_Loop(); |
| 2433 | } |
| 2434 | return this; |
| 2435 | } |
| 2436 | |
| 2437 | OuterStripMinedLoopNode* CountedLoopNode::outer_loop() const { |
| 2438 | assert(is_strip_mined(), "not a strip mined loop")do { if (!(is_strip_mined())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2438, "assert(" "is_strip_mined()" ") failed", "not a strip mined loop" ); ::breakpoint(); } } while (0); |
| 2439 | Node* c = in(EntryControl); |
| 2440 | if (c == NULL__null || c->is_top() || !c->is_OuterStripMinedLoop()) { |
| 2441 | return NULL__null; |
| 2442 | } |
| 2443 | return c->as_OuterStripMinedLoop(); |
| 2444 | } |
| 2445 | |
| 2446 | IfTrueNode* OuterStripMinedLoopNode::outer_loop_tail() const { |
| 2447 | Node* c = in(LoopBackControl); |
| 2448 | if (c == NULL__null || c->is_top()) { |
| 2449 | return NULL__null; |
| 2450 | } |
| 2451 | return c->as_IfTrue(); |
| 2452 | } |
| 2453 | |
| 2454 | IfTrueNode* CountedLoopNode::outer_loop_tail() const { |
| 2455 | LoopNode* l = outer_loop(); |
| 2456 | if (l == NULL__null) { |
| 2457 | return NULL__null; |
| 2458 | } |
| 2459 | return l->outer_loop_tail(); |
| 2460 | } |
| 2461 | |
| 2462 | OuterStripMinedLoopEndNode* OuterStripMinedLoopNode::outer_loop_end() const { |
| 2463 | IfTrueNode* proj = outer_loop_tail(); |
| 2464 | if (proj == NULL__null) { |
| 2465 | return NULL__null; |
| 2466 | } |
| 2467 | Node* c = proj->in(0); |
| 2468 | if (c == NULL__null || c->is_top() || c->outcnt() != 2) { |
| 2469 | return NULL__null; |
| 2470 | } |
| 2471 | return c->as_OuterStripMinedLoopEnd(); |
| 2472 | } |
| 2473 | |
| 2474 | OuterStripMinedLoopEndNode* CountedLoopNode::outer_loop_end() const { |
| 2475 | LoopNode* l = outer_loop(); |
| 2476 | if (l == NULL__null) { |
| 2477 | return NULL__null; |
| 2478 | } |
| 2479 | return l->outer_loop_end(); |
| 2480 | } |
| 2481 | |
| 2482 | IfFalseNode* OuterStripMinedLoopNode::outer_loop_exit() const { |
| 2483 | IfNode* le = outer_loop_end(); |
| 2484 | if (le == NULL__null) { |
| 2485 | return NULL__null; |
| 2486 | } |
| 2487 | Node* c = le->proj_out_or_null(false); |
| 2488 | if (c == NULL__null) { |
| 2489 | return NULL__null; |
| 2490 | } |
| 2491 | return c->as_IfFalse(); |
| 2492 | } |
| 2493 | |
| 2494 | IfFalseNode* CountedLoopNode::outer_loop_exit() const { |
| 2495 | LoopNode* l = outer_loop(); |
| 2496 | if (l == NULL__null) { |
| 2497 | return NULL__null; |
| 2498 | } |
| 2499 | return l->outer_loop_exit(); |
| 2500 | } |
| 2501 | |
| 2502 | SafePointNode* OuterStripMinedLoopNode::outer_safepoint() const { |
| 2503 | IfNode* le = outer_loop_end(); |
| 2504 | if (le == NULL__null) { |
| 2505 | return NULL__null; |
| 2506 | } |
| 2507 | Node* c = le->in(0); |
| 2508 | if (c == NULL__null || c->is_top()) { |
| 2509 | return NULL__null; |
| 2510 | } |
| 2511 | assert(c->Opcode() == Op_SafePoint, "broken outer loop")do { if (!(c->Opcode() == Op_SafePoint)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2511, "assert(" "c->Opcode() == Op_SafePoint" ") failed" , "broken outer loop"); ::breakpoint(); } } while (0); |
| 2512 | return c->as_SafePoint(); |
| 2513 | } |
| 2514 | |
| 2515 | SafePointNode* CountedLoopNode::outer_safepoint() const { |
| 2516 | LoopNode* l = outer_loop(); |
| 2517 | if (l == NULL__null) { |
| 2518 | return NULL__null; |
| 2519 | } |
| 2520 | return l->outer_safepoint(); |
| 2521 | } |
| 2522 | |
| 2523 | Node* CountedLoopNode::skip_predicates_from_entry(Node* ctrl) { |
| 2524 | while (ctrl != NULL__null && ctrl->is_Proj() && ctrl->in(0) != NULL__null && ctrl->in(0)->is_If() && |
| 2525 | (ctrl->in(0)->as_If()->proj_out_or_null(1-ctrl->as_Proj()->_con) == NULL__null || |
| 2526 | (ctrl->in(0)->as_If()->proj_out(1-ctrl->as_Proj()->_con)->outcnt() == 1 && |
| 2527 | ctrl->in(0)->as_If()->proj_out(1-ctrl->as_Proj()->_con)->unique_out()->Opcode() == Op_Halt))) { |
| 2528 | ctrl = ctrl->in(0)->in(0); |
| 2529 | } |
| 2530 | |
| 2531 | return ctrl; |
| 2532 | } |
| 2533 | |
| 2534 | Node* CountedLoopNode::skip_predicates() { |
| 2535 | Node* ctrl = in(LoopNode::EntryControl); |
| 2536 | if (is_main_loop()) { |
| 2537 | ctrl = skip_strip_mined()->in(LoopNode::EntryControl); |
| 2538 | } |
| 2539 | if (is_main_loop() || is_post_loop()) { |
| 2540 | return skip_predicates_from_entry(ctrl); |
| 2541 | } |
| 2542 | return ctrl; |
| 2543 | } |
| 2544 | |
| 2545 | |
| 2546 | int CountedLoopNode::stride_con() const { |
| 2547 | CountedLoopEndNode* cle = loopexit_or_null(); |
| 2548 | return cle != NULL__null ? cle->stride_con() : 0; |
| 2549 | } |
| 2550 | |
| 2551 | BaseCountedLoopNode* BaseCountedLoopNode::make(Node* entry, Node* backedge, BasicType bt) { |
| 2552 | if (bt == T_INT) { |
| 2553 | return new CountedLoopNode(entry, backedge); |
| 2554 | } |
| 2555 | assert(bt == T_LONG, "unsupported")do { if (!(bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2555, "assert(" "bt == T_LONG" ") failed", "unsupported"); :: breakpoint(); } } while (0); |
| 2556 | return new LongCountedLoopNode(entry, backedge); |
| 2557 | } |
| 2558 | |
| 2559 | |
| 2560 | void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { |
| 2561 | // Look for the outer & inner strip mined loop, reduce number of |
| 2562 | // iterations of the inner loop, set exit condition of outer loop, |
| 2563 | // construct required phi nodes for outer loop. |
| 2564 | CountedLoopNode* inner_cl = unique_ctrl_out()->as_CountedLoop(); |
| 2565 | assert(inner_cl->is_strip_mined(), "inner loop should be strip mined")do { if (!(inner_cl->is_strip_mined())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2565, "assert(" "inner_cl->is_strip_mined()" ") failed", "inner loop should be strip mined"); ::breakpoint(); } } while (0); |
| 2566 | Node* inner_iv_phi = inner_cl->phi(); |
| 2567 | if (inner_iv_phi == NULL__null) { |
| 2568 | IfNode* outer_le = outer_loop_end(); |
| 2569 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); |
| 2570 | igvn->replace_node(outer_le, iff); |
| 2571 | inner_cl->clear_strip_mined(); |
| 2572 | return; |
| 2573 | } |
| 2574 | CountedLoopEndNode* inner_cle = inner_cl->loopexit(); |
| 2575 | |
| 2576 | int stride = inner_cl->stride_con(); |
| 2577 | jlong scaled_iters_long = ((jlong)LoopStripMiningIter) * ABS(stride); |
| 2578 | int scaled_iters = (int)scaled_iters_long; |
| 2579 | int short_scaled_iters = LoopStripMiningIterShortLoop* ABS(stride); |
| 2580 | const TypeInt* inner_iv_t = igvn->type(inner_iv_phi)->is_int(); |
| 2581 | jlong iter_estimate = (jlong)inner_iv_t->_hi - (jlong)inner_iv_t->_lo; |
| 2582 | assert(iter_estimate > 0, "broken")do { if (!(iter_estimate > 0)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2582, "assert(" "iter_estimate > 0" ") failed", "broken" ); ::breakpoint(); } } while (0); |
| 2583 | if ((jlong)scaled_iters != scaled_iters_long || iter_estimate <= short_scaled_iters) { |
| 2584 | // Remove outer loop and safepoint (too few iterations) |
| 2585 | Node* outer_sfpt = outer_safepoint(); |
| 2586 | Node* outer_out = outer_loop_exit(); |
| 2587 | igvn->replace_node(outer_out, outer_sfpt->in(0)); |
| 2588 | igvn->replace_input_of(outer_sfpt, 0, igvn->C->top()); |
| 2589 | inner_cl->clear_strip_mined(); |
| 2590 | return; |
| 2591 | } |
| 2592 | if (iter_estimate <= scaled_iters_long) { |
| 2593 | // We would only go through one iteration of |
| 2594 | // the outer loop: drop the outer loop but |
| 2595 | // keep the safepoint so we don't run for |
| 2596 | // too long without a safepoint |
| 2597 | IfNode* outer_le = outer_loop_end(); |
| 2598 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); |
| 2599 | igvn->replace_node(outer_le, iff); |
| 2600 | inner_cl->clear_strip_mined(); |
| 2601 | return; |
| 2602 | } |
| 2603 | |
| 2604 | Node* cle_tail = inner_cle->proj_out(true); |
| 2605 | ResourceMark rm; |
| 2606 | Node_List old_new; |
| 2607 | if (cle_tail->outcnt() > 1) { |
| 2608 | // Look for nodes on backedge of inner loop and clone them |
| 2609 | Unique_Node_List backedge_nodes; |
| 2610 | for (DUIterator_Fast imax, i = cle_tail->fast_outs(imax); i < imax; i++) { |
| 2611 | Node* u = cle_tail->fast_out(i); |
| 2612 | if (u != inner_cl) { |
| 2613 | assert(!u->is_CFG(), "control flow on the backedge?")do { if (!(!u->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2613, "assert(" "!u->is_CFG()" ") failed", "control flow on the backedge?" ); ::breakpoint(); } } while (0); |
| 2614 | backedge_nodes.push(u); |
| 2615 | } |
| 2616 | } |
| 2617 | uint last = igvn->C->unique(); |
| 2618 | for (uint next = 0; next < backedge_nodes.size(); next++) { |
| 2619 | Node* n = backedge_nodes.at(next); |
| 2620 | old_new.map(n->_idx, n->clone()); |
| 2621 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
| 2622 | Node* u = n->fast_out(i); |
| 2623 | assert(!u->is_CFG(), "broken")do { if (!(!u->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2623, "assert(" "!u->is_CFG()" ") failed", "broken"); :: breakpoint(); } } while (0); |
| 2624 | if (u->_idx >= last) { |
| 2625 | continue; |
| 2626 | } |
| 2627 | if (!u->is_Phi()) { |
| 2628 | backedge_nodes.push(u); |
| 2629 | } else { |
| 2630 | assert(u->in(0) == inner_cl, "strange phi on the backedge")do { if (!(u->in(0) == inner_cl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2630, "assert(" "u->in(0) == inner_cl" ") failed", "strange phi on the backedge" ); ::breakpoint(); } } while (0); |
| 2631 | } |
| 2632 | } |
| 2633 | } |
| 2634 | // Put the clones on the outer loop backedge |
| 2635 | Node* le_tail = outer_loop_tail(); |
| 2636 | for (uint next = 0; next < backedge_nodes.size(); next++) { |
| 2637 | Node *n = old_new[backedge_nodes.at(next)->_idx]; |
| 2638 | for (uint i = 1; i < n->req(); i++) { |
| 2639 | if (n->in(i) != NULL__null && old_new[n->in(i)->_idx] != NULL__null) { |
| 2640 | n->set_req(i, old_new[n->in(i)->_idx]); |
| 2641 | } |
| 2642 | } |
| 2643 | if (n->in(0) != NULL__null && n->in(0) == cle_tail) { |
| 2644 | n->set_req(0, le_tail); |
| 2645 | } |
| 2646 | igvn->register_new_node_with_optimizer(n); |
| 2647 | } |
| 2648 | } |
| 2649 | |
| 2650 | Node* iv_phi = NULL__null; |
| 2651 | // Make a clone of each phi in the inner loop |
| 2652 | // for the outer loop |
| 2653 | for (uint i = 0; i < inner_cl->outcnt(); i++) { |
| 2654 | Node* u = inner_cl->raw_out(i); |
| 2655 | if (u->is_Phi()) { |
| 2656 | assert(u->in(0) == inner_cl, "inconsistent")do { if (!(u->in(0) == inner_cl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2656, "assert(" "u->in(0) == inner_cl" ") failed", "inconsistent" ); ::breakpoint(); } } while (0); |
| 2657 | Node* phi = u->clone(); |
| 2658 | phi->set_req(0, this); |
| 2659 | Node* be = old_new[phi->in(LoopNode::LoopBackControl)->_idx]; |
| 2660 | if (be != NULL__null) { |
| 2661 | phi->set_req(LoopNode::LoopBackControl, be); |
| 2662 | } |
| 2663 | phi = igvn->transform(phi); |
| 2664 | igvn->replace_input_of(u, LoopNode::EntryControl, phi); |
| 2665 | if (u == inner_iv_phi) { |
| 2666 | iv_phi = phi; |
| 2667 | } |
| 2668 | } |
| 2669 | } |
| 2670 | Node* cle_out = inner_cle->proj_out(false); |
| 2671 | if (cle_out->outcnt() > 1) { |
| 2672 | // Look for chains of stores that were sunk |
| 2673 | // out of the inner loop and are in the outer loop |
| 2674 | for (DUIterator_Fast imax, i = cle_out->fast_outs(imax); i < imax; i++) { |
| 2675 | Node* u = cle_out->fast_out(i); |
| 2676 | if (u->is_Store()) { |
| 2677 | Node* first = u; |
| 2678 | for(;;) { |
| 2679 | Node* next = first->in(MemNode::Memory); |
| 2680 | if (!next->is_Store() || next->in(0) != cle_out) { |
| 2681 | break; |
| 2682 | } |
| 2683 | first = next; |
| 2684 | } |
| 2685 | Node* last = u; |
| 2686 | for(;;) { |
| 2687 | Node* next = NULL__null; |
| 2688 | for (DUIterator_Fast jmax, j = last->fast_outs(jmax); j < jmax; j++) { |
| 2689 | Node* uu = last->fast_out(j); |
| 2690 | if (uu->is_Store() && uu->in(0) == cle_out) { |
| 2691 | assert(next == NULL, "only one in the outer loop")do { if (!(next == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2691, "assert(" "next == __null" ") failed", "only one in the outer loop" ); ::breakpoint(); } } while (0); |
| 2692 | next = uu; |
| 2693 | } |
| 2694 | } |
| 2695 | if (next == NULL__null) { |
| 2696 | break; |
| 2697 | } |
| 2698 | last = next; |
| 2699 | } |
| 2700 | Node* phi = NULL__null; |
| 2701 | for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) { |
| 2702 | Node* uu = fast_out(j); |
| 2703 | if (uu->is_Phi()) { |
| 2704 | Node* be = uu->in(LoopNode::LoopBackControl); |
| 2705 | if (be->is_Store() && old_new[be->_idx] != NULL__null) { |
| 2706 | assert(false, "store on the backedge + sunk stores: unsupported")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2706, "assert(" "false" ") failed", "store on the backedge + sunk stores: unsupported" ); ::breakpoint(); } } while (0); |
| 2707 | // drop outer loop |
| 2708 | IfNode* outer_le = outer_loop_end(); |
| 2709 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); |
| 2710 | igvn->replace_node(outer_le, iff); |
| 2711 | inner_cl->clear_strip_mined(); |
| 2712 | return; |
| 2713 | } |
| 2714 | if (be == last || be == first->in(MemNode::Memory)) { |
| 2715 | assert(phi == NULL, "only one phi")do { if (!(phi == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2715, "assert(" "phi == __null" ") failed", "only one phi") ; ::breakpoint(); } } while (0); |
| 2716 | phi = uu; |
| 2717 | } |
| 2718 | } |
| 2719 | } |
| 2720 | #ifdef ASSERT1 |
| 2721 | for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) { |
| 2722 | Node* uu = fast_out(j); |
| 2723 | if (uu->is_Phi() && uu->bottom_type() == Type::MEMORY) { |
| 2724 | if (uu->adr_type() == igvn->C->get_adr_type(igvn->C->get_alias_index(u->adr_type()))) { |
| 2725 | assert(phi == uu, "what's that phi?")do { if (!(phi == uu)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2725, "assert(" "phi == uu" ") failed", "what's that phi?") ; ::breakpoint(); } } while (0); |
| 2726 | } else if (uu->adr_type() == TypePtr::BOTTOM) { |
| 2727 | Node* n = uu->in(LoopNode::LoopBackControl); |
| 2728 | uint limit = igvn->C->live_nodes(); |
| 2729 | uint i = 0; |
| 2730 | while (n != uu) { |
| 2731 | i++; |
| 2732 | assert(i < limit, "infinite loop")do { if (!(i < limit)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2732, "assert(" "i < limit" ") failed", "infinite loop") ; ::breakpoint(); } } while (0); |
| 2733 | if (n->is_Proj()) { |
| 2734 | n = n->in(0); |
| 2735 | } else if (n->is_SafePoint() || n->is_MemBar()) { |
| 2736 | n = n->in(TypeFunc::Memory); |
| 2737 | } else if (n->is_Phi()) { |
| 2738 | n = n->in(1); |
| 2739 | } else if (n->is_MergeMem()) { |
| 2740 | n = n->as_MergeMem()->memory_at(igvn->C->get_alias_index(u->adr_type())); |
| 2741 | } else if (n->is_Store() || n->is_LoadStore() || n->is_ClearArray()) { |
| 2742 | n = n->in(MemNode::Memory); |
| 2743 | } else { |
| 2744 | n->dump(); |
| 2745 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2745); ::breakpoint(); } while (0); |
| 2746 | } |
| 2747 | } |
| 2748 | } |
| 2749 | } |
| 2750 | } |
| 2751 | #endif |
| 2752 | if (phi == NULL__null) { |
| 2753 | // If the an entire chains was sunk, the |
| 2754 | // inner loop has no phi for that memory |
| 2755 | // slice, create one for the outer loop |
| 2756 | phi = PhiNode::make(this, first->in(MemNode::Memory), Type::MEMORY, |
| 2757 | igvn->C->get_adr_type(igvn->C->get_alias_index(u->adr_type()))); |
| 2758 | phi->set_req(LoopNode::LoopBackControl, last); |
| 2759 | phi = igvn->transform(phi); |
| 2760 | igvn->replace_input_of(first, MemNode::Memory, phi); |
| 2761 | } else { |
| 2762 | // Or fix the outer loop fix to include |
| 2763 | // that chain of stores. |
| 2764 | Node* be = phi->in(LoopNode::LoopBackControl); |
| 2765 | assert(!(be->is_Store() && old_new[be->_idx] != NULL), "store on the backedge + sunk stores: unsupported")do { if (!(!(be->is_Store() && old_new[be->_idx ] != __null))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2765, "assert(" "!(be->is_Store() && old_new[be->_idx] != __null)" ") failed", "store on the backedge + sunk stores: unsupported" ); ::breakpoint(); } } while (0); |
| 2766 | if (be == first->in(MemNode::Memory)) { |
| 2767 | if (be == phi->in(LoopNode::LoopBackControl)) { |
| 2768 | igvn->replace_input_of(phi, LoopNode::LoopBackControl, last); |
| 2769 | } else { |
| 2770 | igvn->replace_input_of(be, MemNode::Memory, last); |
| 2771 | } |
| 2772 | } else { |
| 2773 | #ifdef ASSERT1 |
| 2774 | if (be == phi->in(LoopNode::LoopBackControl)) { |
| 2775 | assert(phi->in(LoopNode::LoopBackControl) == last, "")do { if (!(phi->in(LoopNode::LoopBackControl) == last)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2775, "assert(" "phi->in(LoopNode::LoopBackControl) == last" ") failed", ""); ::breakpoint(); } } while (0); |
| 2776 | } else { |
| 2777 | assert(be->in(MemNode::Memory) == last, "")do { if (!(be->in(MemNode::Memory) == last)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2777, "assert(" "be->in(MemNode::Memory) == last" ") failed" , ""); ::breakpoint(); } } while (0); |
| 2778 | } |
| 2779 | #endif |
| 2780 | } |
| 2781 | } |
| 2782 | } |
| 2783 | } |
| 2784 | } |
| 2785 | |
| 2786 | if (iv_phi != NULL__null) { |
| 2787 | // Now adjust the inner loop's exit condition |
| 2788 | Node* limit = inner_cl->limit(); |
| 2789 | // If limit < init for stride > 0 (or limit > init for stride < 0), |
| 2790 | // the loop body is run only once. Given limit - init (init - limit resp.) |
| 2791 | // would be negative, the unsigned comparison below would cause |
| 2792 | // the loop body to be run for LoopStripMiningIter. |
| 2793 | Node* max = NULL__null; |
| 2794 | if (stride > 0) { |
| 2795 | max = MaxNode::max_diff_with_zero(limit, iv_phi, TypeInt::INT, *igvn); |
| 2796 | } else { |
| 2797 | max = MaxNode::max_diff_with_zero(iv_phi, limit, TypeInt::INT, *igvn); |
| 2798 | } |
| 2799 | // sub is positive and can be larger than the max signed int |
| 2800 | // value. Use an unsigned min. |
| 2801 | Node* const_iters = igvn->intcon(scaled_iters); |
| 2802 | Node* min = MaxNode::unsigned_min(max, const_iters, TypeInt::make(0, scaled_iters, Type::WidenMin), *igvn); |
| 2803 | // min is the number of iterations for the next inner loop execution: |
| 2804 | // unsigned_min(max(limit - iv_phi, 0), scaled_iters) if stride > 0 |
| 2805 | // unsigned_min(max(iv_phi - limit, 0), scaled_iters) if stride < 0 |
| 2806 | |
| 2807 | Node* new_limit = NULL__null; |
| 2808 | if (stride > 0) { |
| 2809 | new_limit = igvn->transform(new AddINode(min, iv_phi)); |
| 2810 | } else { |
| 2811 | new_limit = igvn->transform(new SubINode(iv_phi, min)); |
| 2812 | } |
| 2813 | Node* inner_cmp = inner_cle->cmp_node(); |
| 2814 | Node* inner_bol = inner_cle->in(CountedLoopEndNode::TestValue); |
| 2815 | Node* outer_bol = inner_bol; |
| 2816 | // cmp node for inner loop may be shared |
| 2817 | inner_cmp = inner_cmp->clone(); |
| 2818 | inner_cmp->set_req(2, new_limit); |
| 2819 | inner_bol = inner_bol->clone(); |
| 2820 | inner_bol->set_req(1, igvn->transform(inner_cmp)); |
| 2821 | igvn->replace_input_of(inner_cle, CountedLoopEndNode::TestValue, igvn->transform(inner_bol)); |
| 2822 | // Set the outer loop's exit condition too |
| 2823 | igvn->replace_input_of(outer_loop_end(), 1, outer_bol); |
| 2824 | } else { |
| 2825 | assert(false, "should be able to adjust outer loop")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2825, "assert(" "false" ") failed", "should be able to adjust outer loop" ); ::breakpoint(); } } while (0); |
| 2826 | IfNode* outer_le = outer_loop_end(); |
| 2827 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); |
| 2828 | igvn->replace_node(outer_le, iff); |
| 2829 | inner_cl->clear_strip_mined(); |
| 2830 | } |
| 2831 | } |
| 2832 | |
| 2833 | const Type* OuterStripMinedLoopEndNode::Value(PhaseGVN* phase) const { |
| 2834 | if (!in(0)) return Type::TOP; |
| 2835 | if (phase->type(in(0)) == Type::TOP) |
| 2836 | return Type::TOP; |
| 2837 | |
| 2838 | // Until expansion, the loop end condition is not set so this should not constant fold. |
| 2839 | if (is_expanded(phase)) { |
| 2840 | return IfNode::Value(phase); |
| 2841 | } |
| 2842 | |
| 2843 | return TypeTuple::IFBOTH; |
| 2844 | } |
| 2845 | |
| 2846 | bool OuterStripMinedLoopEndNode::is_expanded(PhaseGVN *phase) const { |
| 2847 | // The outer strip mined loop head only has Phi uses after expansion |
| 2848 | if (phase->is_IterGVN()) { |
| 2849 | Node* backedge = proj_out_or_null(true); |
| 2850 | if (backedge != NULL__null) { |
| 2851 | Node* head = backedge->unique_ctrl_out(); |
| 2852 | if (head != NULL__null && head->is_OuterStripMinedLoop()) { |
| 2853 | if (head->find_out_with(Op_Phi) != NULL__null) { |
| 2854 | return true; |
| 2855 | } |
| 2856 | } |
| 2857 | } |
| 2858 | } |
| 2859 | return false; |
| 2860 | } |
| 2861 | |
| 2862 | Node *OuterStripMinedLoopEndNode::Ideal(PhaseGVN *phase, bool can_reshape) { |
| 2863 | if (remove_dead_region(phase, can_reshape)) return this; |
| 2864 | |
| 2865 | return NULL__null; |
| 2866 | } |
| 2867 | |
| 2868 | //------------------------------filtered_type-------------------------------- |
| 2869 | // Return a type based on condition control flow |
| 2870 | // A successful return will be a type that is restricted due |
| 2871 | // to a series of dominating if-tests, such as: |
| 2872 | // if (i < 10) { |
| 2873 | // if (i > 0) { |
| 2874 | // here: "i" type is [1..10) |
| 2875 | // } |
| 2876 | // } |
| 2877 | // or a control flow merge |
| 2878 | // if (i < 10) { |
| 2879 | // do { |
| 2880 | // phi( , ) -- at top of loop type is [min_int..10) |
| 2881 | // i = ? |
| 2882 | // } while ( i < 10) |
| 2883 | // |
| 2884 | const TypeInt* PhaseIdealLoop::filtered_type( Node *n, Node* n_ctrl) { |
| 2885 | assert(n && n->bottom_type()->is_int(), "must be int")do { if (!(n && n->bottom_type()->is_int())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2885, "assert(" "n && n->bottom_type()->is_int()" ") failed", "must be int"); ::breakpoint(); } } while (0); |
| 2886 | const TypeInt* filtered_t = NULL__null; |
| 2887 | if (!n->is_Phi()) { |
| 2888 | assert(n_ctrl != NULL || n_ctrl == C->top(), "valid control")do { if (!(n_ctrl != __null || n_ctrl == C->top())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2888, "assert(" "n_ctrl != __null || n_ctrl == C->top()" ") failed", "valid control"); ::breakpoint(); } } while (0); |
| 2889 | filtered_t = filtered_type_from_dominators(n, n_ctrl); |
| 2890 | |
| 2891 | } else { |
| 2892 | Node* phi = n->as_Phi(); |
| 2893 | Node* region = phi->in(0); |
| 2894 | assert(n_ctrl == NULL || n_ctrl == region, "ctrl parameter must be region")do { if (!(n_ctrl == __null || n_ctrl == region)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2894, "assert(" "n_ctrl == __null || n_ctrl == region" ") failed" , "ctrl parameter must be region"); ::breakpoint(); } } while (0); |
| 2895 | if (region && region != C->top()) { |
| 2896 | for (uint i = 1; i < phi->req(); i++) { |
| 2897 | Node* val = phi->in(i); |
| 2898 | Node* use_c = region->in(i); |
| 2899 | const TypeInt* val_t = filtered_type_from_dominators(val, use_c); |
| 2900 | if (val_t != NULL__null) { |
| 2901 | if (filtered_t == NULL__null) { |
| 2902 | filtered_t = val_t; |
| 2903 | } else { |
| 2904 | filtered_t = filtered_t->meet(val_t)->is_int(); |
| 2905 | } |
| 2906 | } |
| 2907 | } |
| 2908 | } |
| 2909 | } |
| 2910 | const TypeInt* n_t = _igvn.type(n)->is_int(); |
| 2911 | if (filtered_t != NULL__null) { |
| 2912 | n_t = n_t->join(filtered_t)->is_int(); |
| 2913 | } |
| 2914 | return n_t; |
| 2915 | } |
| 2916 | |
| 2917 | |
| 2918 | //------------------------------filtered_type_from_dominators-------------------------------- |
| 2919 | // Return a possibly more restrictive type for val based on condition control flow of dominators |
| 2920 | const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *use_ctrl) { |
| 2921 | if (val->is_Con()) { |
| 2922 | return val->bottom_type()->is_int(); |
| 2923 | } |
| 2924 | uint if_limit = 10; // Max number of dominating if's visited |
| 2925 | const TypeInt* rtn_t = NULL__null; |
| 2926 | |
| 2927 | if (use_ctrl && use_ctrl != C->top()) { |
| 2928 | Node* val_ctrl = get_ctrl(val); |
| 2929 | uint val_dom_depth = dom_depth(val_ctrl); |
| 2930 | Node* pred = use_ctrl; |
| 2931 | uint if_cnt = 0; |
| 2932 | while (if_cnt < if_limit) { |
| 2933 | if ((pred->Opcode() == Op_IfTrue || pred->Opcode() == Op_IfFalse)) { |
| 2934 | if_cnt++; |
| 2935 | const TypeInt* if_t = IfNode::filtered_int_type(&_igvn, val, pred); |
| 2936 | if (if_t != NULL__null) { |
| 2937 | if (rtn_t == NULL__null) { |
| 2938 | rtn_t = if_t; |
| 2939 | } else { |
| 2940 | rtn_t = rtn_t->join(if_t)->is_int(); |
| 2941 | } |
| 2942 | } |
| 2943 | } |
| 2944 | pred = idom(pred); |
| 2945 | if (pred == NULL__null || pred == C->top()) { |
| 2946 | break; |
| 2947 | } |
| 2948 | // Stop if going beyond definition block of val |
| 2949 | if (dom_depth(pred) < val_dom_depth) { |
| 2950 | break; |
| 2951 | } |
| 2952 | } |
| 2953 | } |
| 2954 | return rtn_t; |
| 2955 | } |
| 2956 | |
| 2957 | |
| 2958 | //------------------------------dump_spec-------------------------------------- |
| 2959 | // Dump special per-node info |
| 2960 | #ifndef PRODUCT |
| 2961 | void CountedLoopEndNode::dump_spec(outputStream *st) const { |
| 2962 | if( in(TestValue) != NULL__null && in(TestValue)->is_Bool() ) { |
| 2963 | BoolTest bt( test_trip()); // Added this for g++. |
| 2964 | |
| 2965 | st->print("["); |
| 2966 | bt.dump_on(st); |
| 2967 | st->print("]"); |
| 2968 | } |
| 2969 | st->print(" "); |
| 2970 | IfNode::dump_spec(st); |
| 2971 | } |
| 2972 | #endif |
| 2973 | |
| 2974 | //============================================================================= |
| 2975 | //------------------------------is_member-------------------------------------- |
| 2976 | // Is 'l' a member of 'this'? |
| 2977 | bool IdealLoopTree::is_member(const IdealLoopTree *l) const { |
| 2978 | while( l->_nest > _nest ) l = l->_parent; |
| 2979 | return l == this; |
| 2980 | } |
| 2981 | |
| 2982 | //------------------------------set_nest--------------------------------------- |
| 2983 | // Set loop tree nesting depth. Accumulate _has_call bits. |
| 2984 | int IdealLoopTree::set_nest( uint depth ) { |
| 2985 | assert(depth <= SHRT_MAX, "sanity")do { if (!(depth <= 32767)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2985, "assert(" "depth <= 32767" ") failed", "sanity"); :: breakpoint(); } } while (0); |
| 2986 | _nest = depth; |
| 2987 | int bits = _has_call; |
| 2988 | if( _child ) bits |= _child->set_nest(depth+1); |
| 2989 | if( bits ) _has_call = 1; |
| 2990 | if( _next ) bits |= _next ->set_nest(depth ); |
| 2991 | return bits; |
| 2992 | } |
| 2993 | |
| 2994 | //------------------------------split_fall_in---------------------------------- |
| 2995 | // Split out multiple fall-in edges from the loop header. Move them to a |
| 2996 | // private RegionNode before the loop. This becomes the loop landing pad. |
| 2997 | void IdealLoopTree::split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ) { |
| 2998 | PhaseIterGVN &igvn = phase->_igvn; |
| 2999 | uint i; |
| 3000 | |
| 3001 | // Make a new RegionNode to be the landing pad. |
| 3002 | Node *landing_pad = new RegionNode( fall_in_cnt+1 ); |
| 3003 | phase->set_loop(landing_pad,_parent); |
| 3004 | // Gather all the fall-in control paths into the landing pad |
| 3005 | uint icnt = fall_in_cnt; |
| 3006 | uint oreq = _head->req(); |
| 3007 | for( i = oreq-1; i>0; i-- ) |
| 3008 | if( !phase->is_member( this, _head->in(i) ) ) |
| 3009 | landing_pad->set_req(icnt--,_head->in(i)); |
| 3010 | |
| 3011 | // Peel off PhiNode edges as well |
| 3012 | for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) { |
| 3013 | Node *oj = _head->fast_out(j); |
| 3014 | if( oj->is_Phi() ) { |
| 3015 | PhiNode* old_phi = oj->as_Phi(); |
| 3016 | assert( old_phi->region() == _head, "" )do { if (!(old_phi->region() == _head)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3016, "assert(" "old_phi->region() == _head" ") failed", ""); ::breakpoint(); } } while (0); |
| 3017 | igvn.hash_delete(old_phi); // Yank from hash before hacking edges |
| 3018 | Node *p = PhiNode::make_blank(landing_pad, old_phi); |
| 3019 | uint icnt = fall_in_cnt; |
| 3020 | for( i = oreq-1; i>0; i-- ) { |
| 3021 | if( !phase->is_member( this, _head->in(i) ) ) { |
| 3022 | p->init_req(icnt--, old_phi->in(i)); |
| 3023 | // Go ahead and clean out old edges from old phi |
| 3024 | old_phi->del_req(i); |
| 3025 | } |
| 3026 | } |
| 3027 | // Search for CSE's here, because ZKM.jar does a lot of |
| 3028 | // loop hackery and we need to be a little incremental |
| 3029 | // with the CSE to avoid O(N^2) node blow-up. |
| 3030 | Node *p2 = igvn.hash_find_insert(p); // Look for a CSE |
| 3031 | if( p2 ) { // Found CSE |
| 3032 | p->destruct(&igvn); // Recover useless new node |
| 3033 | p = p2; // Use old node |
| 3034 | } else { |
| 3035 | igvn.register_new_node_with_optimizer(p, old_phi); |
| 3036 | } |
| 3037 | // Make old Phi refer to new Phi. |
| 3038 | old_phi->add_req(p); |
| 3039 | // Check for the special case of making the old phi useless and |
| 3040 | // disappear it. In JavaGrande I have a case where this useless |
| 3041 | // Phi is the loop limit and prevents recognizing a CountedLoop |
| 3042 | // which in turn prevents removing an empty loop. |
| 3043 | Node *id_old_phi = old_phi->Identity(&igvn); |
| 3044 | if( id_old_phi != old_phi ) { // Found a simple identity? |
| 3045 | // Note that I cannot call 'replace_node' here, because |
| 3046 | // that will yank the edge from old_phi to the Region and |
| 3047 | // I'm mid-iteration over the Region's uses. |
| 3048 | for (DUIterator_Last imin, i = old_phi->last_outs(imin); i >= imin; ) { |
| 3049 | Node* use = old_phi->last_out(i); |
| 3050 | igvn.rehash_node_delayed(use); |
| 3051 | uint uses_found = 0; |
| 3052 | for (uint j = 0; j < use->len(); j++) { |
| 3053 | if (use->in(j) == old_phi) { |
| 3054 | if (j < use->req()) use->set_req (j, id_old_phi); |
| 3055 | else use->set_prec(j, id_old_phi); |
| 3056 | uses_found++; |
| 3057 | } |
| 3058 | } |
| 3059 | i -= uses_found; // we deleted 1 or more copies of this edge |
| 3060 | } |
| 3061 | } |
| 3062 | igvn._worklist.push(old_phi); |
| 3063 | } |
| 3064 | } |
| 3065 | // Finally clean out the fall-in edges from the RegionNode |
| 3066 | for( i = oreq-1; i>0; i-- ) { |
| 3067 | if( !phase->is_member( this, _head->in(i) ) ) { |
| 3068 | _head->del_req(i); |
| 3069 | } |
| 3070 | } |
| 3071 | igvn.rehash_node_delayed(_head); |
| 3072 | // Transform landing pad |
| 3073 | igvn.register_new_node_with_optimizer(landing_pad, _head); |
| 3074 | // Insert landing pad into the header |
| 3075 | _head->add_req(landing_pad); |
| 3076 | } |
| 3077 | |
| 3078 | //------------------------------split_outer_loop------------------------------- |
| 3079 | // Split out the outermost loop from this shared header. |
| 3080 | void IdealLoopTree::split_outer_loop( PhaseIdealLoop *phase ) { |
| 3081 | PhaseIterGVN &igvn = phase->_igvn; |
| 3082 | |
| 3083 | // Find index of outermost loop; it should also be my tail. |
| 3084 | uint outer_idx = 1; |
| 3085 | while( _head->in(outer_idx) != _tail ) outer_idx++; |
| 3086 | |
| 3087 | // Make a LoopNode for the outermost loop. |
| 3088 | Node *ctl = _head->in(LoopNode::EntryControl); |
| 3089 | Node *outer = new LoopNode( ctl, _head->in(outer_idx) ); |
| 3090 | outer = igvn.register_new_node_with_optimizer(outer, _head); |
| 3091 | phase->set_created_loop_node(); |
| 3092 | |
| 3093 | // Outermost loop falls into '_head' loop |
| 3094 | _head->set_req(LoopNode::EntryControl, outer); |
| 3095 | _head->del_req(outer_idx); |
| 3096 | // Split all the Phis up between '_head' loop and 'outer' loop. |
| 3097 | for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) { |
| 3098 | Node *out = _head->fast_out(j); |
| 3099 | if( out->is_Phi() ) { |
| 3100 | PhiNode *old_phi = out->as_Phi(); |
| 3101 | assert( old_phi->region() == _head, "" )do { if (!(old_phi->region() == _head)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3101, "assert(" "old_phi->region() == _head" ") failed", ""); ::breakpoint(); } } while (0); |
| 3102 | Node *phi = PhiNode::make_blank(outer, old_phi); |
| 3103 | phi->init_req(LoopNode::EntryControl, old_phi->in(LoopNode::EntryControl)); |
| 3104 | phi->init_req(LoopNode::LoopBackControl, old_phi->in(outer_idx)); |
| 3105 | phi = igvn.register_new_node_with_optimizer(phi, old_phi); |
| 3106 | // Make old Phi point to new Phi on the fall-in path |
| 3107 | igvn.replace_input_of(old_phi, LoopNode::EntryControl, phi); |
| 3108 | old_phi->del_req(outer_idx); |
| 3109 | } |
| 3110 | } |
| 3111 | |
| 3112 | // Use the new loop head instead of the old shared one |
| 3113 | _head = outer; |
| 3114 | phase->set_loop(_head, this); |
| 3115 | } |
| 3116 | |
| 3117 | //------------------------------fix_parent------------------------------------- |
| 3118 | static void fix_parent( IdealLoopTree *loop, IdealLoopTree *parent ) { |
| 3119 | loop->_parent = parent; |
| 3120 | if( loop->_child ) fix_parent( loop->_child, loop ); |
| 3121 | if( loop->_next ) fix_parent( loop->_next , parent ); |
| 3122 | } |
| 3123 | |
| 3124 | //------------------------------estimate_path_freq----------------------------- |
| 3125 | static float estimate_path_freq( Node *n ) { |
| 3126 | // Try to extract some path frequency info |
| 3127 | IfNode *iff; |
| 3128 | for( int i = 0; i < 50; i++ ) { // Skip through a bunch of uncommon tests |
| 3129 | uint nop = n->Opcode(); |
| 3130 | if( nop == Op_SafePoint ) { // Skip any safepoint |
| 3131 | n = n->in(0); |
| 3132 | continue; |
| 3133 | } |
| 3134 | if( nop == Op_CatchProj ) { // Get count from a prior call |
| 3135 | // Assume call does not always throw exceptions: means the call-site |
| 3136 | // count is also the frequency of the fall-through path. |
| 3137 | assert( n->is_CatchProj(), "" )do { if (!(n->is_CatchProj())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3137, "assert(" "n->is_CatchProj()" ") failed", ""); ::breakpoint (); } } while (0); |
| 3138 | if( ((CatchProjNode*)n)->_con != CatchProjNode::fall_through_index ) |
| 3139 | return 0.0f; // Assume call exception path is rare |
| 3140 | Node *call = n->in(0)->in(0)->in(0); |
| 3141 | assert( call->is_Call(), "expect a call here" )do { if (!(call->is_Call())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3141, "assert(" "call->is_Call()" ") failed", "expect a call here" ); ::breakpoint(); } } while (0); |
| 3142 | const JVMState *jvms = ((CallNode*)call)->jvms(); |
| 3143 | ciMethodData* methodData = jvms->method()->method_data(); |
| 3144 | if (!methodData->is_mature()) return 0.0f; // No call-site data |
| 3145 | ciProfileData* data = methodData->bci_to_data(jvms->bci()); |
| 3146 | if ((data == NULL__null) || !data->is_CounterData()) { |
| 3147 | // no call profile available, try call's control input |
| 3148 | n = n->in(0); |
| 3149 | continue; |
| 3150 | } |
| 3151 | return data->as_CounterData()->count()/FreqCountInvocations; |
| 3152 | } |
| 3153 | // See if there's a gating IF test |
| 3154 | Node *n_c = n->in(0); |
| 3155 | if( !n_c->is_If() ) break; // No estimate available |
| 3156 | iff = n_c->as_If(); |
| 3157 | if( iff->_fcnt != COUNT_UNKNOWN(-1.0f) ) // Have a valid count? |
| 3158 | // Compute how much count comes on this path |
| 3159 | return ((nop == Op_IfTrue) ? iff->_prob : 1.0f - iff->_prob) * iff->_fcnt; |
| 3160 | // Have no count info. Skip dull uncommon-trap like branches. |
| 3161 | if( (nop == Op_IfTrue && iff->_prob < PROB_LIKELY_MAG(5)(1.0f-(1e-5f))) || |
| 3162 | (nop == Op_IfFalse && iff->_prob > PROB_UNLIKELY_MAG(5)(1e-5f)) ) |
| 3163 | break; |
| 3164 | // Skip through never-taken branch; look for a real loop exit. |
| 3165 | n = iff->in(0); |
| 3166 | } |
| 3167 | return 0.0f; // No estimate available |
| 3168 | } |
| 3169 | |
| 3170 | //------------------------------merge_many_backedges--------------------------- |
| 3171 | // Merge all the backedges from the shared header into a private Region. |
| 3172 | // Feed that region as the one backedge to this loop. |
| 3173 | void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) { |
| 3174 | uint i; |
| 3175 | |
| 3176 | // Scan for the top 2 hottest backedges |
| 3177 | float hotcnt = 0.0f; |
| 3178 | float warmcnt = 0.0f; |
| 3179 | uint hot_idx = 0; |
| 3180 | // Loop starts at 2 because slot 1 is the fall-in path |
| 3181 | for( i = 2; i < _head->req(); i++ ) { |
| 3182 | float cnt = estimate_path_freq(_head->in(i)); |
| 3183 | if( cnt > hotcnt ) { // Grab hottest path |
| 3184 | warmcnt = hotcnt; |
| 3185 | hotcnt = cnt; |
| 3186 | hot_idx = i; |
| 3187 | } else if( cnt > warmcnt ) { // And 2nd hottest path |
| 3188 | warmcnt = cnt; |
| 3189 | } |
| 3190 | } |
| 3191 | |
| 3192 | // See if the hottest backedge is worthy of being an inner loop |
| 3193 | // by being much hotter than the next hottest backedge. |
| 3194 | if( hotcnt <= 0.0001 || |
| 3195 | hotcnt < 2.0*warmcnt ) hot_idx = 0;// No hot backedge |
| 3196 | |
| 3197 | // Peel out the backedges into a private merge point; peel |
| 3198 | // them all except optionally hot_idx. |
| 3199 | PhaseIterGVN &igvn = phase->_igvn; |
| 3200 | |
| 3201 | Node *hot_tail = NULL__null; |
| 3202 | // Make a Region for the merge point |
| 3203 | Node *r = new RegionNode(1); |
| 3204 | for( i = 2; i < _head->req(); i++ ) { |
| 3205 | if( i != hot_idx ) |
| 3206 | r->add_req( _head->in(i) ); |
| 3207 | else hot_tail = _head->in(i); |
| 3208 | } |
| 3209 | igvn.register_new_node_with_optimizer(r, _head); |
| 3210 | // Plug region into end of loop _head, followed by hot_tail |
| 3211 | while( _head->req() > 3 ) _head->del_req( _head->req()-1 ); |
| 3212 | igvn.replace_input_of(_head, 2, r); |
| 3213 | if( hot_idx ) _head->add_req(hot_tail); |
| 3214 | |
| 3215 | // Split all the Phis up between '_head' loop and the Region 'r' |
| 3216 | for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) { |
| 3217 | Node *out = _head->fast_out(j); |
| 3218 | if( out->is_Phi() ) { |
| 3219 | PhiNode* n = out->as_Phi(); |
| 3220 | igvn.hash_delete(n); // Delete from hash before hacking edges |
| 3221 | Node *hot_phi = NULL__null; |
| 3222 | Node *phi = new PhiNode(r, n->type(), n->adr_type()); |
| 3223 | // Check all inputs for the ones to peel out |
| 3224 | uint j = 1; |
| 3225 | for( uint i = 2; i < n->req(); i++ ) { |
| 3226 | if( i != hot_idx ) |
| 3227 | phi->set_req( j++, n->in(i) ); |
| 3228 | else hot_phi = n->in(i); |
| 3229 | } |
| 3230 | // Register the phi but do not transform until whole place transforms |
| 3231 | igvn.register_new_node_with_optimizer(phi, n); |
| 3232 | // Add the merge phi to the old Phi |
| 3233 | while( n->req() > 3 ) n->del_req( n->req()-1 ); |
| 3234 | igvn.replace_input_of(n, 2, phi); |
| 3235 | if( hot_idx ) n->add_req(hot_phi); |
| 3236 | } |
| 3237 | } |
| 3238 | |
| 3239 | |
| 3240 | // Insert a new IdealLoopTree inserted below me. Turn it into a clone |
| 3241 | // of self loop tree. Turn self into a loop headed by _head and with |
| 3242 | // tail being the new merge point. |
| 3243 | IdealLoopTree *ilt = new IdealLoopTree( phase, _head, _tail ); |
| 3244 | phase->set_loop(_tail,ilt); // Adjust tail |
| 3245 | _tail = r; // Self's tail is new merge point |
| 3246 | phase->set_loop(r,this); |
| 3247 | ilt->_child = _child; // New guy has my children |
| 3248 | _child = ilt; // Self has new guy as only child |
| 3249 | ilt->_parent = this; // new guy has self for parent |
| 3250 | ilt->_nest = _nest; // Same nesting depth (for now) |
| 3251 | |
| 3252 | // Starting with 'ilt', look for child loop trees using the same shared |
| 3253 | // header. Flatten these out; they will no longer be loops in the end. |
| 3254 | IdealLoopTree **pilt = &_child; |
| 3255 | while( ilt ) { |
| 3256 | if( ilt->_head == _head ) { |
| 3257 | uint i; |
| 3258 | for( i = 2; i < _head->req(); i++ ) |
| 3259 | if( _head->in(i) == ilt->_tail ) |
| 3260 | break; // Still a loop |
| 3261 | if( i == _head->req() ) { // No longer a loop |
| 3262 | // Flatten ilt. Hang ilt's "_next" list from the end of |
| 3263 | // ilt's '_child' list. Move the ilt's _child up to replace ilt. |
| 3264 | IdealLoopTree **cp = &ilt->_child; |
| 3265 | while( *cp ) cp = &(*cp)->_next; // Find end of child list |
| 3266 | *cp = ilt->_next; // Hang next list at end of child list |
| 3267 | *pilt = ilt->_child; // Move child up to replace ilt |
| 3268 | ilt->_head = NULL__null; // Flag as a loop UNIONED into parent |
| 3269 | ilt = ilt->_child; // Repeat using new ilt |
| 3270 | continue; // do not advance over ilt->_child |
| 3271 | } |
| 3272 | assert( ilt->_tail == hot_tail, "expected to only find the hot inner loop here" )do { if (!(ilt->_tail == hot_tail)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3272, "assert(" "ilt->_tail == hot_tail" ") failed", "expected to only find the hot inner loop here" ); ::breakpoint(); } } while (0); |
| 3273 | phase->set_loop(_head,ilt); |
| 3274 | } |
| 3275 | pilt = &ilt->_child; // Advance to next |
| 3276 | ilt = *pilt; |
| 3277 | } |
| 3278 | |
| 3279 | if( _child ) fix_parent( _child, this ); |
| 3280 | } |
| 3281 | |
| 3282 | //------------------------------beautify_loops--------------------------------- |
| 3283 | // Split shared headers and insert loop landing pads. |
| 3284 | // Insert a LoopNode to replace the RegionNode. |
| 3285 | // Return TRUE if loop tree is structurally changed. |
| 3286 | bool IdealLoopTree::beautify_loops( PhaseIdealLoop *phase ) { |
| 3287 | bool result = false; |
| 3288 | // Cache parts in locals for easy |
| 3289 | PhaseIterGVN &igvn = phase->_igvn; |
| 3290 | |
| 3291 | igvn.hash_delete(_head); // Yank from hash before hacking edges |
| 3292 | |
| 3293 | // Check for multiple fall-in paths. Peel off a landing pad if need be. |
| 3294 | int fall_in_cnt = 0; |
| 3295 | for( uint i = 1; i < _head->req(); i++ ) |
| 3296 | if( !phase->is_member( this, _head->in(i) ) ) |
| 3297 | fall_in_cnt++; |
| 3298 | assert( fall_in_cnt, "at least 1 fall-in path" )do { if (!(fall_in_cnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3298, "assert(" "fall_in_cnt" ") failed", "at least 1 fall-in path" ); ::breakpoint(); } } while (0); |
| 3299 | if( fall_in_cnt > 1 ) // Need a loop landing pad to merge fall-ins |
| 3300 | split_fall_in( phase, fall_in_cnt ); |
| 3301 | |
| 3302 | // Swap inputs to the _head and all Phis to move the fall-in edge to |
| 3303 | // the left. |
| 3304 | fall_in_cnt = 1; |
| 3305 | while( phase->is_member( this, _head->in(fall_in_cnt) ) ) |
| 3306 | fall_in_cnt++; |
| 3307 | if( fall_in_cnt > 1 ) { |
| 3308 | // Since I am just swapping inputs I do not need to update def-use info |
| 3309 | Node *tmp = _head->in(1); |
| 3310 | igvn.rehash_node_delayed(_head); |
| 3311 | _head->set_req( 1, _head->in(fall_in_cnt) ); |
| 3312 | _head->set_req( fall_in_cnt, tmp ); |
| 3313 | // Swap also all Phis |
| 3314 | for (DUIterator_Fast imax, i = _head->fast_outs(imax); i < imax; i++) { |
| 3315 | Node* phi = _head->fast_out(i); |
| 3316 | if( phi->is_Phi() ) { |
| 3317 | igvn.rehash_node_delayed(phi); // Yank from hash before hacking edges |
| 3318 | tmp = phi->in(1); |
| 3319 | phi->set_req( 1, phi->in(fall_in_cnt) ); |
| 3320 | phi->set_req( fall_in_cnt, tmp ); |
| 3321 | } |
| 3322 | } |
| 3323 | } |
| 3324 | assert( !phase->is_member( this, _head->in(1) ), "left edge is fall-in" )do { if (!(!phase->is_member( this, _head->in(1) ))) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3324, "assert(" "!phase->is_member( this, _head->in(1) )" ") failed", "left edge is fall-in"); ::breakpoint(); } } while (0); |
| 3325 | assert( phase->is_member( this, _head->in(2) ), "right edge is loop" )do { if (!(phase->is_member( this, _head->in(2) ))) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3325, "assert(" "phase->is_member( this, _head->in(2) )" ") failed", "right edge is loop"); ::breakpoint(); } } while (0); |
| 3326 | |
| 3327 | // If I am a shared header (multiple backedges), peel off the many |
| 3328 | // backedges into a private merge point and use the merge point as |
| 3329 | // the one true backedge. |
| 3330 | if (_head->req() > 3) { |
| 3331 | // Merge the many backedges into a single backedge but leave |
| 3332 | // the hottest backedge as separate edge for the following peel. |
| 3333 | if (!_irreducible) { |
| 3334 | merge_many_backedges( phase ); |
| 3335 | } |
| 3336 | |
| 3337 | // When recursively beautify my children, split_fall_in can change |
| 3338 | // loop tree structure when I am an irreducible loop. Then the head |
| 3339 | // of my children has a req() not bigger than 3. Here we need to set |
| 3340 | // result to true to catch that case in order to tell the caller to |
| 3341 | // rebuild loop tree. See issue JDK-8244407 for details. |
| 3342 | result = true; |
| 3343 | } |
| 3344 | |
| 3345 | // If I have one hot backedge, peel off myself loop. |
| 3346 | // I better be the outermost loop. |
| 3347 | if (_head->req() > 3 && !_irreducible) { |
| 3348 | split_outer_loop( phase ); |
| 3349 | result = true; |
| 3350 | |
| 3351 | } else if (!_head->is_Loop() && !_irreducible) { |
| 3352 | // Make a new LoopNode to replace the old loop head |
| 3353 | Node *l = new LoopNode( _head->in(1), _head->in(2) ); |
| 3354 | l = igvn.register_new_node_with_optimizer(l, _head); |
| 3355 | phase->set_created_loop_node(); |
| 3356 | // Go ahead and replace _head |
| 3357 | phase->_igvn.replace_node( _head, l ); |
| 3358 | _head = l; |
| 3359 | phase->set_loop(_head, this); |
| 3360 | } |
| 3361 | |
| 3362 | // Now recursively beautify nested loops |
| 3363 | if( _child ) result |= _child->beautify_loops( phase ); |
| 3364 | if( _next ) result |= _next ->beautify_loops( phase ); |
| 3365 | return result; |
| 3366 | } |
| 3367 | |
| 3368 | //------------------------------allpaths_check_safepts---------------------------- |
| 3369 | // Allpaths backwards scan from loop tail, terminating each path at first safepoint |
| 3370 | // encountered. Helper for check_safepts. |
| 3371 | void IdealLoopTree::allpaths_check_safepts(VectorSet &visited, Node_List &stack) { |
| 3372 | assert(stack.size() == 0, "empty stack")do { if (!(stack.size() == 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3372, "assert(" "stack.size() == 0" ") failed", "empty stack" ); ::breakpoint(); } } while (0); |
| 3373 | stack.push(_tail); |
| 3374 | visited.clear(); |
| 3375 | visited.set(_tail->_idx); |
| 3376 | while (stack.size() > 0) { |
| 3377 | Node* n = stack.pop(); |
| 3378 | if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) { |
| 3379 | // Terminate this path |
| 3380 | } else if (n->Opcode() == Op_SafePoint) { |
| 3381 | if (_phase->get_loop(n) != this) { |
| 3382 | if (_required_safept == NULL__null) _required_safept = new Node_List(); |
| 3383 | _required_safept->push(n); // save the one closest to the tail |
| 3384 | } |
| 3385 | // Terminate this path |
| 3386 | } else { |
| 3387 | uint start = n->is_Region() ? 1 : 0; |
| 3388 | uint end = n->is_Region() && !n->is_Loop() ? n->req() : start + 1; |
| 3389 | for (uint i = start; i < end; i++) { |
| 3390 | Node* in = n->in(i); |
| 3391 | assert(in->is_CFG(), "must be")do { if (!(in->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3391, "assert(" "in->is_CFG()" ") failed", "must be"); :: breakpoint(); } } while (0); |
| 3392 | if (!visited.test_set(in->_idx) && is_member(_phase->get_loop(in))) { |
| 3393 | stack.push(in); |
| 3394 | } |
| 3395 | } |
| 3396 | } |
| 3397 | } |
| 3398 | } |
| 3399 | |
| 3400 | //------------------------------check_safepts---------------------------- |
| 3401 | // Given dominators, try to find loops with calls that must always be |
| 3402 | // executed (call dominates loop tail). These loops do not need non-call |
| 3403 | // safepoints (ncsfpt). |
| 3404 | // |
| 3405 | // A complication is that a safepoint in a inner loop may be needed |
| 3406 | // by an outer loop. In the following, the inner loop sees it has a |
| 3407 | // call (block 3) on every path from the head (block 2) to the |
| 3408 | // backedge (arc 3->2). So it deletes the ncsfpt (non-call safepoint) |
| 3409 | // in block 2, _but_ this leaves the outer loop without a safepoint. |
| 3410 | // |
| 3411 | // entry 0 |
| 3412 | // | |
| 3413 | // v |
| 3414 | // outer 1,2 +->1 |
| 3415 | // | | |
| 3416 | // | v |
| 3417 | // | 2<---+ ncsfpt in 2 |
| 3418 | // |_/|\ | |
| 3419 | // | v | |
| 3420 | // inner 2,3 / 3 | call in 3 |
| 3421 | // / | | |
| 3422 | // v +--+ |
| 3423 | // exit 4 |
| 3424 | // |
| 3425 | // |
| 3426 | // This method creates a list (_required_safept) of ncsfpt nodes that must |
| 3427 | // be protected is created for each loop. When a ncsfpt maybe deleted, it |
| 3428 | // is first looked for in the lists for the outer loops of the current loop. |
| 3429 | // |
| 3430 | // The insights into the problem: |
| 3431 | // A) counted loops are okay |
| 3432 | // B) innermost loops are okay (only an inner loop can delete |
| 3433 | // a ncsfpt needed by an outer loop) |
| 3434 | // C) a loop is immune from an inner loop deleting a safepoint |
| 3435 | // if the loop has a call on the idom-path |
| 3436 | // D) a loop is also immune if it has a ncsfpt (non-call safepoint) on the |
| 3437 | // idom-path that is not in a nested loop |
| 3438 | // E) otherwise, an ncsfpt on the idom-path that is nested in an inner |
| 3439 | // loop needs to be prevented from deletion by an inner loop |
| 3440 | // |
| 3441 | // There are two analyses: |
| 3442 | // 1) The first, and cheaper one, scans the loop body from |
| 3443 | // tail to head following the idom (immediate dominator) |
| 3444 | // chain, looking for the cases (C,D,E) above. |
| 3445 | // Since inner loops are scanned before outer loops, there is summary |
| 3446 | // information about inner loops. Inner loops can be skipped over |
| 3447 | // when the tail of an inner loop is encountered. |
| 3448 | // |
| 3449 | // 2) The second, invoked if the first fails to find a call or ncsfpt on |
| 3450 | // the idom path (which is rare), scans all predecessor control paths |
| 3451 | // from the tail to the head, terminating a path when a call or sfpt |
| 3452 | // is encountered, to find the ncsfpt's that are closest to the tail. |
| 3453 | // |
| 3454 | void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { |
| 3455 | // Bottom up traversal |
| 3456 | IdealLoopTree* ch = _child; |
| 3457 | if (_child) _child->check_safepts(visited, stack); |
| 3458 | if (_next) _next ->check_safepts(visited, stack); |
| 3459 | |
| 3460 | if (!_head->is_CountedLoop() && !_has_sfpt && _parent != NULL__null && !_irreducible) { |
| 3461 | bool has_call = false; // call on dom-path |
| 3462 | bool has_local_ncsfpt = false; // ncsfpt on dom-path at this loop depth |
| 3463 | Node* nonlocal_ncsfpt = NULL__null; // ncsfpt on dom-path at a deeper depth |
| 3464 | // Scan the dom-path nodes from tail to head |
| 3465 | for (Node* n = tail(); n != _head; n = _phase->idom(n)) { |
| 3466 | if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) { |
| 3467 | has_call = true; |
| 3468 | _has_sfpt = 1; // Then no need for a safept! |
| 3469 | break; |
| 3470 | } else if (n->Opcode() == Op_SafePoint) { |
| 3471 | if (_phase->get_loop(n) == this) { |
| 3472 | has_local_ncsfpt = true; |
| 3473 | break; |
| 3474 | } |
| 3475 | if (nonlocal_ncsfpt == NULL__null) { |
| 3476 | nonlocal_ncsfpt = n; // save the one closest to the tail |
| 3477 | } |
| 3478 | } else { |
| 3479 | IdealLoopTree* nlpt = _phase->get_loop(n); |
| 3480 | if (this != nlpt) { |
| 3481 | // If at an inner loop tail, see if the inner loop has already |
| 3482 | // recorded seeing a call on the dom-path (and stop.) If not, |
| 3483 | // jump to the head of the inner loop. |
| 3484 | assert(is_member(nlpt), "nested loop")do { if (!(is_member(nlpt))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3484, "assert(" "is_member(nlpt)" ") failed", "nested loop" ); ::breakpoint(); } } while (0); |
| 3485 | Node* tail = nlpt->_tail; |
| 3486 | if (tail->in(0)->is_If()) tail = tail->in(0); |
| 3487 | if (n == tail) { |
| 3488 | // If inner loop has call on dom-path, so does outer loop |
| 3489 | if (nlpt->_has_sfpt) { |
| 3490 | has_call = true; |
| 3491 | _has_sfpt = 1; |
| 3492 | break; |
| 3493 | } |
| 3494 | // Skip to head of inner loop |
| 3495 | assert(_phase->is_dominator(_head, nlpt->_head), "inner head dominated by outer head")do { if (!(_phase->is_dominator(_head, nlpt->_head))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3495, "assert(" "_phase->is_dominator(_head, nlpt->_head)" ") failed", "inner head dominated by outer head"); ::breakpoint (); } } while (0); |
| 3496 | n = nlpt->_head; |
| 3497 | } |
| 3498 | } |
| 3499 | } |
| 3500 | } |
| 3501 | // Record safept's that this loop needs preserved when an |
| 3502 | // inner loop attempts to delete it's safepoints. |
| 3503 | if (_child != NULL__null && !has_call && !has_local_ncsfpt) { |
| 3504 | if (nonlocal_ncsfpt != NULL__null) { |
| 3505 | if (_required_safept == NULL__null) _required_safept = new Node_List(); |
| 3506 | _required_safept->push(nonlocal_ncsfpt); |
| 3507 | } else { |
| 3508 | // Failed to find a suitable safept on the dom-path. Now use |
| 3509 | // an all paths walk from tail to head, looking for safepoints to preserve. |
| 3510 | allpaths_check_safepts(visited, stack); |
| 3511 | } |
| 3512 | } |
| 3513 | } |
| 3514 | } |
| 3515 | |
| 3516 | //---------------------------is_deleteable_safept---------------------------- |
| 3517 | // Is safept not required by an outer loop? |
| 3518 | bool PhaseIdealLoop::is_deleteable_safept(Node* sfpt) { |
| 3519 | assert(sfpt->Opcode() == Op_SafePoint, "")do { if (!(sfpt->Opcode() == Op_SafePoint)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3519, "assert(" "sfpt->Opcode() == Op_SafePoint" ") failed" , ""); ::breakpoint(); } } while (0); |
| 3520 | IdealLoopTree* lp = get_loop(sfpt)->_parent; |
| 3521 | while (lp != NULL__null) { |
| 3522 | Node_List* sfpts = lp->_required_safept; |
| 3523 | if (sfpts != NULL__null) { |
| 3524 | for (uint i = 0; i < sfpts->size(); i++) { |
| 3525 | if (sfpt == sfpts->at(i)) |
| 3526 | return false; |
| 3527 | } |
| 3528 | } |
| 3529 | lp = lp->_parent; |
| 3530 | } |
| 3531 | return true; |
| 3532 | } |
| 3533 | |
| 3534 | //---------------------------replace_parallel_iv------------------------------- |
| 3535 | // Replace parallel induction variable (parallel to trip counter) |
| 3536 | void PhaseIdealLoop::replace_parallel_iv(IdealLoopTree *loop) { |
| 3537 | assert(loop->_head->is_CountedLoop(), "")do { if (!(loop->_head->is_CountedLoop())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3537, "assert(" "loop->_head->is_CountedLoop()" ") failed" , ""); ::breakpoint(); } } while (0); |
| 3538 | CountedLoopNode *cl = loop->_head->as_CountedLoop(); |
| 3539 | if (!cl->is_valid_counted_loop(T_INT)) |
| 3540 | return; // skip malformed counted loop |
| 3541 | Node *incr = cl->incr(); |
| 3542 | if (incr == NULL__null) |
| 3543 | return; // Dead loop? |
| 3544 | Node *init = cl->init_trip(); |
| 3545 | Node *phi = cl->phi(); |
| 3546 | int stride_con = cl->stride_con(); |
| 3547 | |
| 3548 | // Visit all children, looking for Phis |
| 3549 | for (DUIterator i = cl->outs(); cl->has_out(i); i++) { |
| 3550 | Node *out = cl->out(i); |
| 3551 | // Look for other phis (secondary IVs). Skip dead ones |
| 3552 | if (!out->is_Phi() || out == phi || !has_node(out)) |
| 3553 | continue; |
| 3554 | PhiNode* phi2 = out->as_Phi(); |
| 3555 | Node *incr2 = phi2->in( LoopNode::LoopBackControl ); |
| 3556 | // Look for induction variables of the form: X += constant |
| 3557 | if (phi2->region() != loop->_head || |
| 3558 | incr2->req() != 3 || |
| 3559 | incr2->in(1)->uncast() != phi2 || |
| 3560 | incr2 == incr || |
| 3561 | incr2->Opcode() != Op_AddI || |
| 3562 | !incr2->in(2)->is_Con()) |
| 3563 | continue; |
| 3564 | |
| 3565 | // Check for parallel induction variable (parallel to trip counter) |
| 3566 | // via an affine function. In particular, count-down loops with |
| 3567 | // count-up array indices are common. We only RCE references off |
| 3568 | // the trip-counter, so we need to convert all these to trip-counter |
| 3569 | // expressions. |
| 3570 | Node *init2 = phi2->in( LoopNode::EntryControl ); |
| 3571 | int stride_con2 = incr2->in(2)->get_int(); |
| 3572 | |
| 3573 | // The ratio of the two strides cannot be represented as an int |
| 3574 | // if stride_con2 is min_int and stride_con is -1. |
| 3575 | if (stride_con2 == min_jint && stride_con == -1) { |
| 3576 | continue; |
| 3577 | } |
| 3578 | |
| 3579 | // The general case here gets a little tricky. We want to find the |
| 3580 | // GCD of all possible parallel IV's and make a new IV using this |
| 3581 | // GCD for the loop. Then all possible IVs are simple multiples of |
| 3582 | // the GCD. In practice, this will cover very few extra loops. |
| 3583 | // Instead we require 'stride_con2' to be a multiple of 'stride_con', |
| 3584 | // where +/-1 is the common case, but other integer multiples are |
| 3585 | // also easy to handle. |
| 3586 | int ratio_con = stride_con2/stride_con; |
| 3587 | |
| 3588 | if ((ratio_con * stride_con) == stride_con2) { // Check for exact |
| 3589 | #ifndef PRODUCT |
| 3590 | if (TraceLoopOpts) { |
| 3591 | tty->print("Parallel IV: %d ", phi2->_idx); |
| 3592 | loop->dump_head(); |
| 3593 | } |
| 3594 | #endif |
| 3595 | // Convert to using the trip counter. The parallel induction |
| 3596 | // variable differs from the trip counter by a loop-invariant |
| 3597 | // amount, the difference between their respective initial values. |
| 3598 | // It is scaled by the 'ratio_con'. |
| 3599 | Node* ratio = _igvn.intcon(ratio_con); |
| 3600 | set_ctrl(ratio, C->root()); |
| 3601 | Node* ratio_init = new MulINode(init, ratio); |
| 3602 | _igvn.register_new_node_with_optimizer(ratio_init, init); |
| 3603 | set_early_ctrl(ratio_init, false); |
| 3604 | Node* diff = new SubINode(init2, ratio_init); |
| 3605 | _igvn.register_new_node_with_optimizer(diff, init2); |
| 3606 | set_early_ctrl(diff, false); |
| 3607 | Node* ratio_idx = new MulINode(phi, ratio); |
| 3608 | _igvn.register_new_node_with_optimizer(ratio_idx, phi); |
| 3609 | set_ctrl(ratio_idx, cl); |
| 3610 | Node* add = new AddINode(ratio_idx, diff); |
| 3611 | _igvn.register_new_node_with_optimizer(add); |
| 3612 | set_ctrl(add, cl); |
| 3613 | _igvn.replace_node( phi2, add ); |
| 3614 | // Sometimes an induction variable is unused |
| 3615 | if (add->outcnt() == 0) { |
| 3616 | _igvn.remove_dead_node(add); |
| 3617 | } |
| 3618 | --i; // deleted this phi; rescan starting with next position |
| 3619 | continue; |
| 3620 | } |
| 3621 | } |
| 3622 | } |
| 3623 | |
| 3624 | void IdealLoopTree::remove_safepoints(PhaseIdealLoop* phase, bool keep_one) { |
| 3625 | Node* keep = NULL__null; |
| 3626 | if (keep_one) { |
| 3627 | // Look for a safepoint on the idom-path. |
| 3628 | for (Node* i = tail(); i != _head; i = phase->idom(i)) { |
| 3629 | if (i->Opcode() == Op_SafePoint && phase->get_loop(i) == this) { |
| 3630 | keep = i; |
| 3631 | break; // Found one |
| 3632 | } |
| 3633 | } |
| 3634 | } |
| 3635 | |
| 3636 | // Don't remove any safepoints if it is requested to keep a single safepoint and |
| 3637 | // no safepoint was found on idom-path. It is not safe to remove any safepoint |
| 3638 | // in this case since there's no safepoint dominating all paths in the loop body. |
| 3639 | bool prune = !keep_one || keep != NULL__null; |
| 3640 | |
| 3641 | // Delete other safepoints in this loop. |
| 3642 | Node_List* sfpts = _safepts; |
| 3643 | if (prune && sfpts != NULL__null) { |
| 3644 | assert(keep == NULL || keep->Opcode() == Op_SafePoint, "not safepoint")do { if (!(keep == __null || keep->Opcode() == Op_SafePoint )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3644, "assert(" "keep == __null || keep->Opcode() == Op_SafePoint" ") failed", "not safepoint"); ::breakpoint(); } } while (0); |
| 3645 | for (uint i = 0; i < sfpts->size(); i++) { |
| 3646 | Node* n = sfpts->at(i); |
| 3647 | assert(phase->get_loop(n) == this, "")do { if (!(phase->get_loop(n) == this)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3647, "assert(" "phase->get_loop(n) == this" ") failed", ""); ::breakpoint(); } } while (0); |
| 3648 | if (n != keep && phase->is_deleteable_safept(n)) { |
| 3649 | phase->lazy_replace(n, n->in(TypeFunc::Control)); |
| 3650 | } |
| 3651 | } |
| 3652 | } |
| 3653 | } |
| 3654 | |
| 3655 | //------------------------------counted_loop----------------------------------- |
| 3656 | // Convert to counted loops where possible |
| 3657 | void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) { |
| 3658 | |
| 3659 | // For grins, set the inner-loop flag here |
| 3660 | if (!_child) { |
| 3661 | if (_head->is_Loop()) _head->as_Loop()->set_inner_loop(); |
| 3662 | } |
| 3663 | |
| 3664 | IdealLoopTree* loop = this; |
| 3665 | if (_head->is_CountedLoop() || |
| 3666 | phase->is_counted_loop(_head, loop, T_INT)) { |
| 3667 | |
| 3668 | if (LoopStripMiningIter == 0 || (LoopStripMiningIter > 1 && _child == NULL__null)) { |
| 3669 | // Indicate we do not need a safepoint here |
| 3670 | _has_sfpt = 1; |
| 3671 | } |
| 3672 | |
| 3673 | // Remove safepoints |
| 3674 | bool keep_one_sfpt = !(_has_call || _has_sfpt); |
| 3675 | remove_safepoints(phase, keep_one_sfpt); |
| 3676 | |
| 3677 | // Look for induction variables |
| 3678 | phase->replace_parallel_iv(this); |
| 3679 | } else if (_head->is_LongCountedLoop() || |
| 3680 | phase->is_counted_loop(_head, loop, T_LONG)) { |
| 3681 | remove_safepoints(phase, true); |
| 3682 | } else { |
| 3683 | assert(!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop(), "transformation to counted loop should not fail")do { if (!(!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3683, "assert(" "!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop()" ") failed", "transformation to counted loop should not fail" ); ::breakpoint(); } } while (0); |
| 3684 | if (_parent != NULL__null && !_irreducible) { |
| 3685 | // Not a counted loop. Keep one safepoint. |
| 3686 | bool keep_one_sfpt = true; |
| 3687 | remove_safepoints(phase, keep_one_sfpt); |
| 3688 | } |
| 3689 | } |
| 3690 | |
| 3691 | // Recursively |
| 3692 | assert(loop->_child != this || (loop->_head->as_Loop()->is_OuterStripMinedLoop() && _head->as_CountedLoop()->is_strip_mined()), "what kind of loop was added?")do { if (!(loop->_child != this || (loop->_head->as_Loop ()->is_OuterStripMinedLoop() && _head->as_CountedLoop ()->is_strip_mined()))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3692, "assert(" "loop->_child != this || (loop->_head->as_Loop()->is_OuterStripMinedLoop() && _head->as_CountedLoop()->is_strip_mined())" ") failed", "what kind of loop was added?"); ::breakpoint(); } } while (0); |
| 3693 | assert(loop->_child != this || (loop->_child->_child == NULL && loop->_child->_next == NULL), "would miss some loops")do { if (!(loop->_child != this || (loop->_child->_child == __null && loop->_child->_next == __null))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3693, "assert(" "loop->_child != this || (loop->_child->_child == __null && loop->_child->_next == __null)" ") failed", "would miss some loops"); ::breakpoint(); } } while (0); |
| 3694 | if (loop->_child && loop->_child != this) loop->_child->counted_loop(phase); |
| 3695 | if (loop->_next) loop->_next ->counted_loop(phase); |
| 3696 | } |
| 3697 | |
| 3698 | |
| 3699 | // The Estimated Loop Clone Size: |
| 3700 | // CloneFactor * (~112% * BodySize + BC) + CC + FanOutTerm, |
| 3701 | // where BC and CC are totally ad-hoc/magic "body" and "clone" constants, |
| 3702 | // respectively, used to ensure that the node usage estimates made are on the |
| 3703 | // safe side, for the most part. The FanOutTerm is an attempt to estimate the |
| 3704 | // possible additional/excessive nodes generated due to data and control flow |
| 3705 | // merging, for edges reaching outside the loop. |
| 3706 | uint IdealLoopTree::est_loop_clone_sz(uint factor) const { |
| 3707 | |
| 3708 | precond(0 < factor && factor < 16)do { if (!(0 < factor && factor < 16)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3708, "assert(" "0 < factor && factor < 16" ") failed" , "precond"); ::breakpoint(); } } while (0); |
| 3709 | |
| 3710 | uint const bc = 13; |
| 3711 | uint const cc = 17; |
| 3712 | uint const sz = _body.size() + (_body.size() + 7) / 2; |
| 3713 | uint estimate = factor * (sz + bc) + cc; |
| 3714 | |
| 3715 | assert((estimate - cc) / factor == sz + bc, "overflow")do { if (!((estimate - cc) / factor == sz + bc)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3715, "assert(" "(estimate - cc) / factor == sz + bc" ") failed" , "overflow"); ::breakpoint(); } } while (0); |
| 3716 | |
| 3717 | return estimate + est_loop_flow_merge_sz(); |
| 3718 | } |
| 3719 | |
| 3720 | // The Estimated Loop (full-) Unroll Size: |
| 3721 | // UnrollFactor * (~106% * BodySize) + CC + FanOutTerm, |
| 3722 | // where CC is a (totally) ad-hoc/magic "clone" constant, used to ensure that |
| 3723 | // node usage estimates made are on the safe side, for the most part. This is |
| 3724 | // a "light" version of the loop clone size calculation (above), based on the |
| 3725 | // assumption that most of the loop-construct overhead will be unraveled when |
| 3726 | // (fully) unrolled. Defined for unroll factors larger or equal to one (>=1), |
| 3727 | // including an overflow check and returning UINT_MAX in case of an overflow. |
| 3728 | uint IdealLoopTree::est_loop_unroll_sz(uint factor) const { |
| 3729 | |
| 3730 | precond(factor > 0)do { if (!(factor > 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3730, "assert(" "factor > 0" ") failed", "precond"); ::breakpoint (); } } while (0); |
| 3731 | |
| 3732 | // Take into account that after unroll conjoined heads and tails will fold. |
| 3733 | uint const b0 = _body.size() - EMPTY_LOOP_SIZE; |
| 3734 | uint const cc = 7; |
| 3735 | uint const sz = b0 + (b0 + 15) / 16; |
| 3736 | uint estimate = factor * sz + cc; |
| 3737 | |
| 3738 | if ((estimate - cc) / factor != sz) { |
| 3739 | return UINT_MAX(2147483647 *2U +1U); |
| 3740 | } |
| 3741 | |
| 3742 | return estimate + est_loop_flow_merge_sz(); |
| 3743 | } |
| 3744 | |
| 3745 | // Estimate the growth effect (in nodes) of merging control and data flow when |
| 3746 | // cloning a loop body, based on the amount of control and data flow reaching |
| 3747 | // outside of the (current) loop body. |
| 3748 | uint IdealLoopTree::est_loop_flow_merge_sz() const { |
| 3749 | |
| 3750 | uint ctrl_edge_out_cnt = 0; |
| 3751 | uint data_edge_out_cnt = 0; |
| 3752 | |
| 3753 | for (uint i = 0; i < _body.size(); i++) { |
| 3754 | Node* node = _body.at(i); |
| 3755 | uint outcnt = node->outcnt(); |
| 3756 | |
| 3757 | for (uint k = 0; k < outcnt; k++) { |
| 3758 | Node* out = node->raw_out(k); |
| 3759 | if (out == NULL__null) continue; |
| 3760 | if (out->is_CFG()) { |
| 3761 | if (!is_member(_phase->get_loop(out))) { |
| 3762 | ctrl_edge_out_cnt++; |
| 3763 | } |
| 3764 | } else if (_phase->has_ctrl(out)) { |
| 3765 | Node* ctrl = _phase->get_ctrl(out); |
| 3766 | assert(ctrl != NULL, "must be")do { if (!(ctrl != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3766, "assert(" "ctrl != __null" ") failed", "must be"); :: breakpoint(); } } while (0); |
| 3767 | assert(ctrl->is_CFG(), "must be")do { if (!(ctrl->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3767, "assert(" "ctrl->is_CFG()" ") failed", "must be"); ::breakpoint(); } } while (0); |
| 3768 | if (!is_member(_phase->get_loop(ctrl))) { |
| 3769 | data_edge_out_cnt++; |
| 3770 | } |
| 3771 | } |
| 3772 | } |
| 3773 | } |
| 3774 | // Use data and control count (x2.0) in estimate iff both are > 0. This is |
| 3775 | // a rather pessimistic estimate for the most part, in particular for some |
| 3776 | // complex loops, but still not enough to capture all loops. |
| 3777 | if (ctrl_edge_out_cnt > 0 && data_edge_out_cnt > 0) { |
| 3778 | return 2 * (ctrl_edge_out_cnt + data_edge_out_cnt); |
| 3779 | } |
| 3780 | return 0; |
| 3781 | } |
| 3782 | |
| 3783 | #ifndef PRODUCT |
| 3784 | //------------------------------dump_head-------------------------------------- |
| 3785 | // Dump 1 liner for loop header info |
| 3786 | void IdealLoopTree::dump_head() const { |
| 3787 | tty->sp(2 * _nest); |
| 3788 | tty->print("Loop: N%d/N%d ", _head->_idx, _tail->_idx); |
| 3789 | if (_irreducible) tty->print(" IRREDUCIBLE"); |
| 3790 | Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl) : _head->in(LoopNode::EntryControl); |
| 3791 | Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); |
| 3792 | if (predicate != NULL__null ) { |
| 3793 | tty->print(" limit_check"); |
| 3794 | entry = PhaseIdealLoop::skip_loop_predicates(entry); |
| 3795 | } |
| 3796 | if (UseProfiledLoopPredicate) { |
| 3797 | predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); |
| 3798 | if (predicate != NULL__null) { |
| 3799 | tty->print(" profile_predicated"); |
| 3800 | entry = PhaseIdealLoop::skip_loop_predicates(entry); |
| 3801 | } |
| 3802 | } |
| 3803 | if (UseLoopPredicate) { |
| 3804 | predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
| 3805 | if (predicate != NULL__null) { |
| 3806 | tty->print(" predicated"); |
| 3807 | } |
| 3808 | } |
| 3809 | if (_head->is_CountedLoop()) { |
| 3810 | CountedLoopNode *cl = _head->as_CountedLoop(); |
| 3811 | tty->print(" counted"); |
| 3812 | |
| 3813 | Node* init_n = cl->init_trip(); |
| 3814 | if (init_n != NULL__null && init_n->is_Con()) |
| 3815 | tty->print(" [%d,", cl->init_trip()->get_int()); |
| 3816 | else |
| 3817 | tty->print(" [int,"); |
| 3818 | Node* limit_n = cl->limit(); |
| 3819 | if (limit_n != NULL__null && limit_n->is_Con()) |
| 3820 | tty->print("%d),", cl->limit()->get_int()); |
| 3821 | else |
| 3822 | tty->print("int),"); |
| 3823 | int stride_con = cl->stride_con(); |
| 3824 | if (stride_con > 0) tty->print("+"); |
| 3825 | tty->print("%d", stride_con); |
| 3826 | |
| 3827 | tty->print(" (%0.f iters) ", cl->profile_trip_cnt()); |
| 3828 | |
| 3829 | if (cl->is_pre_loop ()) tty->print(" pre" ); |
| 3830 | if (cl->is_main_loop()) tty->print(" main"); |
| 3831 | if (cl->is_post_loop()) tty->print(" post"); |
| 3832 | if (cl->is_vectorized_loop()) tty->print(" vector"); |
| 3833 | if (cl->range_checks_present()) tty->print(" rc "); |
| 3834 | if (cl->is_multiversioned()) tty->print(" multi "); |
| 3835 | } |
| 3836 | if (_has_call) tty->print(" has_call"); |
| 3837 | if (_has_sfpt) tty->print(" has_sfpt"); |
| 3838 | if (_rce_candidate) tty->print(" rce"); |
| 3839 | if (_safepts != NULL__null && _safepts->size() > 0) { |
| 3840 | tty->print(" sfpts={"); _safepts->dump_simple(); tty->print(" }"); |
| 3841 | } |
| 3842 | if (_required_safept != NULL__null && _required_safept->size() > 0) { |
| 3843 | tty->print(" req={"); _required_safept->dump_simple(); tty->print(" }"); |
| 3844 | } |
| 3845 | if (Verbose) { |
| 3846 | tty->print(" body={"); _body.dump_simple(); tty->print(" }"); |
| 3847 | } |
| 3848 | if (_head->is_Loop() && _head->as_Loop()->is_strip_mined()) { |
| 3849 | tty->print(" strip_mined"); |
| 3850 | } |
| 3851 | tty->cr(); |
| 3852 | } |
| 3853 | |
| 3854 | //------------------------------dump------------------------------------------- |
| 3855 | // Dump loops by loop tree |
| 3856 | void IdealLoopTree::dump() const { |
| 3857 | dump_head(); |
| 3858 | if (_child) _child->dump(); |
| 3859 | if (_next) _next ->dump(); |
| 3860 | } |
| 3861 | |
| 3862 | #endif |
| 3863 | |
| 3864 | static void log_loop_tree_helper(IdealLoopTree* root, IdealLoopTree* loop, CompileLog* log) { |
| 3865 | if (loop == root) { |
| 3866 | if (loop->_child != NULL__null) { |
| 3867 | log->begin_head("loop_tree"); |
| 3868 | log->end_head(); |
| 3869 | log_loop_tree_helper(root, loop->_child, log); |
| 3870 | log->tail("loop_tree"); |
| 3871 | assert(loop->_next == NULL, "what?")do { if (!(loop->_next == __null)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3871, "assert(" "loop->_next == __null" ") failed", "what?" ); ::breakpoint(); } } while (0); |
| 3872 | } |
| 3873 | } else if (loop != NULL__null) { |
| 3874 | Node* head = loop->_head; |
| 3875 | log->begin_head("loop"); |
| 3876 | log->print(" idx='%d' ", head->_idx); |
| 3877 | if (loop->_irreducible) log->print("irreducible='1' "); |
| 3878 | if (head->is_Loop()) { |
| 3879 | if (head->as_Loop()->is_inner_loop()) log->print("inner_loop='1' "); |
| 3880 | if (head->as_Loop()->is_partial_peel_loop()) log->print("partial_peel_loop='1' "); |
| 3881 | } else if (head->is_CountedLoop()) { |
| 3882 | CountedLoopNode* cl = head->as_CountedLoop(); |
| 3883 | if (cl->is_pre_loop()) log->print("pre_loop='%d' ", cl->main_idx()); |
| 3884 | if (cl->is_main_loop()) log->print("main_loop='%d' ", cl->_idx); |
| 3885 | if (cl->is_post_loop()) log->print("post_loop='%d' ", cl->main_idx()); |
| 3886 | } |
| 3887 | log->end_head(); |
| 3888 | log_loop_tree_helper(root, loop->_child, log); |
| 3889 | log->tail("loop"); |
| 3890 | log_loop_tree_helper(root, loop->_next, log); |
| 3891 | } |
| 3892 | } |
| 3893 | |
| 3894 | void PhaseIdealLoop::log_loop_tree() { |
| 3895 | if (C->log() != NULL__null) { |
| 3896 | log_loop_tree_helper(_ltree_root, _ltree_root, C->log()); |
| 3897 | } |
| 3898 | } |
| 3899 | |
| 3900 | //---------------------collect_potentially_useful_predicates----------------------- |
| 3901 | // Helper function to collect potentially useful predicates to prevent them from |
| 3902 | // being eliminated by PhaseIdealLoop::eliminate_useless_predicates |
| 3903 | void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, Unique_Node_List &useful_predicates) { |
| 3904 | if (loop->_child) { // child |
| 3905 | collect_potentially_useful_predicates(loop->_child, useful_predicates); |
| 3906 | } |
| 3907 | |
| 3908 | // self (only loops that we can apply loop predication may use their predicates) |
| 3909 | if (loop->_head->is_Loop() && |
| 3910 | !loop->_irreducible && |
| 3911 | !loop->tail()->is_top()) { |
| 3912 | LoopNode* lpn = loop->_head->as_Loop(); |
| 3913 | Node* entry = lpn->in(LoopNode::EntryControl); |
| 3914 | |
| 3915 | Node* predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); |
| 3916 | if (predicate != NULL__null) { // right pattern that can be used by loop predication |
| 3917 | assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be")do { if (!(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3917, "assert(" "entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1" ") failed", "must be"); ::breakpoint(); } } while (0); |
| 3918 | useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one |
| 3919 | entry = skip_loop_predicates(entry); |
| 3920 | } |
| 3921 | if (UseProfiledLoopPredicate) { |
| 3922 | predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); |
| 3923 | if (predicate != NULL__null) { // right pattern that can be used by loop predication |
| 3924 | useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one |
| 3925 | get_skeleton_predicates(entry, useful_predicates, true); |
| 3926 | entry = skip_loop_predicates(entry); |
| 3927 | } |
| 3928 | } |
| 3929 | |
| 3930 | if (UseLoopPredicate) { |
| 3931 | predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); |
| 3932 | if (predicate != NULL__null) { // right pattern that can be used by loop predication |
| 3933 | useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one |
| 3934 | get_skeleton_predicates(entry, useful_predicates, true); |
| 3935 | } |
| 3936 | } |
| 3937 | } |
| 3938 | |
| 3939 | if (loop->_next) { // sibling |
| 3940 | collect_potentially_useful_predicates(loop->_next, useful_predicates); |
| 3941 | } |
| 3942 | } |
| 3943 | |
| 3944 | //------------------------eliminate_useless_predicates----------------------------- |
| 3945 | // Eliminate all inserted predicates if they could not be used by loop predication. |
| 3946 | // Note: it will also eliminates loop limits check predicate since it also uses |
| 3947 | // Opaque1 node (see Parse::add_predicate()). |
| 3948 | void PhaseIdealLoop::eliminate_useless_predicates() { |
| 3949 | if (C->predicate_count() == 0 && C->skeleton_predicate_count() == 0) { |
| 3950 | return; // no predicate left |
| 3951 | } |
| 3952 | |
| 3953 | Unique_Node_List useful_predicates; // to store useful predicates |
| 3954 | if (C->has_loops()) { |
| 3955 | collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates); |
| 3956 | } |
| 3957 | |
| 3958 | for (int i = C->predicate_count(); i > 0; i--) { |
| 3959 | Node* n = C->predicate_opaque1_node(i - 1); |
| 3960 | assert(n->Opcode() == Op_Opaque1, "must be")do { if (!(n->Opcode() == Op_Opaque1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3960, "assert(" "n->Opcode() == Op_Opaque1" ") failed", "must be" ); ::breakpoint(); } } while (0); |
| 3961 | if (!useful_predicates.member(n)) { // not in the useful list |
| 3962 | _igvn.replace_node(n, n->in(1)); |
| 3963 | } |
| 3964 | } |
| 3965 | |
| 3966 | for (int i = C->skeleton_predicate_count(); i > 0; i--) { |
| 3967 | Node* n = C->skeleton_predicate_opaque4_node(i - 1); |
| 3968 | assert(n->Opcode() == Op_Opaque4, "must be")do { if (!(n->Opcode() == Op_Opaque4)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3968, "assert(" "n->Opcode() == Op_Opaque4" ") failed", "must be" ); ::breakpoint(); } } while (0); |
| 3969 | if (!useful_predicates.member(n)) { // not in the useful list |
| 3970 | _igvn.replace_node(n, n->in(2)); |
| 3971 | } |
| 3972 | } |
| 3973 | } |
| 3974 | |
| 3975 | //------------------------process_expensive_nodes----------------------------- |
| 3976 | // Expensive nodes have their control input set to prevent the GVN |
| 3977 | // from commoning them and as a result forcing the resulting node to |
| 3978 | // be in a more frequent path. Use CFG information here, to change the |
| 3979 | // control inputs so that some expensive nodes can be commoned while |
| 3980 | // not executed more frequently. |
| 3981 | bool PhaseIdealLoop::process_expensive_nodes() { |
| 3982 | assert(OptimizeExpensiveOps, "optimization off?")do { if (!(OptimizeExpensiveOps)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3982, "assert(" "OptimizeExpensiveOps" ") failed", "optimization off?" ); ::breakpoint(); } } while (0); |
| 3983 | |
| 3984 | // Sort nodes to bring similar nodes together |
| 3985 | C->sort_expensive_nodes(); |
| 3986 | |
| 3987 | bool progress = false; |
| 3988 | |
| 3989 | for (int i = 0; i < C->expensive_count(); ) { |
| 3990 | Node* n = C->expensive_node(i); |
| 3991 | int start = i; |
| 3992 | // Find nodes similar to n |
| 3993 | i++; |
| 3994 | for (; i < C->expensive_count() && Compile::cmp_expensive_nodes(n, C->expensive_node(i)) == 0; i++); |
| 3995 | int end = i; |
| 3996 | // And compare them two by two |
| 3997 | for (int j = start; j < end; j++) { |
| 3998 | Node* n1 = C->expensive_node(j); |
| 3999 | if (is_node_unreachable(n1)) { |
| 4000 | continue; |
| 4001 | } |
| 4002 | for (int k = j+1; k < end; k++) { |
| 4003 | Node* n2 = C->expensive_node(k); |
| 4004 | if (is_node_unreachable(n2)) { |
| 4005 | continue; |
| 4006 | } |
| 4007 | |
| 4008 | assert(n1 != n2, "should be pair of nodes")do { if (!(n1 != n2)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4008, "assert(" "n1 != n2" ") failed", "should be pair of nodes" ); ::breakpoint(); } } while (0); |
| 4009 | |
| 4010 | Node* c1 = n1->in(0); |
| 4011 | Node* c2 = n2->in(0); |
| 4012 | |
| 4013 | Node* parent_c1 = c1; |
| 4014 | Node* parent_c2 = c2; |
| 4015 | |
| 4016 | // The call to get_early_ctrl_for_expensive() moves the |
| 4017 | // expensive nodes up but stops at loops that are in a if |
| 4018 | // branch. See whether we can exit the loop and move above the |
| 4019 | // If. |
| 4020 | if (c1->is_Loop()) { |
| 4021 | parent_c1 = c1->in(1); |
| 4022 | } |
| 4023 | if (c2->is_Loop()) { |
| 4024 | parent_c2 = c2->in(1); |
| 4025 | } |
| 4026 | |
| 4027 | if (parent_c1 == parent_c2) { |
| 4028 | _igvn._worklist.push(n1); |
| 4029 | _igvn._worklist.push(n2); |
| 4030 | continue; |
| 4031 | } |
| 4032 | |
| 4033 | // Look for identical expensive node up the dominator chain. |
| 4034 | if (is_dominator(c1, c2)) { |
| 4035 | c2 = c1; |
| 4036 | } else if (is_dominator(c2, c1)) { |
| 4037 | c1 = c2; |
| 4038 | } else if (parent_c1->is_Proj() && parent_c1->in(0)->is_If() && |
| 4039 | parent_c2->is_Proj() && parent_c1->in(0) == parent_c2->in(0)) { |
| 4040 | // Both branches have the same expensive node so move it up |
| 4041 | // before the if. |
| 4042 | c1 = c2 = idom(parent_c1->in(0)); |
| 4043 | } |
| 4044 | // Do the actual moves |
| 4045 | if (n1->in(0) != c1) { |
| 4046 | _igvn.hash_delete(n1); |
| 4047 | n1->set_req(0, c1); |
| 4048 | _igvn.hash_insert(n1); |
| 4049 | _igvn._worklist.push(n1); |
| 4050 | progress = true; |
| 4051 | } |
| 4052 | if (n2->in(0) != c2) { |
| 4053 | _igvn.hash_delete(n2); |
| 4054 | n2->set_req(0, c2); |
| 4055 | _igvn.hash_insert(n2); |
| 4056 | _igvn._worklist.push(n2); |
| 4057 | progress = true; |
| 4058 | } |
| 4059 | } |
| 4060 | } |
| 4061 | } |
| 4062 | |
| 4063 | return progress; |
| 4064 | } |
| 4065 | |
| 4066 | #ifdef ASSERT1 |
| 4067 | bool PhaseIdealLoop::only_has_infinite_loops() { |
| 4068 | for (IdealLoopTree* l = _ltree_root->_child; l != NULL__null; l = l->_next) { |
| 4069 | uint i = 1; |
| 4070 | for (; i < C->root()->req(); i++) { |
| 4071 | Node* in = C->root()->in(i); |
| 4072 | if (in != NULL__null && |
| 4073 | in->Opcode() == Op_Halt && |
| 4074 | in->in(0)->is_Proj() && |
| 4075 | in->in(0)->in(0)->Opcode() == Op_NeverBranch && |
| 4076 | in->in(0)->in(0)->in(0) == l->_head) { |
| 4077 | break; |
| 4078 | } |
| 4079 | } |
| 4080 | if (i == C->root()->req()) { |
| 4081 | return false; |
| 4082 | } |
| 4083 | } |
| 4084 | |
| 4085 | return true; |
| 4086 | } |
| 4087 | #endif |
| 4088 | |
| 4089 | |
| 4090 | //============================================================================= |
| 4091 | //----------------------------build_and_optimize------------------------------- |
| 4092 | // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to |
| 4093 | // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. |
| 4094 | void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) { |
| 4095 | assert(!C->post_loop_opts_phase(), "no loop opts allowed")do { if (!(!C->post_loop_opts_phase())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4095, "assert(" "!C->post_loop_opts_phase()" ") failed", "no loop opts allowed"); ::breakpoint(); } } while (0); |
| 4096 | |
| 4097 | bool do_split_ifs = (mode == LoopOptsDefault); |
| 4098 | bool skip_loop_opts = (mode == LoopOptsNone); |
| 4099 | |
| 4100 | int old_progress = C->major_progress(); |
| 4101 | uint orig_worklist_size = _igvn._worklist.size(); |
| 4102 | |
| 4103 | // Reset major-progress flag for the driver's heuristics |
| 4104 | C->clear_major_progress(); |
| 4105 | |
| 4106 | #ifndef PRODUCT |
| 4107 | // Capture for later assert |
| 4108 | uint unique = C->unique(); |
| 4109 | _loop_invokes++; |
| 4110 | _loop_work += unique; |
| 4111 | #endif |
| 4112 | |
| 4113 | // True if the method has at least 1 irreducible loop |
| 4114 | _has_irreducible_loops = false; |
| 4115 | |
| 4116 | _created_loop_node = false; |
| 4117 | |
| 4118 | VectorSet visited; |
| 4119 | // Pre-grow the mapping from Nodes to IdealLoopTrees. |
| 4120 | _nodes.map(C->unique(), NULL__null); |
| 4121 | memset(_nodes.adr(), 0, wordSize * C->unique()); |
| 4122 | |
| 4123 | // Pre-build the top-level outermost loop tree entry |
| 4124 | _ltree_root = new IdealLoopTree( this, C->root(), C->root() ); |
| 4125 | // Do not need a safepoint at the top level |
| 4126 | _ltree_root->_has_sfpt = 1; |
| 4127 | |
| 4128 | // Initialize Dominators. |
| 4129 | // Checked in clone_loop_predicate() during beautify_loops(). |
| 4130 | _idom_size = 0; |
| 4131 | _idom = NULL__null; |
| 4132 | _dom_depth = NULL__null; |
| 4133 | _dom_stk = NULL__null; |
| 4134 | |
| 4135 | // Empty pre-order array |
| 4136 | allocate_preorders(); |
| 4137 | |
| 4138 | // Build a loop tree on the fly. Build a mapping from CFG nodes to |
| 4139 | // IdealLoopTree entries. Data nodes are NOT walked. |
| 4140 | build_loop_tree(); |
| 4141 | // Check for bailout, and return |
| 4142 | if (C->failing()) { |
| 4143 | return; |
| 4144 | } |
| 4145 | |
| 4146 | // Verify that the has_loops() flag set at parse time is consistent |
| 4147 | // with the just built loop tree. With infinite loops, it could be |
| 4148 | // that one pass of loop opts only finds infinite loops, clears the |
| 4149 | // has_loops() flag but adds NeverBranch nodes so the next loop opts |
| 4150 | // verification pass finds a non empty loop tree. When the back edge |
| 4151 | // is an exception edge, parsing doesn't set has_loops(). |
| 4152 | assert(_ltree_root->_child == NULL || C->has_loops() || only_has_infinite_loops() || C->has_exception_backedge(), "parsing found no loops but there are some")do { if (!(_ltree_root->_child == __null || C->has_loops () || only_has_infinite_loops() || C->has_exception_backedge ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4152, "assert(" "_ltree_root->_child == __null || C->has_loops() || only_has_infinite_loops() || C->has_exception_backedge()" ") failed", "parsing found no loops but there are some"); :: breakpoint(); } } while (0); |
| 4153 | // No loops after all |
| 4154 | if( !_ltree_root->_child && !_verify_only ) C->set_has_loops(false); |
| 4155 | |
| 4156 | // There should always be an outer loop containing the Root and Return nodes. |
| 4157 | // If not, we have a degenerate empty program. Bail out in this case. |
| 4158 | if (!has_node(C->root())) { |
| 4159 | if (!_verify_only) { |
| 4160 | C->clear_major_progress(); |
| 4161 | C->record_method_not_compilable("empty program detected during loop optimization"); |
| 4162 | } |
| 4163 | return; |
| 4164 | } |
| 4165 | |
| 4166 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
| 4167 | // Nothing to do, so get out |
| 4168 | bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only && |
| 4169 | !bs->is_gc_specific_loop_opts_pass(mode); |
| 4170 | bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn); |
| 4171 | bool strip_mined_loops_expanded = bs->strip_mined_loops_expanded(mode); |
| 4172 | if (stop_early && !do_expensive_nodes) { |
| 4173 | return; |
| 4174 | } |
| 4175 | |
| 4176 | // Set loop nesting depth |
| 4177 | _ltree_root->set_nest( 0 ); |
| 4178 | |
| 4179 | // Split shared headers and insert loop landing pads. |
| 4180 | // Do not bother doing this on the Root loop of course. |
| 4181 | if( !_verify_me && !_verify_only && _ltree_root->_child ) { |
| 4182 | C->print_method(PHASE_BEFORE_BEAUTIFY_LOOPS, 3); |
| 4183 | if( _ltree_root->_child->beautify_loops( this ) ) { |
| 4184 | // Re-build loop tree! |
| 4185 | _ltree_root->_child = NULL__null; |
| 4186 | _nodes.clear(); |
| 4187 | reallocate_preorders(); |
| 4188 | build_loop_tree(); |
| 4189 | // Check for bailout, and return |
| 4190 | if (C->failing()) { |
| 4191 | return; |
| 4192 | } |
| 4193 | // Reset loop nesting depth |
| 4194 | _ltree_root->set_nest( 0 ); |
| 4195 | |
| 4196 | C->print_method(PHASE_AFTER_BEAUTIFY_LOOPS, 3); |
| 4197 | } |
| 4198 | } |
| 4199 | |
| 4200 | // Build Dominators for elision of NULL checks & loop finding. |
| 4201 | // Since nodes do not have a slot for immediate dominator, make |
| 4202 | // a persistent side array for that info indexed on node->_idx. |
| 4203 | _idom_size = C->unique(); |
| 4204 | _idom = NEW_RESOURCE_ARRAY( Node*, _idom_size )(Node**) resource_allocate_bytes((_idom_size) * sizeof(Node*) ); |
| 4205 | _dom_depth = NEW_RESOURCE_ARRAY( uint, _idom_size )(uint*) resource_allocate_bytes((_idom_size) * sizeof(uint)); |
| 4206 | _dom_stk = NULL__null; // Allocated on demand in recompute_dom_depth |
| 4207 | memset( _dom_depth, 0, _idom_size * sizeof(uint) ); |
| 4208 | |
| 4209 | Dominators(); |
| 4210 | |
| 4211 | if (!_verify_only) { |
| 4212 | // As a side effect, Dominators removed any unreachable CFG paths |
| 4213 | // into RegionNodes. It doesn't do this test against Root, so |
| 4214 | // we do it here. |
| 4215 | for( uint i = 1; i < C->root()->req(); i++ ) { |
| 4216 | if( !_nodes[C->root()->in(i)->_idx] ) { // Dead path into Root? |
| 4217 | _igvn.delete_input_of(C->root(), i); |
| 4218 | i--; // Rerun same iteration on compressed edges |
| 4219 | } |
| 4220 | } |
| 4221 | |
| 4222 | // Given dominators, try to find inner loops with calls that must |
| 4223 | // always be executed (call dominates loop tail). These loops do |
| 4224 | // not need a separate safepoint. |
| 4225 | Node_List cisstack; |
| 4226 | _ltree_root->check_safepts(visited, cisstack); |
| 4227 | } |
| 4228 | |
| 4229 | // Walk the DATA nodes and place into loops. Find earliest control |
| 4230 | // node. For CFG nodes, the _nodes array starts out and remains |
| 4231 | // holding the associated IdealLoopTree pointer. For DATA nodes, the |
| 4232 | // _nodes array holds the earliest legal controlling CFG node. |
| 4233 | |
| 4234 | // Allocate stack with enough space to avoid frequent realloc |
| 4235 | int stack_size = (C->live_nodes() >> 1) + 16; // (live_nodes>>1)+16 from Java2D stats |
| 4236 | Node_Stack nstack(stack_size); |
| 4237 | |
| 4238 | visited.clear(); |
| 4239 | Node_List worklist; |
| 4240 | // Don't need C->root() on worklist since |
| 4241 | // it will be processed among C->top() inputs |
| 4242 | worklist.push(C->top()); |
| 4243 | visited.set(C->top()->_idx); // Set C->top() as visited now |
| 4244 | build_loop_early( visited, worklist, nstack ); |
| 4245 | |
| 4246 | // Given early legal placement, try finding counted loops. This placement |
| 4247 | // is good enough to discover most loop invariants. |
| 4248 | if (!_verify_me && !_verify_only && !strip_mined_loops_expanded) { |
| 4249 | _ltree_root->counted_loop( this ); |
| 4250 | } |
| 4251 | |
| 4252 | // Find latest loop placement. Find ideal loop placement. |
| 4253 | visited.clear(); |
| 4254 | init_dom_lca_tags(); |
| 4255 | // Need C->root() on worklist when processing outs |
| 4256 | worklist.push(C->root()); |
| 4257 | NOT_PRODUCT( C->verify_graph_edges(); )C->verify_graph_edges(); |
| 4258 | worklist.push(C->top()); |
| 4259 | build_loop_late( visited, worklist, nstack ); |
| 4260 | |
| 4261 | if (_verify_only) { |
| 4262 | C->restore_major_progress(old_progress); |
| 4263 | assert(C->unique() == unique, "verification mode made Nodes? ? ?")do { if (!(C->unique() == unique)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4263, "assert(" "C->unique() == unique" ") failed", "verification mode made Nodes? ? ?" ); ::breakpoint(); } } while (0); |
| 4264 | assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything")do { if (!(_igvn._worklist.size() == orig_worklist_size)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4264, "assert(" "_igvn._worklist.size() == orig_worklist_size" ") failed", "shouldn't push anything"); ::breakpoint(); } } while (0); |
| 4265 | return; |
| 4266 | } |
| 4267 | |
| 4268 | // clear out the dead code after build_loop_late |
| 4269 | while (_deadlist.size()) { |
| 4270 | _igvn.remove_globally_dead_node(_deadlist.pop()); |
| 4271 | } |
| 4272 | |
| 4273 | if (stop_early) { |
| 4274 | assert(do_expensive_nodes, "why are we here?")do { if (!(do_expensive_nodes)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4274, "assert(" "do_expensive_nodes" ") failed", "why are we here?" ); ::breakpoint(); } } while (0); |
| 4275 | if (process_expensive_nodes()) { |
| 4276 | // If we made some progress when processing expensive nodes then |
| 4277 | // the IGVN may modify the graph in a way that will allow us to |
| 4278 | // make some more progress: we need to try processing expensive |
| 4279 | // nodes again. |
| 4280 | C->set_major_progress(); |
| 4281 | } |
| 4282 | return; |
| 4283 | } |
| 4284 | |
| 4285 | // Some parser-inserted loop predicates could never be used by loop |
| 4286 | // predication or they were moved away from loop during some optimizations. |
| 4287 | // For example, peeling. Eliminate them before next loop optimizations. |
| 4288 | eliminate_useless_predicates(); |
| 4289 | |
| 4290 | #ifndef PRODUCT |
| 4291 | C->verify_graph_edges(); |
| 4292 | if (_verify_me) { // Nested verify pass? |
| 4293 | // Check to see if the verify mode is broken |
| 4294 | assert(C->unique() == unique, "non-optimize mode made Nodes? ? ?")do { if (!(C->unique() == unique)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4294, "assert(" "C->unique() == unique" ") failed", "non-optimize mode made Nodes? ? ?" ); ::breakpoint(); } } while (0); |
| 4295 | return; |
| 4296 | } |
| 4297 | if (VerifyLoopOptimizations) verify(); |
| 4298 | if (TraceLoopOpts && C->has_loops()) { |
| 4299 | _ltree_root->dump(); |
| 4300 | } |
| 4301 | #endif |
| 4302 | |
| 4303 | if (skip_loop_opts) { |
| 4304 | C->restore_major_progress(old_progress); |
| 4305 | return; |
| 4306 | } |
| 4307 | |
| 4308 | if (mode == LoopOptsMaxUnroll) { |
| 4309 | for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { |
| 4310 | IdealLoopTree* lpt = iter.current(); |
| 4311 | if (lpt->is_innermost() && lpt->_allow_optimizations && !lpt->_has_call && lpt->is_counted()) { |
| 4312 | lpt->compute_trip_count(this); |
| 4313 | if (!lpt->do_one_iteration_loop(this) && |
| 4314 | !lpt->do_remove_empty_loop(this)) { |
| 4315 | AutoNodeBudget node_budget(this); |
| 4316 | if (lpt->_head->as_CountedLoop()->is_normal_loop() && |
| 4317 | lpt->policy_maximally_unroll(this)) { |
| 4318 | memset( worklist.adr(), 0, worklist.Size()*sizeof(Node*) ); |
| 4319 | do_maximally_unroll(lpt, worklist); |
| 4320 | } |
| 4321 | } |
| 4322 | } |
| 4323 | } |
| 4324 | |
| 4325 | C->restore_major_progress(old_progress); |
| 4326 | return; |
| 4327 | } |
| 4328 | |
| 4329 | if (bs->optimize_loops(this, mode, visited, nstack, worklist)) { |
| 4330 | return; |
| 4331 | } |
| 4332 | |
| 4333 | if (ReassociateInvariants && !C->major_progress()) { |
| 4334 | // Reassociate invariants and prep for split_thru_phi |
| 4335 | for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { |
| 4336 | IdealLoopTree* lpt = iter.current(); |
| 4337 | if (!lpt->is_loop()) { |
| 4338 | continue; |
| 4339 | } |
| 4340 | Node* head = lpt->_head; |
| 4341 | if (!head->is_BaseCountedLoop() || !lpt->is_innermost()) continue; |
| 4342 | |
| 4343 | // check for vectorized loops, any reassociation of invariants was already done |
| 4344 | if (head->is_CountedLoop()) { |
| 4345 | if (head->as_CountedLoop()->is_unroll_only()) { |
| 4346 | continue; |
| 4347 | } else { |
| 4348 | AutoNodeBudget node_budget(this); |
| 4349 | lpt->reassociate_invariants(this); |
| 4350 | } |
| 4351 | } |
| 4352 | // Because RCE opportunities can be masked by split_thru_phi, |
| 4353 | // look for RCE candidates and inhibit split_thru_phi |
| 4354 | // on just their loop-phi's for this pass of loop opts |
| 4355 | if (SplitIfBlocks && do_split_ifs && |
| 4356 | (lpt->policy_range_check(this, true, T_LONG) || |
| 4357 | (head->is_CountedLoop() && lpt->policy_range_check(this, true, T_INT)))) { |
| 4358 | lpt->_rce_candidate = 1; // = true |
| 4359 | } |
| 4360 | } |
| 4361 | } |
| 4362 | |
| 4363 | // Check for aggressive application of split-if and other transforms |
| 4364 | // that require basic-block info (like cloning through Phi's) |
| 4365 | if (!C->major_progress() && SplitIfBlocks && do_split_ifs) { |
| 4366 | visited.clear(); |
| 4367 | split_if_with_blocks( visited, nstack); |
| 4368 | NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); )if( VerifyLoopOptimizations ) verify();; |
| 4369 | } |
| 4370 | |
| 4371 | if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) { |
| 4372 | C->set_major_progress(); |
| 4373 | } |
| 4374 | |
| 4375 | // Perform loop predication before iteration splitting |
| 4376 | if (C->has_loops() && !C->major_progress() && (C->predicate_count() > 0)) { |
| 4377 | _ltree_root->_child->loop_predication(this); |
| 4378 | } |
| 4379 | |
| 4380 | if (OptimizeFill && UseLoopPredicate && C->has_loops() && !C->major_progress()) { |
| 4381 | if (do_intrinsify_fill()) { |
| 4382 | C->set_major_progress(); |
| 4383 | } |
| 4384 | } |
| 4385 | |
| 4386 | // Perform iteration-splitting on inner loops. Split iterations to avoid |
| 4387 | // range checks or one-shot null checks. |
| 4388 | |
| 4389 | // If split-if's didn't hack the graph too bad (no CFG changes) |
| 4390 | // then do loop opts. |
| 4391 | if (C->has_loops() && !C->major_progress()) { |
| 4392 | memset( worklist.adr(), 0, worklist.Size()*sizeof(Node*) ); |
| 4393 | _ltree_root->_child->iteration_split( this, worklist ); |
| 4394 | // No verify after peeling! GCM has hoisted code out of the loop. |
| 4395 | // After peeling, the hoisted code could sink inside the peeled area. |
| 4396 | // The peeling code does not try to recompute the best location for |
| 4397 | // all the code before the peeled area, so the verify pass will always |
| 4398 | // complain about it. |
| 4399 | } |
| 4400 | |
| 4401 | // Check for bailout, and return |
| 4402 | if (C->failing()) { |
| 4403 | return; |
| 4404 | } |
| 4405 | |
| 4406 | // Do verify graph edges in any case |
| 4407 | NOT_PRODUCT( C->verify_graph_edges(); )C->verify_graph_edges();; |
| 4408 | |
| 4409 | if (!do_split_ifs) { |
| 4410 | // We saw major progress in Split-If to get here. We forced a |
| 4411 | // pass with unrolling and not split-if, however more split-if's |
| 4412 | // might make progress. If the unrolling didn't make progress |
| 4413 | // then the major-progress flag got cleared and we won't try |
| 4414 | // another round of Split-If. In particular the ever-common |
| 4415 | // instance-of/check-cast pattern requires at least 2 rounds of |
| 4416 | // Split-If to clear out. |
| 4417 | C->set_major_progress(); |
| 4418 | } |
| 4419 | |
| 4420 | // Repeat loop optimizations if new loops were seen |
| 4421 | if (created_loop_node()) { |
| 4422 | C->set_major_progress(); |
| 4423 | } |
| 4424 | |
| 4425 | // Keep loop predicates and perform optimizations with them |
| 4426 | // until no more loop optimizations could be done. |
| 4427 | // After that switch predicates off and do more loop optimizations. |
| 4428 | if (!C->major_progress() && (C->predicate_count() > 0)) { |
| 4429 | C->cleanup_loop_predicates(_igvn); |
| 4430 | if (TraceLoopOpts) { |
| 4431 | tty->print_cr("PredicatesOff"); |
| 4432 | } |
| 4433 | C->set_major_progress(); |
| 4434 | } |
| 4435 | |
| 4436 | // Convert scalar to superword operations at the end of all loop opts. |
| 4437 | if (UseSuperWord && C->has_loops() && !C->major_progress()) { |
| 4438 | // SuperWord transform |
| 4439 | SuperWord sw(this); |
| 4440 | for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { |
| 4441 | IdealLoopTree* lpt = iter.current(); |
| 4442 | if (lpt->is_counted()) { |
| 4443 | CountedLoopNode *cl = lpt->_head->as_CountedLoop(); |
| 4444 | |
| 4445 | if (PostLoopMultiversioning && cl->is_rce_post_loop() && !cl->is_vectorized_loop()) { |
| 4446 | // Check that the rce'd post loop is encountered first, multiversion after all |
| 4447 | // major main loop optimization are concluded |
| 4448 | if (!C->major_progress()) { |
| 4449 | IdealLoopTree *lpt_next = lpt->_next; |
| 4450 | if (lpt_next && lpt_next->is_counted()) { |
| 4451 | CountedLoopNode *cl = lpt_next->_head->as_CountedLoop(); |
| 4452 | has_range_checks(lpt_next); |
| 4453 | if (cl->is_post_loop() && cl->range_checks_present()) { |
| 4454 | if (!cl->is_multiversioned()) { |
| 4455 | if (multi_version_post_loops(lpt, lpt_next) == false) { |
| 4456 | // Cause the rce loop to be optimized away if we fail |
| 4457 | cl->mark_is_multiversioned(); |
| 4458 | cl->set_slp_max_unroll(0); |
| 4459 | poison_rce_post_loop(lpt); |
| 4460 | } |
| 4461 | } |
| 4462 | } |
| 4463 | } |
| 4464 | sw.transform_loop(lpt, true); |
| 4465 | } |
| 4466 | } else if (cl->is_main_loop()) { |
| 4467 | sw.transform_loop(lpt, true); |
| 4468 | } |
| 4469 | } |
| 4470 | } |
| 4471 | } |
| 4472 | |
| 4473 | // disable assert until issue with split_flow_path is resolved (6742111) |
| 4474 | // assert(!_has_irreducible_loops || C->parsed_irreducible_loop() || C->is_osr_compilation(), |
| 4475 | // "shouldn't introduce irreducible loops"); |
| 4476 | } |
| 4477 | |
| 4478 | #ifndef PRODUCT |
| 4479 | //------------------------------print_statistics------------------------------- |
| 4480 | int PhaseIdealLoop::_loop_invokes=0;// Count of PhaseIdealLoop invokes |
| 4481 | int PhaseIdealLoop::_loop_work=0; // Sum of PhaseIdealLoop x unique |
| 4482 | volatile int PhaseIdealLoop::_long_loop_candidates=0; // Number of long loops seen |
| 4483 | volatile int PhaseIdealLoop::_long_loop_nests=0; // Number of long loops successfully transformed to a nest |
| 4484 | volatile int PhaseIdealLoop::_long_loop_counted_loops=0; // Number of long loops successfully transformed to a counted loop |
| 4485 | void PhaseIdealLoop::print_statistics() { |
| 4486 | tty->print_cr("PhaseIdealLoop=%d, sum _unique=%d, long loops=%d/%d/%d", _loop_invokes, _loop_work, _long_loop_counted_loops, _long_loop_nests, _long_loop_candidates); |
| 4487 | } |
| 4488 | |
| 4489 | //------------------------------verify----------------------------------------- |
| 4490 | // Build a verify-only PhaseIdealLoop, and see that it agrees with me. |
| 4491 | static int fail; // debug only, so its multi-thread dont care |
| 4492 | void PhaseIdealLoop::verify() const { |
| 4493 | int old_progress = C->major_progress(); |
| 4494 | ResourceMark rm; |
| 4495 | PhaseIdealLoop loop_verify(_igvn, this); |
| 4496 | VectorSet visited; |
| 4497 | |
| 4498 | fail = 0; |
| 4499 | verify_compare(C->root(), &loop_verify, visited); |
| 4500 | assert(fail == 0, "verify loops failed")do { if (!(fail == 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4500, "assert(" "fail == 0" ") failed", "verify loops failed" ); ::breakpoint(); } } while (0); |
| 4501 | // Verify loop structure is the same |
| 4502 | _ltree_root->verify_tree(loop_verify._ltree_root, NULL__null); |
| 4503 | // Reset major-progress. It was cleared by creating a verify version of |
| 4504 | // PhaseIdealLoop. |
| 4505 | C->restore_major_progress(old_progress); |
| 4506 | } |
| 4507 | |
| 4508 | //------------------------------verify_compare--------------------------------- |
| 4509 | // Make sure me and the given PhaseIdealLoop agree on key data structures |
| 4510 | void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const { |
| 4511 | if( !n ) return; |
| 4512 | if( visited.test_set( n->_idx ) ) return; |
| 4513 | if( !_nodes[n->_idx] ) { // Unreachable |
| 4514 | assert( !loop_verify->_nodes[n->_idx], "both should be unreachable" )do { if (!(!loop_verify->_nodes[n->_idx])) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4514, "assert(" "!loop_verify->_nodes[n->_idx]" ") failed" , "both should be unreachable"); ::breakpoint(); } } while (0 ); |
| 4515 | return; |
| 4516 | } |
| 4517 | |
| 4518 | uint i; |
| 4519 | for( i = 0; i < n->req(); i++ ) |
| 4520 | verify_compare( n->in(i), loop_verify, visited ); |
| 4521 | |
| 4522 | // Check the '_nodes' block/loop structure |
| 4523 | i = n->_idx; |
| 4524 | if( has_ctrl(n) ) { // We have control; verify has loop or ctrl |
| 4525 | if( _nodes[i] != loop_verify->_nodes[i] && |
| 4526 | get_ctrl_no_update(n) != loop_verify->get_ctrl_no_update(n) ) { |
| 4527 | tty->print("Mismatched control setting for: "); |
| 4528 | n->dump(); |
| 4529 | if( fail++ > 10 ) return; |
| 4530 | Node *c = get_ctrl_no_update(n); |
| 4531 | tty->print("We have it as: "); |
| 4532 | if( c->in(0) ) c->dump(); |
| 4533 | else tty->print_cr("N%d",c->_idx); |
| 4534 | tty->print("Verify thinks: "); |
| 4535 | if( loop_verify->has_ctrl(n) ) |
| 4536 | loop_verify->get_ctrl_no_update(n)->dump(); |
| 4537 | else |
| 4538 | loop_verify->get_loop_idx(n)->dump(); |
| 4539 | tty->cr(); |
| 4540 | } |
| 4541 | } else { // We have a loop |
| 4542 | IdealLoopTree *us = get_loop_idx(n); |
| 4543 | if( loop_verify->has_ctrl(n) ) { |
| 4544 | tty->print("Mismatched loop setting for: "); |
| 4545 | n->dump(); |
| 4546 | if( fail++ > 10 ) return; |
| 4547 | tty->print("We have it as: "); |
| 4548 | us->dump(); |
| 4549 | tty->print("Verify thinks: "); |
| 4550 | loop_verify->get_ctrl_no_update(n)->dump(); |
| 4551 | tty->cr(); |
| 4552 | } else if (!C->major_progress()) { |
| 4553 | // Loop selection can be messed up if we did a major progress |
| 4554 | // operation, like split-if. Do not verify in that case. |
| 4555 | IdealLoopTree *them = loop_verify->get_loop_idx(n); |
| 4556 | if( us->_head != them->_head || us->_tail != them->_tail ) { |
| 4557 | tty->print("Unequals loops for: "); |
| 4558 | n->dump(); |
| 4559 | if( fail++ > 10 ) return; |
| 4560 | tty->print("We have it as: "); |
| 4561 | us->dump(); |
| 4562 | tty->print("Verify thinks: "); |
| 4563 | them->dump(); |
| 4564 | tty->cr(); |
| 4565 | } |
| 4566 | } |
| 4567 | } |
| 4568 | |
| 4569 | // Check for immediate dominators being equal |
| 4570 | if( i >= _idom_size ) { |
| 4571 | if( !n->is_CFG() ) return; |
| 4572 | tty->print("CFG Node with no idom: "); |
| 4573 | n->dump(); |
| 4574 | return; |
| 4575 | } |
| 4576 | if( !n->is_CFG() ) return; |
| 4577 | if( n == C->root() ) return; // No IDOM here |
| 4578 | |
| 4579 | assert(n->_idx == i, "sanity")do { if (!(n->_idx == i)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4579, "assert(" "n->_idx == i" ") failed", "sanity"); :: breakpoint(); } } while (0); |
| 4580 | Node *id = idom_no_update(n); |
| 4581 | if( id != loop_verify->idom_no_update(n) ) { |
| 4582 | tty->print("Unequals idoms for: "); |
| 4583 | n->dump(); |
| 4584 | if( fail++ > 10 ) return; |
| 4585 | tty->print("We have it as: "); |
| 4586 | id->dump(); |
| 4587 | tty->print("Verify thinks: "); |
| 4588 | loop_verify->idom_no_update(n)->dump(); |
| 4589 | tty->cr(); |
| 4590 | } |
| 4591 | |
| 4592 | } |
| 4593 | |
| 4594 | //------------------------------verify_tree------------------------------------ |
| 4595 | // Verify that tree structures match. Because the CFG can change, siblings |
| 4596 | // within the loop tree can be reordered. We attempt to deal with that by |
| 4597 | // reordering the verify's loop tree if possible. |
| 4598 | void IdealLoopTree::verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const { |
| 4599 | assert( _parent == parent, "Badly formed loop tree" )do { if (!(_parent == parent)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4599, "assert(" "_parent == parent" ") failed", "Badly formed loop tree" ); ::breakpoint(); } } while (0); |
| 4600 | |
| 4601 | // Siblings not in same order? Attempt to re-order. |
| 4602 | if( _head != loop->_head ) { |
| 4603 | // Find _next pointer to update |
| 4604 | IdealLoopTree **pp = &loop->_parent->_child; |
| 4605 | while( *pp != loop ) |
| 4606 | pp = &((*pp)->_next); |
| 4607 | // Find proper sibling to be next |
| 4608 | IdealLoopTree **nn = &loop->_next; |
| 4609 | while( (*nn) && (*nn)->_head != _head ) |
| 4610 | nn = &((*nn)->_next); |
| 4611 | |
| 4612 | // Check for no match. |
| 4613 | if( !(*nn) ) { |
| 4614 | // Annoyingly, irreducible loops can pick different headers |
| 4615 | // after a major_progress operation, so the rest of the loop |
| 4616 | // tree cannot be matched. |
| 4617 | if (_irreducible && Compile::current()->major_progress()) return; |
| 4618 | assert( 0, "failed to match loop tree" )do { if (!(0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4618, "assert(" "0" ") failed", "failed to match loop tree" ); ::breakpoint(); } } while (0); |
| 4619 | } |
| 4620 | |
| 4621 | // Move (*nn) to (*pp) |
| 4622 | IdealLoopTree *hit = *nn; |
| 4623 | *nn = hit->_next; |
| 4624 | hit->_next = loop; |
| 4625 | *pp = loop; |
| 4626 | loop = hit; |
| 4627 | // Now try again to verify |
| 4628 | } |
| 4629 | |
| 4630 | assert( _head == loop->_head , "mismatched loop head" )do { if (!(_head == loop->_head)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4630, "assert(" "_head == loop->_head" ") failed", "mismatched loop head" ); ::breakpoint(); } } while (0); |
| 4631 | Node *tail = _tail; // Inline a non-updating version of |
| 4632 | while( !tail->in(0) ) // the 'tail()' call. |
| 4633 | tail = tail->in(1); |
| 4634 | assert( tail == loop->_tail, "mismatched loop tail" )do { if (!(tail == loop->_tail)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4634, "assert(" "tail == loop->_tail" ") failed", "mismatched loop tail" ); ::breakpoint(); } } while (0); |
| 4635 | |
| 4636 | // Counted loops that are guarded should be able to find their guards |
| 4637 | if( _head->is_CountedLoop() && _head->as_CountedLoop()->is_main_loop() ) { |
| 4638 | CountedLoopNode *cl = _head->as_CountedLoop(); |
| 4639 | Node *init = cl->init_trip(); |
| 4640 | Node *ctrl = cl->in(LoopNode::EntryControl); |
| 4641 | assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" )do { if (!(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode( ) == Op_IfFalse)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4641, "assert(" "ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse" ") failed", ""); ::breakpoint(); } } while (0); |
| 4642 | Node *iff = ctrl->in(0); |
| 4643 | assert( iff->Opcode() == Op_If, "" )do { if (!(iff->Opcode() == Op_If)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4643, "assert(" "iff->Opcode() == Op_If" ") failed", "") ; ::breakpoint(); } } while (0); |
| 4644 | Node *bol = iff->in(1); |
| 4645 | assert( bol->Opcode() == Op_Bool, "" )do { if (!(bol->Opcode() == Op_Bool)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4645, "assert(" "bol->Opcode() == Op_Bool" ") failed", "" ); ::breakpoint(); } } while (0); |
| 4646 | Node *cmp = bol->in(1); |
| 4647 | assert( cmp->Opcode() == Op_CmpI, "" )do { if (!(cmp->Opcode() == Op_CmpI)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4647, "assert(" "cmp->Opcode() == Op_CmpI" ") failed", "" ); ::breakpoint(); } } while (0); |
| 4648 | Node *add = cmp->in(1); |
| 4649 | Node *opaq; |
| 4650 | if( add->Opcode() == Op_Opaque1 ) { |
| 4651 | opaq = add; |
| 4652 | } else { |
| 4653 | assert( add->Opcode() == Op_AddI || add->Opcode() == Op_ConI , "" )do { if (!(add->Opcode() == Op_AddI || add->Opcode() == Op_ConI)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4653, "assert(" "add->Opcode() == Op_AddI || add->Opcode() == Op_ConI" ") failed", ""); ::breakpoint(); } } while (0); |
| 4654 | assert( add == init, "" )do { if (!(add == init)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4654, "assert(" "add == init" ") failed", ""); ::breakpoint (); } } while (0); |
| 4655 | opaq = cmp->in(2); |
| 4656 | } |
| 4657 | assert( opaq->Opcode() == Op_Opaque1, "" )do { if (!(opaq->Opcode() == Op_Opaque1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4657, "assert(" "opaq->Opcode() == Op_Opaque1" ") failed" , ""); ::breakpoint(); } } while (0); |
| 4658 | |
| 4659 | } |
| 4660 | |
| 4661 | if (_child != NULL__null) _child->verify_tree(loop->_child, this); |
| 4662 | if (_next != NULL__null) _next ->verify_tree(loop->_next, parent); |
| 4663 | // Innermost loops need to verify loop bodies, |
| 4664 | // but only if no 'major_progress' |
| 4665 | int fail = 0; |
| 4666 | if (!Compile::current()->major_progress() && _child == NULL__null) { |
| 4667 | for( uint i = 0; i < _body.size(); i++ ) { |
| 4668 | Node *n = _body.at(i); |
| 4669 | if (n->outcnt() == 0) continue; // Ignore dead |
| 4670 | uint j; |
| 4671 | for( j = 0; j < loop->_body.size(); j++ ) |
| 4672 | if( loop->_body.at(j) == n ) |
| 4673 | break; |
| 4674 | if( j == loop->_body.size() ) { // Not found in loop body |
| 4675 | // Last ditch effort to avoid assertion: Its possible that we |
| 4676 | // have some users (so outcnt not zero) but are still dead. |
| 4677 | // Try to find from root. |
| 4678 | if (Compile::current()->root()->find(n->_idx)) { |
| 4679 | fail++; |
| 4680 | tty->print("We have that verify does not: "); |
| 4681 | n->dump(); |
| 4682 | } |
| 4683 | } |
| 4684 | } |
| 4685 | for( uint i2 = 0; i2 < loop->_body.size(); i2++ ) { |
| 4686 | Node *n = loop->_body.at(i2); |
| 4687 | if (n->outcnt() == 0) continue; // Ignore dead |
| 4688 | uint j; |
| 4689 | for( j = 0; j < _body.size(); j++ ) |
| 4690 | if( _body.at(j) == n ) |
| 4691 | break; |
| 4692 | if( j == _body.size() ) { // Not found in loop body |
| 4693 | // Last ditch effort to avoid assertion: Its possible that we |
| 4694 | // have some users (so outcnt not zero) but are still dead. |
| 4695 | // Try to find from root. |
| 4696 | if (Compile::current()->root()->find(n->_idx)) { |
| 4697 | fail++; |
| 4698 | tty->print("Verify has that we do not: "); |
| 4699 | n->dump(); |
| 4700 | } |
| 4701 | } |
| 4702 | } |
| 4703 | assert( !fail, "loop body mismatch" )do { if (!(!fail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4703, "assert(" "!fail" ") failed", "loop body mismatch"); :: breakpoint(); } } while (0); |
| 4704 | } |
| 4705 | } |
| 4706 | |
| 4707 | #endif |
| 4708 | |
| 4709 | //------------------------------set_idom--------------------------------------- |
| 4710 | void PhaseIdealLoop::set_idom(Node* d, Node* n, uint dom_depth) { |
| 4711 | uint idx = d->_idx; |
| 4712 | if (idx >= _idom_size) { |
| 4713 | uint newsize = next_power_of_2(idx); |
| 4714 | _idom = REALLOC_RESOURCE_ARRAY( Node*, _idom,_idom_size,newsize)(Node**) resource_reallocate_bytes((char*)(_idom), (_idom_size ) * sizeof(Node*), (newsize) * sizeof(Node*)); |
| 4715 | _dom_depth = REALLOC_RESOURCE_ARRAY( uint, _dom_depth,_idom_size,newsize)(uint*) resource_reallocate_bytes((char*)(_dom_depth), (_idom_size ) * sizeof(uint), (newsize) * sizeof(uint)); |
| 4716 | memset( _dom_depth + _idom_size, 0, (newsize - _idom_size) * sizeof(uint) ); |
| 4717 | _idom_size = newsize; |
| 4718 | } |
| 4719 | _idom[idx] = n; |
| 4720 | _dom_depth[idx] = dom_depth; |
| 4721 | } |
| 4722 | |
| 4723 | //------------------------------recompute_dom_depth--------------------------------------- |
| 4724 | // The dominator tree is constructed with only parent pointers. |
| 4725 | // This recomputes the depth in the tree by first tagging all |
| 4726 | // nodes as "no depth yet" marker. The next pass then runs up |
| 4727 | // the dom tree from each node marked "no depth yet", and computes |
| 4728 | // the depth on the way back down. |
| 4729 | void PhaseIdealLoop::recompute_dom_depth() { |
| 4730 | uint no_depth_marker = C->unique(); |
| 4731 | uint i; |
| 4732 | // Initialize depth to "no depth yet" and realize all lazy updates |
| 4733 | for (i = 0; i < _idom_size; i++) { |
| 4734 | // Only indices with a _dom_depth has a Node* or NULL (otherwise uninitalized). |
| 4735 | if (_dom_depth[i] > 0 && _idom[i] != NULL__null) { |
| 4736 | _dom_depth[i] = no_depth_marker; |
| 4737 | |
| 4738 | // heal _idom if it has a fwd mapping in _nodes |
| 4739 | if (_idom[i]->in(0) == NULL__null) { |
| 4740 | idom(i); |
| 4741 | } |
| 4742 | } |
| 4743 | } |
| 4744 | if (_dom_stk == NULL__null) { |
| 4745 | uint init_size = C->live_nodes() / 100; // Guess that 1/100 is a reasonable initial size. |
| 4746 | if (init_size < 10) init_size = 10; |
| 4747 | _dom_stk = new GrowableArray<uint>(init_size); |
| 4748 | } |
| 4749 | // Compute new depth for each node. |
| 4750 | for (i = 0; i < _idom_size; i++) { |
| 4751 | uint j = i; |
| 4752 | // Run up the dom tree to find a node with a depth |
| 4753 | while (_dom_depth[j] == no_depth_marker) { |
| 4754 | _dom_stk->push(j); |
| 4755 | j = _idom[j]->_idx; |
| 4756 | } |
| 4757 | // Compute the depth on the way back down this tree branch |
| 4758 | uint dd = _dom_depth[j] + 1; |
| 4759 | while (_dom_stk->length() > 0) { |
| 4760 | uint j = _dom_stk->pop(); |
| 4761 | _dom_depth[j] = dd; |
| 4762 | dd++; |
| 4763 | } |
| 4764 | } |
| 4765 | } |
| 4766 | |
| 4767 | //------------------------------sort------------------------------------------- |
| 4768 | // Insert 'loop' into the existing loop tree. 'innermost' is a leaf of the |
| 4769 | // loop tree, not the root. |
| 4770 | IdealLoopTree *PhaseIdealLoop::sort( IdealLoopTree *loop, IdealLoopTree *innermost ) { |
| 4771 | if( !innermost ) return loop; // New innermost loop |
| 4772 | |
| 4773 | int loop_preorder = get_preorder(loop->_head); // Cache pre-order number |
| 4774 | assert( loop_preorder, "not yet post-walked loop" )do { if (!(loop_preorder)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4774, "assert(" "loop_preorder" ") failed", "not yet post-walked loop" ); ::breakpoint(); } } while (0); |
| 4775 | IdealLoopTree **pp = &innermost; // Pointer to previous next-pointer |
| 4776 | IdealLoopTree *l = *pp; // Do I go before or after 'l'? |
| 4777 | |
| 4778 | // Insert at start of list |
| 4779 | while( l ) { // Insertion sort based on pre-order |
| 4780 | if( l == loop ) return innermost; // Already on list! |
| 4781 | int l_preorder = get_preorder(l->_head); // Cache pre-order number |
| 4782 | assert( l_preorder, "not yet post-walked l" )do { if (!(l_preorder)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4782, "assert(" "l_preorder" ") failed", "not yet post-walked l" ); ::breakpoint(); } } while (0); |
| 4783 | // Check header pre-order number to figure proper nesting |
| 4784 | if( loop_preorder > l_preorder ) |
| 4785 | break; // End of insertion |
| 4786 | // If headers tie (e.g., shared headers) check tail pre-order numbers. |
| 4787 | // Since I split shared headers, you'd think this could not happen. |
| 4788 | // BUT: I must first do the preorder numbering before I can discover I |
| 4789 | // have shared headers, so the split headers all get the same preorder |
| 4790 | // number as the RegionNode they split from. |
| 4791 | if( loop_preorder == l_preorder && |
| 4792 | get_preorder(loop->_tail) < get_preorder(l->_tail) ) |
| 4793 | break; // Also check for shared headers (same pre#) |
| 4794 | pp = &l->_parent; // Chain up list |
| 4795 | l = *pp; |
| 4796 | } |
| 4797 | // Link into list |
| 4798 | // Point predecessor to me |
| 4799 | *pp = loop; |
| 4800 | // Point me to successor |
| 4801 | IdealLoopTree *p = loop->_parent; |
| 4802 | loop->_parent = l; // Point me to successor |
| 4803 | if( p ) sort( p, innermost ); // Insert my parents into list as well |
| 4804 | return innermost; |
| 4805 | } |
| 4806 | |
| 4807 | //------------------------------build_loop_tree-------------------------------- |
| 4808 | // I use a modified Vick/Tarjan algorithm. I need pre- and a post- visit |
| 4809 | // bits. The _nodes[] array is mapped by Node index and holds a NULL for |
| 4810 | // not-yet-pre-walked, pre-order # for pre-but-not-post-walked and holds the |
| 4811 | // tightest enclosing IdealLoopTree for post-walked. |
| 4812 | // |
| 4813 | // During my forward walk I do a short 1-layer lookahead to see if I can find |
| 4814 | // a loop backedge with that doesn't have any work on the backedge. This |
| 4815 | // helps me construct nested loops with shared headers better. |
| 4816 | // |
| 4817 | // Once I've done the forward recursion, I do the post-work. For each child |
| 4818 | // I check to see if there is a backedge. Backedges define a loop! I |
| 4819 | // insert an IdealLoopTree at the target of the backedge. |
| 4820 | // |
| 4821 | // During the post-work I also check to see if I have several children |
| 4822 | // belonging to different loops. If so, then this Node is a decision point |
| 4823 | // where control flow can choose to change loop nests. It is at this |
| 4824 | // decision point where I can figure out how loops are nested. At this |
| 4825 | // time I can properly order the different loop nests from my children. |
| 4826 | // Note that there may not be any backedges at the decision point! |
| 4827 | // |
| 4828 | // Since the decision point can be far removed from the backedges, I can't |
| 4829 | // order my loops at the time I discover them. Thus at the decision point |
| 4830 | // I need to inspect loop header pre-order numbers to properly nest my |
| 4831 | // loops. This means I need to sort my childrens' loops by pre-order. |
| 4832 | // The sort is of size number-of-control-children, which generally limits |
| 4833 | // it to size 2 (i.e., I just choose between my 2 target loops). |
| 4834 | void PhaseIdealLoop::build_loop_tree() { |
| 4835 | // Allocate stack of size C->live_nodes()/2 to avoid frequent realloc |
| 4836 | GrowableArray <Node *> bltstack(C->live_nodes() >> 1); |
| 4837 | Node *n = C->root(); |
| 4838 | bltstack.push(n); |
| 4839 | int pre_order = 1; |
| 4840 | int stack_size; |
| 4841 | |
| 4842 | while ( ( stack_size = bltstack.length() ) != 0 ) { |
| 4843 | n = bltstack.top(); // Leave node on stack |
| 4844 | if ( !is_visited(n) ) { |
| 4845 | // ---- Pre-pass Work ---- |
| 4846 | // Pre-walked but not post-walked nodes need a pre_order number. |
| 4847 | |
| 4848 | set_preorder_visited( n, pre_order ); // set as visited |
| 4849 | |
| 4850 | // ---- Scan over children ---- |
| 4851 | // Scan first over control projections that lead to loop headers. |
| 4852 | // This helps us find inner-to-outer loops with shared headers better. |
| 4853 | |
| 4854 | // Scan children's children for loop headers. |
| 4855 | for ( int i = n->outcnt() - 1; i >= 0; --i ) { |
| 4856 | Node* m = n->raw_out(i); // Child |
| 4857 | if( m->is_CFG() && !is_visited(m) ) { // Only for CFG children |
| 4858 | // Scan over children's children to find loop |
| 4859 | for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { |
| 4860 | Node* l = m->fast_out(j); |
| 4861 | if( is_visited(l) && // Been visited? |
| 4862 | !is_postvisited(l) && // But not post-visited |
| 4863 | get_preorder(l) < pre_order ) { // And smaller pre-order |
| 4864 | // Found! Scan the DFS down this path before doing other paths |
| 4865 | bltstack.push(m); |
| 4866 | break; |
| 4867 | } |
| 4868 | } |
| 4869 | } |
| 4870 | } |
| 4871 | pre_order++; |
| 4872 | } |
| 4873 | else if ( !is_postvisited(n) ) { |
| 4874 | // Note: build_loop_tree_impl() adds out edges on rare occasions, |
| 4875 | // such as com.sun.rsasign.am::a. |
| 4876 | // For non-recursive version, first, process current children. |
| 4877 | // On next iteration, check if additional children were added. |
| 4878 | for ( int k = n->outcnt() - 1; k >= 0; --k ) { |
| 4879 | Node* u = n->raw_out(k); |
| 4880 | if ( u->is_CFG() && !is_visited(u) ) { |
| 4881 | bltstack.push(u); |
| 4882 | } |
| 4883 | } |
| 4884 | if ( bltstack.length() == stack_size ) { |
| 4885 | // There were no additional children, post visit node now |
| 4886 | (void)bltstack.pop(); // Remove node from stack |
| 4887 | pre_order = build_loop_tree_impl( n, pre_order ); |
| 4888 | // Check for bailout |
| 4889 | if (C->failing()) { |
| 4890 | return; |
| 4891 | } |
| 4892 | // Check to grow _preorders[] array for the case when |
| 4893 | // build_loop_tree_impl() adds new nodes. |
| 4894 | check_grow_preorders(); |
| 4895 | } |
| 4896 | } |
| 4897 | else { |
| 4898 | (void)bltstack.pop(); // Remove post-visited node from stack |
| 4899 | } |
| 4900 | } |
| 4901 | } |
| 4902 | |
| 4903 | //------------------------------build_loop_tree_impl--------------------------- |
| 4904 | int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) { |
| 4905 | // ---- Post-pass Work ---- |
| 4906 | // Pre-walked but not post-walked nodes need a pre_order number. |
| 4907 | |
| 4908 | // Tightest enclosing loop for this Node |
| 4909 | IdealLoopTree *innermost = NULL__null; |
| 4910 | |
| 4911 | // For all children, see if any edge is a backedge. If so, make a loop |
| 4912 | // for it. Then find the tightest enclosing loop for the self Node. |
| 4913 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
| 4914 | Node* m = n->fast_out(i); // Child |
| 4915 | if( n == m ) continue; // Ignore control self-cycles |
| 4916 | if( !m->is_CFG() ) continue;// Ignore non-CFG edges |
| 4917 | |
| 4918 | IdealLoopTree *l; // Child's loop |
| 4919 | if( !is_postvisited(m) ) { // Child visited but not post-visited? |
| 4920 | // Found a backedge |
| 4921 | assert( get_preorder(m) < pre_order, "should be backedge" )do { if (!(get_preorder(m) < pre_order)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4921, "assert(" "get_preorder(m) < pre_order" ") failed" , "should be backedge"); ::breakpoint(); } } while (0); |
| 4922 | // Check for the RootNode, which is already a LoopNode and is allowed |
| 4923 | // to have multiple "backedges". |
| 4924 | if( m == C->root()) { // Found the root? |
| 4925 | l = _ltree_root; // Root is the outermost LoopNode |
| 4926 | } else { // Else found a nested loop |
| 4927 | // Insert a LoopNode to mark this loop. |
| 4928 | l = new IdealLoopTree(this, m, n); |
| 4929 | } // End of Else found a nested loop |
| 4930 | if( !has_loop(m) ) // If 'm' does not already have a loop set |
| 4931 | set_loop(m, l); // Set loop header to loop now |
| 4932 | |
| 4933 | } else { // Else not a nested loop |
| 4934 | if( !_nodes[m->_idx] ) continue; // Dead code has no loop |
| 4935 | l = get_loop(m); // Get previously determined loop |
| 4936 | // If successor is header of a loop (nest), move up-loop till it |
| 4937 | // is a member of some outer enclosing loop. Since there are no |
| 4938 | // shared headers (I've split them already) I only need to go up |
| 4939 | // at most 1 level. |
| 4940 | while( l && l->_head == m ) // Successor heads loop? |
| 4941 | l = l->_parent; // Move up 1 for me |
| 4942 | // If this loop is not properly parented, then this loop |
| 4943 | // has no exit path out, i.e. its an infinite loop. |
| 4944 | if( !l ) { |
| 4945 | // Make loop "reachable" from root so the CFG is reachable. Basically |
| 4946 | // insert a bogus loop exit that is never taken. 'm', the loop head, |
| 4947 | // points to 'n', one (of possibly many) fall-in paths. There may be |
| 4948 | // many backedges as well. |
| 4949 | |
| 4950 | // Here I set the loop to be the root loop. I could have, after |
| 4951 | // inserting a bogus loop exit, restarted the recursion and found my |
| 4952 | // new loop exit. This would make the infinite loop a first-class |
| 4953 | // loop and it would then get properly optimized. What's the use of |
| 4954 | // optimizing an infinite loop? |
| 4955 | l = _ltree_root; // Oops, found infinite loop |
| 4956 | |
| 4957 | if (!_verify_only) { |
| 4958 | // Insert the NeverBranch between 'm' and it's control user. |
| 4959 | NeverBranchNode *iff = new NeverBranchNode( m ); |
| 4960 | _igvn.register_new_node_with_optimizer(iff); |
| 4961 | set_loop(iff, l); |
| 4962 | Node *if_t = new CProjNode( iff, 0 ); |
| 4963 | _igvn.register_new_node_with_optimizer(if_t); |
| 4964 | set_loop(if_t, l); |
| 4965 | |
| 4966 | Node* cfg = NULL__null; // Find the One True Control User of m |
| 4967 | for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { |
| 4968 | Node* x = m->fast_out(j); |
| 4969 | if (x->is_CFG() && x != m && x != iff) |
| 4970 | { cfg = x; break; } |
| 4971 | } |
| 4972 | assert(cfg != NULL, "must find the control user of m")do { if (!(cfg != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4972, "assert(" "cfg != __null" ") failed", "must find the control user of m" ); ::breakpoint(); } } while (0); |
| 4973 | uint k = 0; // Probably cfg->in(0) |
| 4974 | while( cfg->in(k) != m ) k++; // But check incase cfg is a Region |
| 4975 | cfg->set_req( k, if_t ); // Now point to NeverBranch |
| 4976 | _igvn._worklist.push(cfg); |
| 4977 | |
| 4978 | // Now create the never-taken loop exit |
| 4979 | Node *if_f = new CProjNode( iff, 1 ); |
| 4980 | _igvn.register_new_node_with_optimizer(if_f); |
| 4981 | set_loop(if_f, l); |
| 4982 | // Find frame ptr for Halt. Relies on the optimizer |
| 4983 | // V-N'ing. Easier and quicker than searching through |
| 4984 | // the program structure. |
| 4985 | Node *frame = new ParmNode( C->start(), TypeFunc::FramePtr ); |
| 4986 | _igvn.register_new_node_with_optimizer(frame); |
| 4987 | // Halt & Catch Fire |
| 4988 | Node* halt = new HaltNode(if_f, frame, "never-taken loop exit reached"); |
| 4989 | _igvn.register_new_node_with_optimizer(halt); |
| 4990 | set_loop(halt, l); |
| 4991 | C->root()->add_req(halt); |
| 4992 | } |
| 4993 | set_loop(C->root(), _ltree_root); |
| 4994 | } |
| 4995 | } |
| 4996 | // Weeny check for irreducible. This child was already visited (this |
| 4997 | // IS the post-work phase). Is this child's loop header post-visited |
| 4998 | // as well? If so, then I found another entry into the loop. |
| 4999 | if (!_verify_only) { |
| 5000 | while( is_postvisited(l->_head) ) { |
| 5001 | // found irreducible |
| 5002 | l->_irreducible = 1; // = true |
| 5003 | l = l->_parent; |
| 5004 | _has_irreducible_loops = true; |
| 5005 | // Check for bad CFG here to prevent crash, and bailout of compile |
| 5006 | if (l == NULL__null) { |
| 5007 | C->record_method_not_compilable("unhandled CFG detected during loop optimization"); |
| 5008 | return pre_order; |
| 5009 | } |
| 5010 | } |
| 5011 | C->set_has_irreducible_loop(_has_irreducible_loops); |
| 5012 | } |
| 5013 | |
| 5014 | // This Node might be a decision point for loops. It is only if |
| 5015 | // it's children belong to several different loops. The sort call |
| 5016 | // does a trivial amount of work if there is only 1 child or all |
| 5017 | // children belong to the same loop. If however, the children |
| 5018 | // belong to different loops, the sort call will properly set the |
| 5019 | // _parent pointers to show how the loops nest. |
| 5020 | // |
| 5021 | // In any case, it returns the tightest enclosing loop. |
| 5022 | innermost = sort( l, innermost ); |
| 5023 | } |
| 5024 | |
| 5025 | // Def-use info will have some dead stuff; dead stuff will have no |
| 5026 | // loop decided on. |
| 5027 | |
| 5028 | // Am I a loop header? If so fix up my parent's child and next ptrs. |
| 5029 | if( innermost && innermost->_head == n ) { |
| 5030 | assert( get_loop(n) == innermost, "" )do { if (!(get_loop(n) == innermost)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5030, "assert(" "get_loop(n) == innermost" ") failed", ""); ::breakpoint(); } } while (0); |
| 5031 | IdealLoopTree *p = innermost->_parent; |
| 5032 | IdealLoopTree *l = innermost; |
| 5033 | while( p && l->_head == n ) { |
| 5034 | l->_next = p->_child; // Put self on parents 'next child' |
| 5035 | p->_child = l; // Make self as first child of parent |
| 5036 | l = p; // Now walk up the parent chain |
| 5037 | p = l->_parent; |
| 5038 | } |
| 5039 | } else { |
| 5040 | // Note that it is possible for a LoopNode to reach here, if the |
| 5041 | // backedge has been made unreachable (hence the LoopNode no longer |
| 5042 | // denotes a Loop, and will eventually be removed). |
| 5043 | |
| 5044 | // Record tightest enclosing loop for self. Mark as post-visited. |
| 5045 | set_loop(n, innermost); |
| 5046 | // Also record has_call flag early on |
| 5047 | if( innermost ) { |
| 5048 | if( n->is_Call() && !n->is_CallLeaf() && !n->is_macro() ) { |
| 5049 | // Do not count uncommon calls |
| 5050 | if( !n->is_CallStaticJava() || !n->as_CallStaticJava()->_name ) { |
| 5051 | Node *iff = n->in(0)->in(0); |
| 5052 | // No any calls for vectorized loops. |
| 5053 | if( UseSuperWord || !iff->is_If() || |
| 5054 | (n->in(0)->Opcode() == Op_IfFalse && |
| 5055 | (1.0 - iff->as_If()->_prob) >= 0.01) || |
| 5056 | (iff->as_If()->_prob >= 0.01) ) |
| 5057 | innermost->_has_call = 1; |
| 5058 | } |
| 5059 | } else if( n->is_Allocate() && n->as_Allocate()->_is_scalar_replaceable ) { |
| 5060 | // Disable loop optimizations if the loop has a scalar replaceable |
| 5061 | // allocation. This disabling may cause a potential performance lost |
| 5062 | // if the allocation is not eliminated for some reason. |
| 5063 | innermost->_allow_optimizations = false; |
| 5064 | innermost->_has_call = 1; // = true |
| 5065 | } else if (n->Opcode() == Op_SafePoint) { |
| 5066 | // Record all safepoints in this loop. |
| 5067 | if (innermost->_safepts == NULL__null) innermost->_safepts = new Node_List(); |
| 5068 | innermost->_safepts->push(n); |
| 5069 | } |
| 5070 | } |
| 5071 | } |
| 5072 | |
| 5073 | // Flag as post-visited now |
| 5074 | set_postvisited(n); |
| 5075 | return pre_order; |
| 5076 | } |
| 5077 | |
| 5078 | |
| 5079 | //------------------------------build_loop_early------------------------------- |
| 5080 | // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. |
| 5081 | // First pass computes the earliest controlling node possible. This is the |
| 5082 | // controlling input with the deepest dominating depth. |
| 5083 | void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { |
| 5084 | while (worklist.size() != 0) { |
| 5085 | // Use local variables nstack_top_n & nstack_top_i to cache values |
| 5086 | // on nstack's top. |
| 5087 | Node *nstack_top_n = worklist.pop(); |
| 5088 | uint nstack_top_i = 0; |
| 5089 | //while_nstack_nonempty: |
| 5090 | while (true) { |
| 5091 | // Get parent node and next input's index from stack's top. |
| 5092 | Node *n = nstack_top_n; |
| 5093 | uint i = nstack_top_i; |
| 5094 | uint cnt = n->req(); // Count of inputs |
| 5095 | if (i == 0) { // Pre-process the node. |
| 5096 | if( has_node(n) && // Have either loop or control already? |
| 5097 | !has_ctrl(n) ) { // Have loop picked out already? |
| 5098 | // During "merge_many_backedges" we fold up several nested loops |
| 5099 | // into a single loop. This makes the members of the original |
| 5100 | // loop bodies pointing to dead loops; they need to move up |
| 5101 | // to the new UNION'd larger loop. I set the _head field of these |
| 5102 | // dead loops to NULL and the _parent field points to the owning |
| 5103 | // loop. Shades of UNION-FIND algorithm. |
| 5104 | IdealLoopTree *ilt; |
| 5105 | while( !(ilt = get_loop(n))->_head ) { |
| 5106 | // Normally I would use a set_loop here. But in this one special |
| 5107 | // case, it is legal (and expected) to change what loop a Node |
| 5108 | // belongs to. |
| 5109 | _nodes.map(n->_idx, (Node*)(ilt->_parent) ); |
| 5110 | } |
| 5111 | // Remove safepoints ONLY if I've already seen I don't need one. |
| 5112 | // (the old code here would yank a 2nd safepoint after seeing a |
| 5113 | // first one, even though the 1st did not dominate in the loop body |
| 5114 | // and thus could be avoided indefinitely) |
| 5115 | if( !_verify_only && !_verify_me && ilt->_has_sfpt && n->Opcode() == Op_SafePoint && |
| 5116 | is_deleteable_safept(n)) { |
| 5117 | Node *in = n->in(TypeFunc::Control); |
| 5118 | lazy_replace(n,in); // Pull safepoint now |
| 5119 | if (ilt->_safepts != NULL__null) { |
| 5120 | ilt->_safepts->yank(n); |
| 5121 | } |
| 5122 | // Carry on with the recursion "as if" we are walking |
| 5123 | // only the control input |
| 5124 | if( !visited.test_set( in->_idx ) ) { |
| 5125 | worklist.push(in); // Visit this guy later, using worklist |
| 5126 | } |
| 5127 | // Get next node from nstack: |
| 5128 | // - skip n's inputs processing by setting i > cnt; |
| 5129 | // - we also will not call set_early_ctrl(n) since |
| 5130 | // has_node(n) == true (see the condition above). |
| 5131 | i = cnt + 1; |
| 5132 | } |
| 5133 | } |
| 5134 | } // if (i == 0) |
| 5135 | |
| 5136 | // Visit all inputs |
| 5137 | bool done = true; // Assume all n's inputs will be processed |
| 5138 | while (i < cnt) { |
| 5139 | Node *in = n->in(i); |
| 5140 | ++i; |
| 5141 | if (in == NULL__null) continue; |
| 5142 | if (in->pinned() && !in->is_CFG()) |
| 5143 | set_ctrl(in, in->in(0)); |
| 5144 | int is_visited = visited.test_set( in->_idx ); |
| 5145 | if (!has_node(in)) { // No controlling input yet? |
| 5146 | assert( !in->is_CFG(), "CFG Node with no controlling input?" )do { if (!(!in->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5146, "assert(" "!in->is_CFG()" ") failed", "CFG Node with no controlling input?" ); ::breakpoint(); } } while (0); |
| 5147 | assert( !is_visited, "visit only once" )do { if (!(!is_visited)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5147, "assert(" "!is_visited" ") failed", "visit only once" ); ::breakpoint(); } } while (0); |
| 5148 | nstack.push(n, i); // Save parent node and next input's index. |
| 5149 | nstack_top_n = in; // Process current input now. |
| 5150 | nstack_top_i = 0; |
| 5151 | done = false; // Not all n's inputs processed. |
| 5152 | break; // continue while_nstack_nonempty; |
| 5153 | } else if (!is_visited) { |
| 5154 | // This guy has a location picked out for him, but has not yet |
| 5155 | // been visited. Happens to all CFG nodes, for instance. |
| 5156 | // Visit him using the worklist instead of recursion, to break |
| 5157 | // cycles. Since he has a location already we do not need to |
| 5158 | // find his location before proceeding with the current Node. |
| 5159 | worklist.push(in); // Visit this guy later, using worklist |
| 5160 | } |
| 5161 | } |
| 5162 | if (done) { |
| 5163 | // All of n's inputs have been processed, complete post-processing. |
| 5164 | |
| 5165 | // Compute earliest point this Node can go. |
| 5166 | // CFG, Phi, pinned nodes already know their controlling input. |
| 5167 | if (!has_node(n)) { |
| 5168 | // Record earliest legal location |
| 5169 | set_early_ctrl(n, false); |
| 5170 | } |
| 5171 | if (nstack.is_empty()) { |
| 5172 | // Finished all nodes on stack. |
| 5173 | // Process next node on the worklist. |
| 5174 | break; |
| 5175 | } |
| 5176 | // Get saved parent node and next input's index. |
| 5177 | nstack_top_n = nstack.node(); |
| 5178 | nstack_top_i = nstack.index(); |
| 5179 | nstack.pop(); |
| 5180 | } |
| 5181 | } // while (true) |
| 5182 | } |
| 5183 | } |
| 5184 | |
| 5185 | //------------------------------dom_lca_internal-------------------------------- |
| 5186 | // Pair-wise LCA |
| 5187 | Node *PhaseIdealLoop::dom_lca_internal( Node *n1, Node *n2 ) const { |
| 5188 | if( !n1 ) return n2; // Handle NULL original LCA |
| 5189 | assert( n1->is_CFG(), "" )do { if (!(n1->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5189, "assert(" "n1->is_CFG()" ") failed", ""); ::breakpoint (); } } while (0); |
| 5190 | assert( n2->is_CFG(), "" )do { if (!(n2->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5190, "assert(" "n2->is_CFG()" ") failed", ""); ::breakpoint (); } } while (0); |
| 5191 | // find LCA of all uses |
| 5192 | uint d1 = dom_depth(n1); |
| 5193 | uint d2 = dom_depth(n2); |
| 5194 | while (n1 != n2) { |
| 5195 | if (d1 > d2) { |
| 5196 | n1 = idom(n1); |
| 5197 | d1 = dom_depth(n1); |
| 5198 | } else if (d1 < d2) { |
| 5199 | n2 = idom(n2); |
| 5200 | d2 = dom_depth(n2); |
| 5201 | } else { |
| 5202 | // Here d1 == d2. Due to edits of the dominator-tree, sections |
| 5203 | // of the tree might have the same depth. These sections have |
| 5204 | // to be searched more carefully. |
| 5205 | |
| 5206 | // Scan up all the n1's with equal depth, looking for n2. |
| 5207 | Node *t1 = idom(n1); |
| 5208 | while (dom_depth(t1) == d1) { |
| 5209 | if (t1 == n2) return n2; |
| 5210 | t1 = idom(t1); |
| 5211 | } |
| 5212 | // Scan up all the n2's with equal depth, looking for n1. |
| 5213 | Node *t2 = idom(n2); |
| 5214 | while (dom_depth(t2) == d2) { |
| 5215 | if (t2 == n1) return n1; |
| 5216 | t2 = idom(t2); |
| 5217 | } |
| 5218 | // Move up to a new dominator-depth value as well as up the dom-tree. |
| 5219 | n1 = t1; |
| 5220 | n2 = t2; |
| 5221 | d1 = dom_depth(n1); |
| 5222 | d2 = dom_depth(n2); |
| 5223 | } |
| 5224 | } |
| 5225 | return n1; |
| 5226 | } |
| 5227 | |
| 5228 | //------------------------------compute_idom----------------------------------- |
| 5229 | // Locally compute IDOM using dom_lca call. Correct only if the incoming |
| 5230 | // IDOMs are correct. |
| 5231 | Node *PhaseIdealLoop::compute_idom( Node *region ) const { |
| 5232 | assert( region->is_Region(), "" )do { if (!(region->is_Region())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5232, "assert(" "region->is_Region()" ") failed", ""); :: breakpoint(); } } while (0); |
| 5233 | Node *LCA = NULL__null; |
| 5234 | for( uint i = 1; i < region->req(); i++ ) { |
| 5235 | if( region->in(i) != C->top() ) |
| 5236 | LCA = dom_lca( LCA, region->in(i) ); |
| 5237 | } |
| 5238 | return LCA; |
| 5239 | } |
| 5240 | |
| 5241 | bool PhaseIdealLoop::verify_dominance(Node* n, Node* use, Node* LCA, Node* early) { |
| 5242 | bool had_error = false; |
| 5243 | #ifdef ASSERT1 |
| 5244 | if (early != C->root()) { |
| 5245 | // Make sure that there's a dominance path from LCA to early |
| 5246 | Node* d = LCA; |
| 5247 | while (d != early) { |
| 5248 | if (d == C->root()) { |
| 5249 | dump_bad_graph("Bad graph detected in compute_lca_of_uses", n, early, LCA); |
| 5250 | tty->print_cr("*** Use %d isn't dominated by def %d ***", use->_idx, n->_idx); |
| 5251 | had_error = true; |
| 5252 | break; |
| 5253 | } |
| 5254 | d = idom(d); |
| 5255 | } |
| 5256 | } |
| 5257 | #endif |
| 5258 | return had_error; |
| 5259 | } |
| 5260 | |
| 5261 | |
| 5262 | Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) { |
| 5263 | // Compute LCA over list of uses |
| 5264 | bool had_error = false; |
| 5265 | Node *LCA = NULL__null; |
| 5266 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && LCA != early; i++) { |
| 5267 | Node* c = n->fast_out(i); |
| 5268 | if (_nodes[c->_idx] == NULL__null) |
| 5269 | continue; // Skip the occasional dead node |
| 5270 | if( c->is_Phi() ) { // For Phis, we must land above on the path |
| 5271 | for( uint j=1; j<c->req(); j++ ) {// For all inputs |
| 5272 | if( c->in(j) == n ) { // Found matching input? |
| 5273 | Node *use = c->in(0)->in(j); |
| 5274 | if (_verify_only && use->is_top()) continue; |
| 5275 | LCA = dom_lca_for_get_late_ctrl( LCA, use, n ); |
| 5276 | if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error; |
| 5277 | } |
| 5278 | } |
| 5279 | } else { |
| 5280 | // For CFG data-users, use is in the block just prior |
| 5281 | Node *use = has_ctrl(c) ? get_ctrl(c) : c->in(0); |
| 5282 | LCA = dom_lca_for_get_late_ctrl( LCA, use, n ); |
| 5283 | if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error; |
| 5284 | } |
| 5285 | } |
| 5286 | assert(!had_error, "bad dominance")do { if (!(!had_error)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5286, "assert(" "!had_error" ") failed", "bad dominance"); :: breakpoint(); } } while (0); |
| 5287 | return LCA; |
| 5288 | } |
| 5289 | |
| 5290 | // Check the shape of the graph at the loop entry. In some cases, |
| 5291 | // the shape of the graph does not match the shape outlined below. |
| 5292 | // That is caused by the Opaque1 node "protecting" the shape of |
| 5293 | // the graph being removed by, for example, the IGVN performed |
| 5294 | // in PhaseIdealLoop::build_and_optimize(). |
| 5295 | // |
| 5296 | // After the Opaque1 node has been removed, optimizations (e.g., split-if, |
| 5297 | // loop unswitching, and IGVN, or a combination of them) can freely change |
| 5298 | // the graph's shape. As a result, the graph shape outlined below cannot |
| 5299 | // be guaranteed anymore. |
| 5300 | Node* CountedLoopNode::is_canonical_loop_entry() { |
| 5301 | if (!is_main_loop() && !is_post_loop()) { |
| 5302 | return NULL__null; |
| 5303 | } |
| 5304 | Node* ctrl = skip_predicates(); |
| 5305 | |
| 5306 | if (ctrl == NULL__null || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) { |
| 5307 | return NULL__null; |
| 5308 | } |
| 5309 | Node* iffm = ctrl->in(0); |
| 5310 | if (iffm == NULL__null || !iffm->is_If()) { |
| 5311 | return NULL__null; |
| 5312 | } |
| 5313 | Node* bolzm = iffm->in(1); |
| 5314 | if (bolzm == NULL__null || !bolzm->is_Bool()) { |
| 5315 | return NULL__null; |
| 5316 | } |
| 5317 | Node* cmpzm = bolzm->in(1); |
| 5318 | if (cmpzm == NULL__null || !cmpzm->is_Cmp()) { |
| 5319 | return NULL__null; |
| 5320 | } |
| 5321 | |
| 5322 | uint input = is_main_loop() ? 2 : 1; |
| 5323 | if (input >= cmpzm->req() || cmpzm->in(input) == NULL__null) { |
| 5324 | return NULL__null; |
| 5325 | } |
| 5326 | bool res = cmpzm->in(input)->Opcode() == Op_Opaque1; |
| 5327 | #ifdef ASSERT1 |
| 5328 | bool found_opaque = false; |
| 5329 | for (uint i = 1; i < cmpzm->req(); i++) { |
| 5330 | Node* opnd = cmpzm->in(i); |
| 5331 | if (opnd && opnd->Opcode() == Op_Opaque1) { |
| 5332 | found_opaque = true; |
| 5333 | break; |
| 5334 | } |
| 5335 | } |
| 5336 | assert(found_opaque == res, "wrong pattern")do { if (!(found_opaque == res)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5336, "assert(" "found_opaque == res" ") failed", "wrong pattern" ); ::breakpoint(); } } while (0); |
| 5337 | #endif |
| 5338 | return res ? cmpzm->in(input) : NULL__null; |
| 5339 | } |
| 5340 | |
| 5341 | //------------------------------get_late_ctrl---------------------------------- |
| 5342 | // Compute latest legal control. |
| 5343 | Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) { |
| 5344 | assert(early != NULL, "early control should not be NULL")do { if (!(early != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5344, "assert(" "early != __null" ") failed", "early control should not be NULL" ); ::breakpoint(); } } while (0); |
| 5345 | |
| 5346 | Node* LCA = compute_lca_of_uses(n, early); |
| 5347 | #ifdef ASSERT1 |
| 5348 | if (LCA == C->root() && LCA != early) { |
| 5349 | // def doesn't dominate uses so print some useful debugging output |
| 5350 | compute_lca_of_uses(n, early, true); |
| 5351 | } |
| 5352 | #endif |
| 5353 | |
| 5354 | if (n->is_Load() && LCA != early) { |
| 5355 | LCA = get_late_ctrl_with_anti_dep(n->as_Load(), early, LCA); |
| 5356 | } |
| 5357 | |
| 5358 | assert(LCA == find_non_split_ctrl(LCA), "unexpected late control")do { if (!(LCA == find_non_split_ctrl(LCA))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5358, "assert(" "LCA == find_non_split_ctrl(LCA)" ") failed" , "unexpected late control"); ::breakpoint(); } } while (0); |
| 5359 | return LCA; |
| 5360 | } |
| 5361 | |
| 5362 | // if this is a load, check for anti-dependent stores |
| 5363 | // We use a conservative algorithm to identify potential interfering |
| 5364 | // instructions and for rescheduling the load. The users of the memory |
| 5365 | // input of this load are examined. Any use which is not a load and is |
| 5366 | // dominated by early is considered a potentially interfering store. |
| 5367 | // This can produce false positives. |
| 5368 | Node* PhaseIdealLoop::get_late_ctrl_with_anti_dep(LoadNode* n, Node* early, Node* LCA) { |
| 5369 | int load_alias_idx = C->get_alias_index(n->adr_type()); |
| 5370 | if (C->alias_type(load_alias_idx)->is_rewritable()) { |
| 5371 | Unique_Node_List worklist; |
| 5372 | |
| 5373 | Node* mem = n->in(MemNode::Memory); |
| 5374 | for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { |
| 5375 | Node* s = mem->fast_out(i); |
| 5376 | worklist.push(s); |
| 5377 | } |
| 5378 | for (uint i = 0; i < worklist.size() && LCA != early; i++) { |
| 5379 | Node* s = worklist.at(i); |
| 5380 | if (s->is_Load() || s->Opcode() == Op_SafePoint || |
| 5381 | (s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0) || |
| 5382 | s->is_Phi()) { |
| 5383 | continue; |
| 5384 | } else if (s->is_MergeMem()) { |
| 5385 | for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { |
| 5386 | Node* s1 = s->fast_out(i); |
| 5387 | worklist.push(s1); |
| 5388 | } |
| 5389 | } else { |
| 5390 | Node* sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0); |
| 5391 | assert(sctrl != NULL || !s->is_reachable_from_root(), "must have control")do { if (!(sctrl != __null || !s->is_reachable_from_root() )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5391, "assert(" "sctrl != __null || !s->is_reachable_from_root()" ") failed", "must have control"); ::breakpoint(); } } while ( 0); |
| 5392 | if (sctrl != NULL__null && !sctrl->is_top() && is_dominator(early, sctrl)) { |
| 5393 | const TypePtr* adr_type = s->adr_type(); |
| 5394 | if (s->is_ArrayCopy()) { |
| 5395 | // Copy to known instance needs destination type to test for aliasing |
| 5396 | const TypePtr* dest_type = s->as_ArrayCopy()->_dest_type; |
| 5397 | if (dest_type != TypeOopPtr::BOTTOM) { |
| 5398 | adr_type = dest_type; |
| 5399 | } |
| 5400 | } |
| 5401 | if (C->can_alias(adr_type, load_alias_idx)) { |
| 5402 | LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n); |
| 5403 | } else if (s->is_CFG() && s->is_Multi()) { |
| 5404 | // Look for the memory use of s (that is the use of its memory projection) |
| 5405 | for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { |
| 5406 | Node* s1 = s->fast_out(i); |
| 5407 | assert(s1->is_Proj(), "projection expected")do { if (!(s1->is_Proj())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5407, "assert(" "s1->is_Proj()" ") failed", "projection expected" ); ::breakpoint(); } } while (0); |
| 5408 | if (_igvn.type(s1) == Type::MEMORY) { |
| 5409 | for (DUIterator_Fast jmax, j = s1->fast_outs(jmax); j < jmax; j++) { |
| 5410 | Node* s2 = s1->fast_out(j); |
| 5411 | worklist.push(s2); |
| 5412 | } |
| 5413 | } |
| 5414 | } |
| 5415 | } |
| 5416 | } |
| 5417 | } |
| 5418 | } |
| 5419 | // For Phis only consider Region's inputs that were reached by following the memory edges |
| 5420 | if (LCA != early) { |
| 5421 | for (uint i = 0; i < worklist.size(); i++) { |
| 5422 | Node* s = worklist.at(i); |
| 5423 | if (s->is_Phi() && C->can_alias(s->adr_type(), load_alias_idx)) { |
| 5424 | Node* r = s->in(0); |
| 5425 | for (uint j = 1; j < s->req(); j++) { |
| 5426 | Node* in = s->in(j); |
| 5427 | Node* r_in = r->in(j); |
| 5428 | // We can't reach any node from a Phi because we don't enqueue Phi's uses above |
| 5429 | if (((worklist.member(in) && !in->is_Phi()) || in == mem) && is_dominator(early, r_in)) { |
| 5430 | LCA = dom_lca_for_get_late_ctrl(LCA, r_in, n); |
| 5431 | } |
| 5432 | } |
| 5433 | } |
| 5434 | } |
| 5435 | } |
| 5436 | } |
| 5437 | return LCA; |
| 5438 | } |
| 5439 | |
| 5440 | // true if CFG node d dominates CFG node n |
| 5441 | bool PhaseIdealLoop::is_dominator(Node *d, Node *n) { |
| 5442 | if (d == n) |
| 5443 | return true; |
| 5444 | assert(d->is_CFG() && n->is_CFG(), "must have CFG nodes")do { if (!(d->is_CFG() && n->is_CFG())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5444, "assert(" "d->is_CFG() && n->is_CFG()" ") failed" , "must have CFG nodes"); ::breakpoint(); } } while (0); |
| 5445 | uint dd = dom_depth(d); |
| 5446 | while (dom_depth(n) >= dd) { |
| 5447 | if (n == d) |
| 5448 | return true; |
| 5449 | n = idom(n); |
| 5450 | } |
| 5451 | return false; |
| 5452 | } |
| 5453 | |
| 5454 | //------------------------------dom_lca_for_get_late_ctrl_internal------------- |
| 5455 | // Pair-wise LCA with tags. |
| 5456 | // Tag each index with the node 'tag' currently being processed |
| 5457 | // before advancing up the dominator chain using idom(). |
| 5458 | // Later calls that find a match to 'tag' know that this path has already |
| 5459 | // been considered in the current LCA (which is input 'n1' by convention). |
| 5460 | // Since get_late_ctrl() is only called once for each node, the tag array |
| 5461 | // does not need to be cleared between calls to get_late_ctrl(). |
| 5462 | // Algorithm trades a larger constant factor for better asymptotic behavior |
| 5463 | // |
| 5464 | Node *PhaseIdealLoop::dom_lca_for_get_late_ctrl_internal(Node *n1, Node *n2, Node *tag_node) { |
| 5465 | uint d1 = dom_depth(n1); |
| 5466 | uint d2 = dom_depth(n2); |
| 5467 | jlong tag = tag_node->_idx | (((jlong)_dom_lca_tags_round) << 32); |
| 5468 | |
| 5469 | do { |
| 5470 | if (d1 > d2) { |
| 5471 | // current lca is deeper than n2 |
| 5472 | _dom_lca_tags.at_put_grow(n1->_idx, tag); |
| 5473 | n1 = idom(n1); |
| 5474 | d1 = dom_depth(n1); |
| 5475 | } else if (d1 < d2) { |
| 5476 | // n2 is deeper than current lca |
| 5477 | jlong memo = _dom_lca_tags.at_grow(n2->_idx, 0); |
| 5478 | if (memo == tag) { |
| 5479 | return n1; // Return the current LCA |
| 5480 | } |
| 5481 | _dom_lca_tags.at_put_grow(n2->_idx, tag); |
| 5482 | n2 = idom(n2); |
| 5483 | d2 = dom_depth(n2); |
| 5484 | } else { |
| 5485 | // Here d1 == d2. Due to edits of the dominator-tree, sections |
| 5486 | // of the tree might have the same depth. These sections have |
| 5487 | // to be searched more carefully. |
| 5488 | |
| 5489 | // Scan up all the n1's with equal depth, looking for n2. |
| 5490 | _dom_lca_tags.at_put_grow(n1->_idx, tag); |
| 5491 | Node *t1 = idom(n1); |
| 5492 | while (dom_depth(t1) == d1) { |
| 5493 | if (t1 == n2) return n2; |
| 5494 | _dom_lca_tags.at_put_grow(t1->_idx, tag); |
| 5495 | t1 = idom(t1); |
| 5496 | } |
| 5497 | // Scan up all the n2's with equal depth, looking for n1. |
| 5498 | _dom_lca_tags.at_put_grow(n2->_idx, tag); |
| 5499 | Node *t2 = idom(n2); |
| 5500 | while (dom_depth(t2) == d2) { |
| 5501 | if (t2 == n1) return n1; |
| 5502 | _dom_lca_tags.at_put_grow(t2->_idx, tag); |
| 5503 | t2 = idom(t2); |
| 5504 | } |
| 5505 | // Move up to a new dominator-depth value as well as up the dom-tree. |
| 5506 | n1 = t1; |
| 5507 | n2 = t2; |
| 5508 | d1 = dom_depth(n1); |
| 5509 | d2 = dom_depth(n2); |
| 5510 | } |
| 5511 | } while (n1 != n2); |
| 5512 | return n1; |
| 5513 | } |
| 5514 | |
| 5515 | //------------------------------init_dom_lca_tags------------------------------ |
| 5516 | // Tag could be a node's integer index, 32bits instead of 64bits in some cases |
| 5517 | // Intended use does not involve any growth for the array, so it could |
| 5518 | // be of fixed size. |
| 5519 | void PhaseIdealLoop::init_dom_lca_tags() { |
| 5520 | uint limit = C->unique() + 1; |
| 5521 | _dom_lca_tags.at_grow(limit, 0); |
| 5522 | _dom_lca_tags_round = 0; |
| 5523 | #ifdef ASSERT1 |
| 5524 | for (uint i = 0; i < limit; ++i) { |
| 5525 | assert(_dom_lca_tags.at(i) == 0, "Must be distinct from each node pointer")do { if (!(_dom_lca_tags.at(i) == 0)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5525, "assert(" "_dom_lca_tags.at(i) == 0" ") failed", "Must be distinct from each node pointer" ); ::breakpoint(); } } while (0); |
| 5526 | } |
| 5527 | #endif // ASSERT |
| 5528 | } |
| 5529 | |
| 5530 | //------------------------------build_loop_late-------------------------------- |
| 5531 | // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. |
| 5532 | // Second pass finds latest legal placement, and ideal loop placement. |
| 5533 | void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { |
| 5534 | while (worklist.size() != 0) { |
| 5535 | Node *n = worklist.pop(); |
| 5536 | // Only visit once |
| 5537 | if (visited.test_set(n->_idx)) continue; |
| 5538 | uint cnt = n->outcnt(); |
| 5539 | uint i = 0; |
| 5540 | while (true) { |
| 5541 | assert( _nodes[n->_idx], "no dead nodes" )do { if (!(_nodes[n->_idx])) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5541, "assert(" "_nodes[n->_idx]" ") failed", "no dead nodes" ); ::breakpoint(); } } while (0); |
| 5542 | // Visit all children |
| 5543 | if (i < cnt) { |
| 5544 | Node* use = n->raw_out(i); |
| 5545 | ++i; |
| 5546 | // Check for dead uses. Aggressively prune such junk. It might be |
| 5547 | // dead in the global sense, but still have local uses so I cannot |
| 5548 | // easily call 'remove_dead_node'. |
| 5549 | if( _nodes[use->_idx] != NULL__null || use->is_top() ) { // Not dead? |
| 5550 | // Due to cycles, we might not hit the same fixed point in the verify |
| 5551 | // pass as we do in the regular pass. Instead, visit such phis as |
| 5552 | // simple uses of the loop head. |
| 5553 | if( use->in(0) && (use->is_CFG() || use->is_Phi()) ) { |
| 5554 | if( !visited.test(use->_idx) ) |
| 5555 | worklist.push(use); |
| 5556 | } else if( !visited.test_set(use->_idx) ) { |
| 5557 | nstack.push(n, i); // Save parent and next use's index. |
| 5558 | n = use; // Process all children of current use. |
| 5559 | cnt = use->outcnt(); |
| 5560 | i = 0; |
| 5561 | } |
| 5562 | } else { |
| 5563 | // Do not visit around the backedge of loops via data edges. |
| 5564 | // push dead code onto a worklist |
| 5565 | _deadlist.push(use); |
| 5566 | } |
| 5567 | } else { |
| 5568 | // All of n's children have been processed, complete post-processing. |
| 5569 | build_loop_late_post(n); |
| 5570 | if (nstack.is_empty()) { |
| 5571 | // Finished all nodes on stack. |
| 5572 | // Process next node on the worklist. |
| 5573 | break; |
| 5574 | } |
| 5575 | // Get saved parent node and next use's index. Visit the rest of uses. |
| 5576 | n = nstack.node(); |
| 5577 | cnt = n->outcnt(); |
| 5578 | i = nstack.index(); |
| 5579 | nstack.pop(); |
| 5580 | } |
| 5581 | } |
| 5582 | } |
| 5583 | } |
| 5584 | |
| 5585 | // Verify that no data node is scheduled in the outer loop of a strip |
| 5586 | // mined loop. |
| 5587 | void PhaseIdealLoop::verify_strip_mined_scheduling(Node *n, Node* least) { |
| 5588 | #ifdef ASSERT1 |
| 5589 | if (get_loop(least)->_nest == 0) { |
| 5590 | return; |
| 5591 | } |
| 5592 | IdealLoopTree* loop = get_loop(least); |
| 5593 | Node* head = loop->_head; |
| 5594 | if (head->is_OuterStripMinedLoop() && |
| 5595 | // Verification can't be applied to fully built strip mined loops |
| 5596 | head->as_Loop()->outer_loop_end()->in(1)->find_int_con(-1) == 0) { |
| 5597 | Node* sfpt = head->as_Loop()->outer_safepoint(); |
| 5598 | ResourceMark rm; |
| 5599 | Unique_Node_List wq; |
| 5600 | wq.push(sfpt); |
| 5601 | for (uint i = 0; i < wq.size(); i++) { |
| 5602 | Node *m = wq.at(i); |
| 5603 | for (uint i = 1; i < m->req(); i++) { |
| 5604 | Node* nn = m->in(i); |
| 5605 | if (nn == n) { |
| 5606 | return; |
| 5607 | } |
| 5608 | if (nn != NULL__null && has_ctrl(nn) && get_loop(get_ctrl(nn)) == loop) { |
| 5609 | wq.push(nn); |
| 5610 | } |
| 5611 | } |
| 5612 | } |
| 5613 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5613); ::breakpoint(); } while (0); |
| 5614 | } |
| 5615 | #endif |
| 5616 | } |
| 5617 | |
| 5618 | |
| 5619 | //------------------------------build_loop_late_post--------------------------- |
| 5620 | // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. |
| 5621 | // Second pass finds latest legal placement, and ideal loop placement. |
| 5622 | void PhaseIdealLoop::build_loop_late_post(Node *n) { |
| 5623 | build_loop_late_post_work(n, true); |
| 5624 | } |
| 5625 | |
| 5626 | void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { |
| 5627 | |
| 5628 | if (n->req() == 2 && (n->Opcode() == Op_ConvI2L || n->Opcode() == Op_CastII) && !C->major_progress() && !_verify_only) { |
| 5629 | _igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops. |
| 5630 | } |
| 5631 | |
| 5632 | #ifdef ASSERT1 |
| 5633 | if (_verify_only && !n->is_CFG()) { |
| 5634 | // Check def-use domination. |
| 5635 | compute_lca_of_uses(n, get_ctrl(n), true /* verify */); |
| 5636 | } |
| 5637 | #endif |
| 5638 | |
| 5639 | // CFG and pinned nodes already handled |
| 5640 | if( n->in(0) ) { |
| 5641 | if( n->in(0)->is_top() ) return; // Dead? |
| 5642 | |
| 5643 | // We'd like +VerifyLoopOptimizations to not believe that Mod's/Loads |
| 5644 | // _must_ be pinned (they have to observe their control edge of course). |
| 5645 | // Unlike Stores (which modify an unallocable resource, the memory |
| 5646 | // state), Mods/Loads can float around. So free them up. |
| 5647 | switch( n->Opcode() ) { |
| 5648 | case Op_DivI: |
| 5649 | case Op_DivF: |
| 5650 | case Op_DivD: |
| 5651 | case Op_ModI: |
| 5652 | case Op_ModF: |
| 5653 | case Op_ModD: |
| 5654 | case Op_LoadB: // Same with Loads; they can sink |
| 5655 | case Op_LoadUB: // during loop optimizations. |
| 5656 | case Op_LoadUS: |
| 5657 | case Op_LoadD: |
| 5658 | case Op_LoadF: |
| 5659 | case Op_LoadI: |
| 5660 | case Op_LoadKlass: |
| 5661 | case Op_LoadNKlass: |
| 5662 | case Op_LoadL: |
| 5663 | case Op_LoadS: |
| 5664 | case Op_LoadP: |
| 5665 | case Op_LoadN: |
| 5666 | case Op_LoadRange: |
| 5667 | case Op_LoadD_unaligned: |
| 5668 | case Op_LoadL_unaligned: |
| 5669 | case Op_StrComp: // Does a bunch of load-like effects |
| 5670 | case Op_StrEquals: |
| 5671 | case Op_StrIndexOf: |
| 5672 | case Op_StrIndexOfChar: |
| 5673 | case Op_AryEq: |
| 5674 | case Op_HasNegatives: |
| 5675 | pinned = false; |
| 5676 | } |
| 5677 | if (n->is_CMove() || n->is_ConstraintCast()) { |
| 5678 | pinned = false; |
| 5679 | } |
| 5680 | if( pinned ) { |
| 5681 | IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n)); |
| 5682 | if( !chosen_loop->_child ) // Inner loop? |
| 5683 | chosen_loop->_body.push(n); // Collect inner loops |
| 5684 | return; |
| 5685 | } |
| 5686 | } else { // No slot zero |
| 5687 | if( n->is_CFG() ) { // CFG with no slot 0 is dead |
| 5688 | _nodes.map(n->_idx,0); // No block setting, it's globally dead |
| 5689 | return; |
| 5690 | } |
| 5691 | assert(!n->is_CFG() || n->outcnt() == 0, "")do { if (!(!n->is_CFG() || n->outcnt() == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5691, "assert(" "!n->is_CFG() || n->outcnt() == 0" ") failed" , ""); ::breakpoint(); } } while (0); |
| 5692 | } |
| 5693 | |
| 5694 | // Do I have a "safe range" I can select over? |
| 5695 | Node *early = get_ctrl(n);// Early location already computed |
| 5696 | |
| 5697 | // Compute latest point this Node can go |
| 5698 | Node *LCA = get_late_ctrl( n, early ); |
| 5699 | // LCA is NULL due to uses being dead |
| 5700 | if( LCA == NULL__null ) { |
| 5701 | #ifdef ASSERT1 |
| 5702 | for (DUIterator i1 = n->outs(); n->has_out(i1); i1++) { |
| 5703 | assert( _nodes[n->out(i1)->_idx] == NULL, "all uses must also be dead")do { if (!(_nodes[n->out(i1)->_idx] == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5703, "assert(" "_nodes[n->out(i1)->_idx] == __null" ") failed" , "all uses must also be dead"); ::breakpoint(); } } while (0 ); |
| 5704 | } |
| 5705 | #endif |
| 5706 | _nodes.map(n->_idx, 0); // This node is useless |
| 5707 | _deadlist.push(n); |
| 5708 | return; |
| 5709 | } |
| 5710 | assert(LCA != NULL && !LCA->is_top(), "no dead nodes")do { if (!(LCA != __null && !LCA->is_top())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5710, "assert(" "LCA != __null && !LCA->is_top()" ") failed", "no dead nodes"); ::breakpoint(); } } while (0); |
| 5711 | |
| 5712 | Node *legal = LCA; // Walk 'legal' up the IDOM chain |
| 5713 | Node *least = legal; // Best legal position so far |
| 5714 | while( early != legal ) { // While not at earliest legal |
| 5715 | #ifdef ASSERT1 |
| 5716 | if (legal->is_Start() && !early->is_Root()) { |
| 5717 | // Bad graph. Print idom path and fail. |
| 5718 | dump_bad_graph("Bad graph detected in build_loop_late", n, early, LCA); |
| 5719 | assert(false, "Bad graph detected in build_loop_late")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5719, "assert(" "false" ") failed", "Bad graph detected in build_loop_late" ); ::breakpoint(); } } while (0); |
| 5720 | } |
| 5721 | #endif |
| 5722 | // Find least loop nesting depth |
| 5723 | legal = idom(legal); // Bump up the IDOM tree |
| 5724 | // Check for lower nesting depth |
| 5725 | if( get_loop(legal)->_nest < get_loop(least)->_nest ) |
| 5726 | least = legal; |
| 5727 | } |
| 5728 | assert(early == legal || legal != C->root(), "bad dominance of inputs")do { if (!(early == legal || legal != C->root())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5728, "assert(" "early == legal || legal != C->root()" ") failed" , "bad dominance of inputs"); ::breakpoint(); } } while (0); |
| 5729 | |
| 5730 | // Try not to place code on a loop entry projection |
| 5731 | // which can inhibit range check elimination. |
| 5732 | if (least != early) { |
| 5733 | Node* ctrl_out = least->unique_ctrl_out(); |
| 5734 | if (ctrl_out && ctrl_out->is_Loop() && |
| 5735 | least == ctrl_out->in(LoopNode::EntryControl)) { |
| 5736 | // Move the node above predicates as far up as possible so a |
| 5737 | // following pass of loop predication doesn't hoist a predicate |
| 5738 | // that depends on it above that node. |
| 5739 | Node* new_ctrl = least; |
| 5740 | for (;;) { |
| 5741 | if (!new_ctrl->is_Proj()) { |
| 5742 | break; |
| 5743 | } |
| 5744 | CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); |
| 5745 | if (call == NULL__null) { |
| 5746 | break; |
| 5747 | } |
| 5748 | int req = call->uncommon_trap_request(); |
| 5749 | Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); |
| 5750 | if (trap_reason != Deoptimization::Reason_loop_limit_check && |
| 5751 | trap_reason != Deoptimization::Reason_predicate && |
| 5752 | trap_reason != Deoptimization::Reason_profile_predicate) { |
| 5753 | break; |
| 5754 | } |
| 5755 | Node* c = new_ctrl->in(0)->in(0); |
| 5756 | if (is_dominator(c, early) && c != early) { |
| 5757 | break; |
| 5758 | } |
| 5759 | new_ctrl = c; |
| 5760 | } |
| 5761 | least = new_ctrl; |
| 5762 | } |
| 5763 | } |
| 5764 | |
| 5765 | #ifdef ASSERT1 |
| 5766 | // If verifying, verify that 'verify_me' has a legal location |
| 5767 | // and choose it as our location. |
| 5768 | if( _verify_me ) { |
| 5769 | Node *v_ctrl = _verify_me->get_ctrl_no_update(n); |
| 5770 | Node *legal = LCA; |
| 5771 | while( early != legal ) { // While not at earliest legal |
| 5772 | if( legal == v_ctrl ) break; // Check for prior good location |
| 5773 | legal = idom(legal) ;// Bump up the IDOM tree |
| 5774 | } |
| 5775 | // Check for prior good location |
| 5776 | if( legal == v_ctrl ) least = legal; // Keep prior if found |
| 5777 | } |
| 5778 | #endif |
| 5779 | |
| 5780 | // Assign discovered "here or above" point |
| 5781 | least = find_non_split_ctrl(least); |
| 5782 | verify_strip_mined_scheduling(n, least); |
| 5783 | set_ctrl(n, least); |
| 5784 | |
| 5785 | // Collect inner loop bodies |
| 5786 | IdealLoopTree *chosen_loop = get_loop(least); |
| 5787 | if( !chosen_loop->_child ) // Inner loop? |
| 5788 | chosen_loop->_body.push(n);// Collect inner loops |
| 5789 | } |
| 5790 | |
| 5791 | #ifdef ASSERT1 |
| 5792 | void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) { |
| 5793 | tty->print_cr("%s", msg); |
| 5794 | tty->print("n: "); n->dump(); |
| 5795 | tty->print("early(n): "); early->dump(); |
| 5796 | if (n->in(0) != NULL__null && !n->in(0)->is_top() && |
| 5797 | n->in(0) != early && !n->in(0)->is_Root()) { |
| 5798 | tty->print("n->in(0): "); n->in(0)->dump(); |
| 5799 | } |
| 5800 | for (uint i = 1; i < n->req(); i++) { |
| 5801 | Node* in1 = n->in(i); |
| 5802 | if (in1 != NULL__null && in1 != n && !in1->is_top()) { |
| 5803 | tty->print("n->in(%d): ", i); in1->dump(); |
| 5804 | Node* in1_early = get_ctrl(in1); |
| 5805 | tty->print("early(n->in(%d)): ", i); in1_early->dump(); |
| 5806 | if (in1->in(0) != NULL__null && !in1->in(0)->is_top() && |
| 5807 | in1->in(0) != in1_early && !in1->in(0)->is_Root()) { |
| 5808 | tty->print("n->in(%d)->in(0): ", i); in1->in(0)->dump(); |
| 5809 | } |
| 5810 | for (uint j = 1; j < in1->req(); j++) { |
| 5811 | Node* in2 = in1->in(j); |
| 5812 | if (in2 != NULL__null && in2 != n && in2 != in1 && !in2->is_top()) { |
| 5813 | tty->print("n->in(%d)->in(%d): ", i, j); in2->dump(); |
| 5814 | Node* in2_early = get_ctrl(in2); |
| 5815 | tty->print("early(n->in(%d)->in(%d)): ", i, j); in2_early->dump(); |
| 5816 | if (in2->in(0) != NULL__null && !in2->in(0)->is_top() && |
| 5817 | in2->in(0) != in2_early && !in2->in(0)->is_Root()) { |
| 5818 | tty->print("n->in(%d)->in(%d)->in(0): ", i, j); in2->in(0)->dump(); |
| 5819 | } |
| 5820 | } |
| 5821 | } |
| 5822 | } |
| 5823 | } |
| 5824 | tty->cr(); |
| 5825 | tty->print("LCA(n): "); LCA->dump(); |
| 5826 | for (uint i = 0; i < n->outcnt(); i++) { |
| 5827 | Node* u1 = n->raw_out(i); |
| 5828 | if (u1 == n) |
| 5829 | continue; |
| 5830 | tty->print("n->out(%d): ", i); u1->dump(); |
| 5831 | if (u1->is_CFG()) { |
| 5832 | for (uint j = 0; j < u1->outcnt(); j++) { |
| 5833 | Node* u2 = u1->raw_out(j); |
| 5834 | if (u2 != u1 && u2 != n && u2->is_CFG()) { |
| 5835 | tty->print("n->out(%d)->out(%d): ", i, j); u2->dump(); |
| 5836 | } |
| 5837 | } |
| 5838 | } else { |
| 5839 | Node* u1_later = get_ctrl(u1); |
| 5840 | tty->print("later(n->out(%d)): ", i); u1_later->dump(); |
| 5841 | if (u1->in(0) != NULL__null && !u1->in(0)->is_top() && |
| 5842 | u1->in(0) != u1_later && !u1->in(0)->is_Root()) { |
| 5843 | tty->print("n->out(%d)->in(0): ", i); u1->in(0)->dump(); |
| 5844 | } |
| 5845 | for (uint j = 0; j < u1->outcnt(); j++) { |
| 5846 | Node* u2 = u1->raw_out(j); |
| 5847 | if (u2 == n || u2 == u1) |
| 5848 | continue; |
| 5849 | tty->print("n->out(%d)->out(%d): ", i, j); u2->dump(); |
| 5850 | if (!u2->is_CFG()) { |
| 5851 | Node* u2_later = get_ctrl(u2); |
| 5852 | tty->print("later(n->out(%d)->out(%d)): ", i, j); u2_later->dump(); |
| 5853 | if (u2->in(0) != NULL__null && !u2->in(0)->is_top() && |
| 5854 | u2->in(0) != u2_later && !u2->in(0)->is_Root()) { |
| 5855 | tty->print("n->out(%d)->in(0): ", i); u2->in(0)->dump(); |
| 5856 | } |
| 5857 | } |
| 5858 | } |
| 5859 | } |
| 5860 | } |
| 5861 | tty->cr(); |
| 5862 | tty->print_cr("idoms of early %d:", early->_idx); |
| 5863 | dump_idom(early); |
| 5864 | tty->cr(); |
| 5865 | tty->print_cr("idoms of (wrong) LCA %d:", LCA->_idx); |
| 5866 | dump_idom(LCA); |
| 5867 | tty->cr(); |
| 5868 | dump_real_LCA(early, LCA); |
| 5869 | tty->cr(); |
| 5870 | } |
| 5871 | |
| 5872 | // Find the real LCA of early and the wrongly assumed LCA. |
| 5873 | void PhaseIdealLoop::dump_real_LCA(Node* early, Node* wrong_lca) { |
| 5874 | assert(!is_dominator(early, wrong_lca) && !is_dominator(early, wrong_lca),do { if (!(!is_dominator(early, wrong_lca) && !is_dominator (early, wrong_lca))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5875, "assert(" "!is_dominator(early, wrong_lca) && !is_dominator(early, wrong_lca)" ") failed", "sanity check that one node does not dominate the other" ); ::breakpoint(); } } while (0) |
| 5875 | "sanity check that one node does not dominate the other")do { if (!(!is_dominator(early, wrong_lca) && !is_dominator (early, wrong_lca))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5875, "assert(" "!is_dominator(early, wrong_lca) && !is_dominator(early, wrong_lca)" ") failed", "sanity check that one node does not dominate the other" ); ::breakpoint(); } } while (0); |
| 5876 | assert(!has_ctrl(early) && !has_ctrl(wrong_lca), "sanity check, no data nodes")do { if (!(!has_ctrl(early) && !has_ctrl(wrong_lca))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5876, "assert(" "!has_ctrl(early) && !has_ctrl(wrong_lca)" ") failed", "sanity check, no data nodes"); ::breakpoint(); } } while (0); |
| 5877 | |
| 5878 | ResourceMark rm; |
| 5879 | Node_List nodes_seen; |
| 5880 | Node* real_LCA = NULL__null; |
| 5881 | Node* n1 = wrong_lca; |
| 5882 | Node* n2 = early; |
| 5883 | uint count_1 = 0; |
| 5884 | uint count_2 = 0; |
| 5885 | // Add early and wrong_lca to simplify calculation of idom indices |
| 5886 | nodes_seen.push(n1); |
| 5887 | nodes_seen.push(n2); |
| 5888 | |
| 5889 | // Walk the idom chain up from early and wrong_lca and stop when they intersect. |
| 5890 | while (!n1->is_Start() && !n2->is_Start()) { |
| 5891 | n1 = idom(n1); |
| 5892 | n2 = idom(n2); |
| 5893 | if (n1 == n2) { |
| 5894 | // Both idom chains intersect at the same index |
| 5895 | real_LCA = n1; |
| 5896 | count_1 = nodes_seen.size() / 2; |
| 5897 | count_2 = count_1; |
| 5898 | break; |
| 5899 | } |
| 5900 | if (check_idom_chains_intersection(n1, count_1, count_2, &nodes_seen)) { |
| 5901 | real_LCA = n1; |
| 5902 | break; |
| 5903 | } |
| 5904 | if (check_idom_chains_intersection(n2, count_2, count_1, &nodes_seen)) { |
| 5905 | real_LCA = n2; |
| 5906 | break; |
| 5907 | } |
| 5908 | nodes_seen.push(n1); |
| 5909 | nodes_seen.push(n2); |
| 5910 | } |
| 5911 | |
| 5912 | assert(real_LCA != NULL, "must always find an LCA")do { if (!(real_LCA != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5912, "assert(" "real_LCA != __null" ") failed", "must always find an LCA" ); ::breakpoint(); } } while (0); |
| 5913 | tty->print_cr("Real LCA of early %d (idom[%d]) and (wrong) LCA %d (idom[%d]):", early->_idx, count_2, wrong_lca->_idx, count_1); |
| 5914 | real_LCA->dump(); |
| 5915 | } |
| 5916 | |
| 5917 | // Check if n is already on nodes_seen (i.e. idom chains of early and wrong_lca intersect at n). Determine the idom index of n |
| 5918 | // on both idom chains and return them in idom_idx_new and idom_idx_other, respectively. |
| 5919 | bool PhaseIdealLoop::check_idom_chains_intersection(const Node* n, uint& idom_idx_new, uint& idom_idx_other, const Node_List* nodes_seen) const { |
| 5920 | if (nodes_seen->contains(n)) { |
| 5921 | // The idom chain has just discovered n. |
| 5922 | // Divide by 2 because nodes_seen contains the same amount of nodes from both chains. |
| 5923 | idom_idx_new = nodes_seen->size() / 2; |
| 5924 | |
| 5925 | // The other chain already contained n. Search the index. |
| 5926 | for (uint i = 0; i < nodes_seen->size(); i++) { |
| 5927 | if (nodes_seen->at(i) == n) { |
| 5928 | // Divide by 2 because nodes_seen contains the same amount of nodes from both chains. |
| 5929 | idom_idx_other = i / 2; |
| 5930 | } |
| 5931 | } |
| 5932 | return true; |
| 5933 | } |
| 5934 | return false; |
| 5935 | } |
| 5936 | #endif // ASSERT |
| 5937 | |
| 5938 | #ifndef PRODUCT |
| 5939 | //------------------------------dump------------------------------------------- |
| 5940 | void PhaseIdealLoop::dump() const { |
| 5941 | ResourceMark rm; |
| 5942 | Node_Stack stack(C->live_nodes() >> 2); |
| 5943 | Node_List rpo_list; |
| 5944 | VectorSet visited; |
| 5945 | visited.set(C->top()->_idx); |
| 5946 | rpo(C->root(), stack, visited, rpo_list); |
| 5947 | // Dump root loop indexed by last element in PO order |
| 5948 | dump(_ltree_root, rpo_list.size(), rpo_list); |
| 5949 | } |
| 5950 | |
| 5951 | void PhaseIdealLoop::dump(IdealLoopTree* loop, uint idx, Node_List &rpo_list) const { |
| 5952 | loop->dump_head(); |
| 5953 | |
| 5954 | // Now scan for CFG nodes in the same loop |
| 5955 | for (uint j = idx; j > 0; j--) { |
| 5956 | Node* n = rpo_list[j-1]; |
| 5957 | if (!_nodes[n->_idx]) // Skip dead nodes |
| 5958 | continue; |
| 5959 | |
| 5960 | if (get_loop(n) != loop) { // Wrong loop nest |
| 5961 | if (get_loop(n)->_head == n && // Found nested loop? |
| 5962 | get_loop(n)->_parent == loop) |
| 5963 | dump(get_loop(n), rpo_list.size(), rpo_list); // Print it nested-ly |
| 5964 | continue; |
| 5965 | } |
| 5966 | |
| 5967 | // Dump controlling node |
| 5968 | tty->sp(2 * loop->_nest); |
| 5969 | tty->print("C"); |
| 5970 | if (n == C->root()) { |
| 5971 | n->dump(); |
| 5972 | } else { |
| 5973 | Node* cached_idom = idom_no_update(n); |
| 5974 | Node* computed_idom = n->in(0); |
| 5975 | if (n->is_Region()) { |
| 5976 | computed_idom = compute_idom(n); |
| 5977 | // computed_idom() will return n->in(0) when idom(n) is an IfNode (or |
| 5978 | // any MultiBranch ctrl node), so apply a similar transform to |
| 5979 | // the cached idom returned from idom_no_update. |
| 5980 | cached_idom = find_non_split_ctrl(cached_idom); |
| 5981 | } |
| 5982 | tty->print(" ID:%d", computed_idom->_idx); |
| 5983 | n->dump(); |
| 5984 | if (cached_idom != computed_idom) { |
| 5985 | tty->print_cr("*** BROKEN IDOM! Computed as: %d, cached as: %d", |
| 5986 | computed_idom->_idx, cached_idom->_idx); |
| 5987 | } |
| 5988 | } |
| 5989 | // Dump nodes it controls |
| 5990 | for (uint k = 0; k < _nodes.Size(); k++) { |
| 5991 | // (k < C->unique() && get_ctrl(find(k)) == n) |
| 5992 | if (k < C->unique() && _nodes[k] == (Node*)((intptr_t)n + 1)) { |
| 5993 | Node* m = C->root()->find(k); |
| 5994 | if (m && m->outcnt() > 0) { |
| 5995 | if (!(has_ctrl(m) && get_ctrl_no_update(m) == n)) { |
| 5996 | tty->print_cr("*** BROKEN CTRL ACCESSOR! _nodes[k] is %p, ctrl is %p", |
| 5997 | _nodes[k], has_ctrl(m) ? get_ctrl_no_update(m) : NULL__null); |
| 5998 | } |
| 5999 | tty->sp(2 * loop->_nest + 1); |
| 6000 | m->dump(); |
| 6001 | } |
| 6002 | } |
| 6003 | } |
| 6004 | } |
| 6005 | } |
| 6006 | |
| 6007 | void PhaseIdealLoop::dump_idom(Node* n) const { |
| 6008 | if (has_ctrl(n)) { |
| 6009 | tty->print_cr("No idom for data nodes"); |
| 6010 | } else { |
| 6011 | for (int i = 0; i < 100 && !n->is_Start(); i++) { |
| 6012 | tty->print("idom[%d] ", i); |
| 6013 | n->dump(); |
| 6014 | n = idom(n); |
| 6015 | } |
| 6016 | } |
| 6017 | } |
| 6018 | #endif // NOT PRODUCT |
| 6019 | |
| 6020 | // Collect a R-P-O for the whole CFG. |
| 6021 | // Result list is in post-order (scan backwards for RPO) |
| 6022 | void PhaseIdealLoop::rpo(Node* start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list) const { |
| 6023 | stk.push(start, 0); |
| 6024 | visited.set(start->_idx); |
| 6025 | |
| 6026 | while (stk.is_nonempty()) { |
| 6027 | Node* m = stk.node(); |
| 6028 | uint idx = stk.index(); |
| 6029 | if (idx < m->outcnt()) { |
| 6030 | stk.set_index(idx + 1); |
| 6031 | Node* n = m->raw_out(idx); |
| 6032 | if (n->is_CFG() && !visited.test_set(n->_idx)) { |
| 6033 | stk.push(n, 0); |
| 6034 | } |
| 6035 | } else { |
| 6036 | rpo_list.push(m); |
| 6037 | stk.pop(); |
| 6038 | } |
| 6039 | } |
| 6040 | } |
| 6041 | |
| 6042 | |
| 6043 | //============================================================================= |
| 6044 | //------------------------------LoopTreeIterator------------------------------- |
| 6045 | |
| 6046 | // Advance to next loop tree using a preorder, left-to-right traversal. |
| 6047 | void LoopTreeIterator::next() { |
| 6048 | assert(!done(), "must not be done.")do { if (!(!done())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 6048, "assert(" "!done()" ") failed", "must not be done."); ::breakpoint(); } } while (0); |
| 6049 | if (_curnt->_child != NULL__null) { |
| 6050 | _curnt = _curnt->_child; |
| 6051 | } else if (_curnt->_next != NULL__null) { |
| 6052 | _curnt = _curnt->_next; |
| 6053 | } else { |
| 6054 | while (_curnt != _root && _curnt->_next == NULL__null) { |
| 6055 | _curnt = _curnt->_parent; |
| 6056 | } |
| 6057 | if (_curnt == _root) { |
| 6058 | _curnt = NULL__null; |
| 6059 | assert(done(), "must be done.")do { if (!(done())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 6059, "assert(" "done()" ") failed", "must be done."); ::breakpoint (); } } while (0); |
| 6060 | } else { |
| 6061 | assert(_curnt->_next != NULL, "must be more to do")do { if (!(_curnt->_next != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 6061, "assert(" "_curnt->_next != __null" ") failed", "must be more to do" ); ::breakpoint(); } } while (0); |
| 6062 | _curnt = _curnt->_next; |
| 6063 | } |
| 6064 | } |
| 6065 | } |